x10 perfs by processing blocks of 250 records. Handled HTTP connection is closed during the upload. Used constants for CSV separator. Returned specific HTTP error code for missing CSV required header. Return to front-end the number of successful/failed import. Improve error handling.
MODIFY
src/scheduler/tasks/SynchronizeTagsImportTask.ts
src/scheduler/tasks/SynchronizeTagsImportTask.ts
@@ -40,7 +40,7 @@ export default class SynchronizeTagsImportTask extends SchedulerTask {
tenantID: tenant.id,
action: ServerAction.SYNCHRONIZE_TAGS,
module: MODULE_NAME, method: 'processTenant',
- message: `Tag with id: ${importedTag.id} have been updated successfully in Tenant ${Utils.buildTenantName(tenant)}`
+ message: `Tag ID '${importedTag.id}' has been updated successfully in Tenant ${Utils.buildTenantName(tenant)}`
});
continue;
}
@@ -63,7 +63,7 @@ export default class SynchronizeTagsImportTask extends SchedulerTask {
tenantID: tenant.id,
action: ServerAction.SYNCHRONIZE_TAGS,
module: MODULE_NAME, method: 'processTenant',
- message: `Tag with id: ${importedTag.id} have been created in Tenant ${Utils.buildTenantName(tenant)}`
+ message: `Tag ID '${importedTag.id}' have been created in Tenant ${Utils.buildTenantName(tenant)}`
});
} catch (error) {
// Update the imported Tag

MODIFY
src/scheduler/tasks/SynchronizeUsersImportTask.ts
src/scheduler/tasks/SynchronizeUsersImportTask.ts
@@ -42,7 +42,7 @@ export default class SynchronizeUsersImportTask extends SchedulerTask {
tenantID: tenant.id,
action: ServerAction.SYNCHRONIZE_USERS,
module: MODULE_NAME, method: 'processTenant',
- message: `User with email: ${importedUser.email} have been updated successfully in Tenant ${Utils.buildTenantName(tenant)}`
+ message: `User with email '${importedUser.email}' have been updated successfully in Tenant ${Utils.buildTenantName(tenant)}`
});
continue;
}
@@ -73,7 +73,7 @@ export default class SynchronizeUsersImportTask extends SchedulerTask {
tenantID: tenant.id,
action: ServerAction.SYNCHRONIZE_USERS,
module: MODULE_NAME, method: 'processTenant',
- message: `User with email: ${importedUser.email} have been created in Tenant ${Utils.buildTenantName(tenant)}`
+ message: `User with email '${importedUser.email}' have been created in Tenant ${Utils.buildTenantName(tenant)}`
});
} catch (error) {
importedUser.status = ImportStatus.ERROR;
@@ -85,7 +85,7 @@ export default class SynchronizeUsersImportTask extends SchedulerTask {
tenantID: tenant.id,
action: ServerAction.SYNCHRONIZE_USERS,
module: MODULE_NAME, method: 'processTenant',
- message: `An error occurred when importing user with email: ${importedUser.email}`,
+ message: `An error occurred when importing user with email '${importedUser.email}'`,
detailedMessages: { error: error.message, stack: error.stack }
});
}

MODIFY
src/server/rest/v1/service/TagService.ts
src/server/rest/v1/service/TagService.ts
@@ -476,48 +476,59 @@ export default class TagService {
// Default values for Tag import
const importedBy = req.user.id;
const importedOn = new Date();
+ const tagsToBeImported: ImportedTag[] = [];
const result: ActionsResponse = {
inSuccess: 0,
inError: 0
};
+ // Delete all previously imported tags
+ await TagStorage.deleteImportedTags(req.user.tenantID);
+ // Get the stream
const busboy = new Busboy({ headers: req.headers });
req.pipe(busboy);
+ // Handle closed socket
+ let connectionClosed = false;
+ req.socket.on('close', () => {
+ connectionClosed = true;
+ });
// eslint-disable-next-line @typescript-eslint/no-misused-promises
busboy.on('file', async (fieldname: string, file: any, filename: string, encoding: string, mimetype: string) => {
if (mimetype === 'text/csv') {
const converter = csvToJson({
trim: true,
- delimiter: ['\t'],
- quote: 'off'
+ delimiter: Constants.CSV_SEPARATOR,
+ output: 'json',
+ quote: 'on',
});
void converter.subscribe(async (tag: ImportedTag) => {
+ // Check connection
+ if (connectionClosed) {
+ throw new Error('HTTP connection has been closed');
+ }
// Check the format of the first entry
if (!result.inSuccess && !result.inError) {
- if (!TagRequiredImportProperties.every((property) => Object.keys(tag).includes(property))) {
- await Logging.logError({
- tenantID: req.user.tenantID,
- module: MODULE_NAME, method: 'handleImportTags',
- action: action,
- user: req.user.id,
- message: `Invalid Csv file '${filename}', all properties: '${TagRequiredImportProperties.join(', ')}' are required`,
- });
+ // Check header
+ const tagKeys = Object.keys(tag);
+ if (!TagRequiredImportProperties.every((property) => tagKeys.includes(property))) {
if (!res.headersSent) {
- res.writeHead(HTTPError.INVALID_FILE_FORMAT);
+ res.writeHead(HTTPError.INVALID_FILE_CSV_HEADER_FORMAT);
res.end();
}
- return;
+ throw new Error(`Missing one of required properties: '${TagRequiredImportProperties.join(', ')}'`);
}
}
// Set default value
tag.importedBy = importedBy;
tag.importedOn = importedOn;
// Import
- const importSuccess = await TagService.importTag(action, req, tag);
- if (importSuccess) {
- result.inSuccess++;
- } else {
+ const importSuccess = await TagService.processTag(action, req, tag, tagsToBeImported);
+ if (!importSuccess) {
result.inError++;
}
+ // Insert batched
+ if ((tagsToBeImported.length % Constants.IMPORT_BATCH_INSERT_SIZE) === 0) {
+ await TagService.insertTags(req.user.tenantID, req.user, action, tagsToBeImported, result);
+ }
// eslint-disable-next-line @typescript-eslint/no-misused-promises
}, async (error) => {
await Logging.logError({
@@ -525,7 +536,7 @@ export default class TagService {
module: MODULE_NAME, method: 'handleImportTags',
action: action,
user: req.user.id,
- message: `Invalid Csv file '${filename}'`,
+ message: `Exception while parsing the CSV '${filename}': ${error.message}`,
detailedMessages: { error: error.message, stack: error.stack }
});
if (!res.headersSent) {
@@ -533,6 +544,7 @@ export default class TagService {
res.end();
}
});
+ // Start processing the file
void file.pipe(converter);
} else if (mimetype === 'application/json') {
const parser = JSONStream.parse('tags.*');
@@ -542,12 +554,14 @@ export default class TagService {
tag.importedBy = importedBy;
tag.importedOn = importedOn;
// Import
- const importSuccess = await TagService.importTag(action, req, tag);
- if (importSuccess) {
- result.inSuccess++;
- } else {
+ const importSuccess = await TagService.processTag(action, req, tag, tagsToBeImported);
+ if (!importSuccess) {
result.inError++;
}
+ // Insert batched
+ if ((tagsToBeImported.length % Constants.IMPORT_BATCH_INSERT_SIZE) === 0) {
+ await TagService.insertTags(req.user.tenantID, req.user, action, tagsToBeImported, result);
+ }
});
// eslint-disable-next-line @typescript-eslint/no-misused-promises
parser.on('error', async (error) => {
@@ -589,11 +603,35 @@ export default class TagService {
'{{inSuccess}} Tag(s) were successfully uploaded and ready for asynchronous import and {{inError}} failed to be uploaded',
'No Tag have been uploaded', req.user
);
- res.end();
+ // Insert batched
+ if (tagsToBeImported.length > 0) {
+ await TagService.insertTags(req.user.tenantID, req.user, action, tagsToBeImported, result);
+ }
+ res.json({ ...result, ...Constants.REST_RESPONSE_SUCCESS });
next();
});
}
+ private static async insertTags(tenantID: string, user: UserToken, action: ServerAction, tagsToBeImported: ImportedTag[], result: ActionsResponse): Promise<void> {
+ try {
+ const nbrInsertedTags = await TagStorage.saveImportedTags(tenantID, tagsToBeImported);
+ result.inSuccess += nbrInsertedTags;
+ } catch (error) {
+ // Handle dup keys
+ result.inSuccess += error.result.nInserted;
+ result.inError += error.writeErrors.length;
+ await Logging.logError({
+ tenantID: tenantID,
+ module: MODULE_NAME, method: 'insertTags',
+ action: action,
+ user: user.id,
+ message: `Cannot import '${error.writeErrors.length as number}' tags!`,
+ detailedMessages: { error: error.message, stack: error.stack, tagsError: error.writeErrors }
+ });
+ }
+ tagsToBeImported.length = 0;
+ }
+
private static async deleteTags(action: ServerAction, loggedUser: UserToken, tagsIDs: string[]): Promise<ActionsResponse> {
const result: ActionsResponse = {
inSuccess: 0,
@@ -693,7 +731,7 @@ export default class TagService {
return result;
}
- private static async importTag(action: ServerAction, req: Request, importedTag: ImportedTag): Promise<boolean> {
+ private static async processTag(action: ServerAction, req: Request, importedTag: ImportedTag, tagsToBeImported: ImportedTag[]): Promise<boolean> {
try {
const newImportedTag: ImportedTag = {
id: importedTag.id.toUpperCase(),
@@ -705,15 +743,15 @@ export default class TagService {
newImportedTag.importedBy = importedTag.importedBy;
newImportedTag.importedOn = importedTag.importedOn;
newImportedTag.status = ImportStatus.READY;
- // Save it for import
- await TagStorage.saveImportedTag(req.user.tenantID, newImportedTag);
+ // Save it later on
+ tagsToBeImported.push(newImportedTag);
return true;
} catch (error) {
await Logging.logError({
tenantID: req.user.tenantID,
module: MODULE_NAME, method: 'importTag',
action: action,
- message: 'Tag cannot be imported',
+ message: `Tag ID '${importedTag.id}' cannot be imported`,
detailedMessages: { tag: importedTag, error: error.message, stack: error.stack }
});
return false;

MODIFY
src/server/rest/v1/service/UserService.ts
src/server/rest/v1/service/UserService.ts
@@ -37,6 +37,7 @@ import { UserInErrorType } from '../../../../types/InError';
import UserNotifications from '../../../../types/UserNotifications';
import UserSecurity from './security/UserSecurity';
import UserStorage from '../../../../storage/mongodb/UserStorage';
+import UserToken from '../../../../types/UserToken';
import UserValidator from '../validator/UserValidation';
import Utils from '../../../../utils/Utils';
import UtilsService from './UtilsService';
@@ -874,48 +875,59 @@ export default class UserService {
// Default values for User import
const importedBy = req.user.id;
const importedOn = new Date();
+ const usersToBeImported: ImportedUser[] = [];
const result: ActionsResponse = {
inSuccess: 0,
inError: 0
};
+ // Delete all previously imported tags
+ await UserStorage.deleteImportedUsers(req.user.tenantID);
+ // Get the stream
const busboy = new Busboy({ headers: req.headers });
req.pipe(busboy);
+ // Handle closed socket
+ let connectionClosed = false;
+ req.socket.on('close', () => {
+ connectionClosed = true;
+ });
// eslint-disable-next-line @typescript-eslint/no-misused-promises
busboy.on('file', async (fieldname, file, filename, encoding, mimetype) => {
if (mimetype === 'text/csv') {
const converter = csvToJson({
trim: true,
- delimiter: ['\t'],
- quote: 'off'
+ delimiter: Constants.CSV_SEPARATOR,
+ output: 'json',
+ quote: 'on',
});
void converter.subscribe(async (user: ImportedUser) => {
+ // Check connection
+ if (connectionClosed) {
+ throw new Error('HTTP connection has been closed');
+ }
// Check the format of the first entry
if (!result.inSuccess && !result.inError) {
- if (!UserRequiredImportProperties.every((property) => Object.keys(user).includes(property))) {
- await Logging.logError({
- tenantID: req.user.tenantID,
- module: MODULE_NAME, method: 'handleImportUsers',
- action: action,
- user: req.user.id,
- message: `Invalid Csv file '${filename}', all properties: '${UserRequiredImportProperties.join(', ')}' are required`,
- });
+ // Check header
+ const userKeys = Object.keys(user);
+ if (!UserRequiredImportProperties.every((property) => userKeys.includes(property))) {
if (!res.headersSent) {
- res.writeHead(HTTPError.INVALID_FILE_FORMAT);
+ res.writeHead(HTTPError.INVALID_FILE_CSV_HEADER_FORMAT);
res.end();
}
- return;
+ throw new Error(`Missing one of required properties: '${UserRequiredImportProperties.join(', ')}'`);
}
}
// Set default value
user.importedBy = importedBy;
user.importedOn = importedOn;
// Import
- const importSuccess = await UserService.importUser(action, req, user);
- if (importSuccess) {
- result.inSuccess++;
- } else {
+ const importSuccess = await UserService.processUser(action, req, user);
+ if (!importSuccess) {
result.inError++;
}
+ // Insert batched
+ if ((usersToBeImported.length % Constants.IMPORT_BATCH_INSERT_SIZE) === 0) {
+ await UserService.insertUsers(req.user.tenantID, req.user, action, usersToBeImported, result);
+ }
// eslint-disable-next-line @typescript-eslint/no-misused-promises
}, async (error) => {
await Logging.logError({
@@ -940,12 +952,14 @@ export default class UserService {
user.importedBy = importedBy;
user.importedOn = importedOn;
// Import
- const importSuccess = await UserService.importUser(action, req, user);
- if (importSuccess) {
- result.inSuccess++;
- } else {
+ const importSuccess = await UserService.processUser(action, req, user);
+ if (!importSuccess) {
result.inError++;
}
+ // Insert batched
+ if ((usersToBeImported.length % Constants.IMPORT_BATCH_INSERT_SIZE) === 0) {
+ await UserService.insertUsers(req.user.tenantID, req.user, action, usersToBeImported, result);
+ }
});
// eslint-disable-next-line @typescript-eslint/no-misused-promises
parser.on('error', async (error) => {
@@ -987,7 +1001,11 @@ export default class UserService {
'{{inSuccess}} User(s) were successfully uploaded and ready for asynchronous import and {{inError}} failed to be uploaded',
'No User have been uploaded', req.user
);
- res.end();
+ // Insert batched
+ if (usersToBeImported.length > 0) {
+ await UserService.insertUsers(req.user.tenantID, req.user, action, usersToBeImported, result);
+ }
+ res.json({ ...result, ...Constants.REST_RESPONSE_SUCCESS });
next();
});
}
@@ -1252,6 +1270,26 @@ export default class UserService {
}
}
+ private static async insertUsers(tenantID: string, user: UserToken, action: ServerAction, usersToBeImported: ImportedUser[], result: ActionsResponse): Promise<void> {
+ try {
+ const nbrInsertedTags = await UserStorage.saveImportedUsers(tenantID, usersToBeImported);
+ result.inSuccess += nbrInsertedTags;
+ } catch (error) {
+ // Handle dup keys
+ result.inSuccess += error.result.nInserted;
+ result.inError += error.writeErrors.length;
+ await Logging.logError({
+ tenantID: tenantID,
+ module: MODULE_NAME, method: 'insertUsers',
+ action: action,
+ user: user.id,
+ message: `Cannot import '${error.writeErrors.length as number}' users!`,
+ detailedMessages: { error: error.message, stack: error.stack, tagsError: error.writeErrors }
+ });
+ }
+ usersToBeImported.length = 0;
+ }
+
private static convertToCSV(req: Request, users: User[], writeHeader = true): string {
let csv = '';
const i18nManager = I18nManager.getInstanceForLocale(req.user.locale);
@@ -1339,7 +1377,7 @@ export default class UserService {
return users;
}
- private static async importUser(action: ServerAction, req: Request, importedUser: ImportedUser): Promise<boolean> {
+ private static async processUser(action: ServerAction, req: Request, importedUser: ImportedUser): Promise<boolean> {
try {
const newImportedUser: ImportedUser = {
name: importedUser.name.toUpperCase(),

MODIFY
src/server/rest/v1/service/UtilsService.ts
src/server/rest/v1/service/UtilsService.ts
@@ -282,7 +282,7 @@ export default class UtilsService {
}
// Handle closed socket
let connectionClosed = false;
- req.connection.on('close', () => {
+ req.socket.on('close', () => {
connectionClosed = true;
});
do {
@@ -348,6 +348,15 @@ export default class UtilsService {
public static checkIfChargingProfileIsValid(chargingStation: ChargingStation, chargePoint: ChargePoint,
filteredRequest: ChargingProfile, req: Request): void {
+ if (req.method !== 'POST' && !filteredRequest.id) {
+ throw new AppError({
+ source: Constants.CENTRAL_SERVER,
+ errorCode: HTTPError.GENERAL_ERROR,
+ message: 'The Charging Profile ID is mandatory',
+ module: MODULE_NAME,
+ method: 'checkIfOICPEndpointValid'
+ });
+ }
if (!Utils.objectHasProperty(filteredRequest, 'chargingStationID')) {
throw new AppError({
source: Constants.CENTRAL_SERVER,

MODIFY
src/storage/mongodb/CarStorage.ts
src/storage/mongodb/CarStorage.ts
@@ -558,7 +558,10 @@ export default class CarStorage {
carUsersMDB.push(carUserMDB);
}
// Execute
- await global.database.getCollection<any>(tenantID, 'carusers').insertMany(carUsersMDB);
+ await global.database.getCollection<any>(tenantID, 'carusers').insertMany(
+ carUsersMDB,
+ { ordered: false }
+ );
}
// Debug
await Logging.traceEnd(tenantID, MODULE_NAME, 'insertCarUsers', uniqueTimerID, carUsersMDB);

MODIFY
src/storage/mongodb/TagStorage.ts
src/storage/mongodb/TagStorage.ts
@@ -49,7 +49,7 @@ export default class TagStorage {
importedOn: Utils.convertToDate(importedTagToSave.importedOn),
importedBy: Utils.convertToObjectID(importedTagToSave.importedBy)
};
- await global.database.getCollection<any>(tenantID, 'importtags').findOneAndUpdate(
+ await global.database.getCollection<any>(tenantID, 'importedtags').findOneAndUpdate(
{ _id: tagMDB._id },
{ $set: tagMDB },
{ upsert: true, returnOriginal: false }
@@ -59,13 +59,33 @@ export default class TagStorage {
return tagMDB._id;
}
+ public static async saveImportedTags(tenantID: string, importedTagsToSave: ImportedTag[]): Promise<number> {
+ const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'saveImportedTags');
+ const importedTagsToSaveMDB: any = importedTagsToSave.map((importedTagToSave) => ({
+ _id: importedTagToSave.id,
+ description: importedTagToSave.description,
+ status: importedTagToSave.status,
+ errorDescription: importedTagToSave.errorDescription,
+ importedOn: Utils.convertToDate(importedTagToSave.importedOn),
+ importedBy: Utils.convertToObjectID(importedTagToSave.importedBy)
+ }));
+ // Insert all at once
+ const result = await global.database.getCollection<any>(tenantID, 'importedtags').insertMany(
+ importedTagsToSaveMDB,
+ { ordered: false }
+ );
+ // Debug
+ await Logging.traceEnd(tenantID, MODULE_NAME, 'saveImportedTags', uniqueTimerID);
+ return result.insertedCount;
+ }
+
public static async deleteImportedTag(tenantID: string, importedTagID: string): Promise<void> {
// Debug
const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'deleteImportedTag');
// Check Tenant
await DatabaseUtils.checkTenant(tenantID);
// Delete
- await global.database.getCollection<any>(tenantID, 'importtags').deleteOne(
+ await global.database.getCollection<any>(tenantID, 'importedtags').deleteOne(
{
'_id': importedTagID,
});
@@ -73,6 +93,17 @@ export default class TagStorage {
await Logging.traceEnd(tenantID, MODULE_NAME, 'deleteImportedTag', uniqueTimerID, { id: importedTagID });
}
+ public static async deleteImportedTags(tenantID: string): Promise<void> {
+ // Debug
+ const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'deleteImportedTags');
+ // Check Tenant
+ await DatabaseUtils.checkTenant(tenantID);
+ // Delete
+ await global.database.getCollection<any>(tenantID, 'importedtags').deleteMany({});
+ // Debug
+ await Logging.traceEnd(tenantID, MODULE_NAME, 'deleteImportedTags', uniqueTimerID);
+ }
+
public static async getImportedTags(tenantID: string,
params: { status?: ImportStatus; search?: string },
dbParams: DbParams, projectFields?: string[]): Promise<DataResult<ImportedTag>> {
@@ -110,7 +141,7 @@ export default class TagStorage {
aggregation.push({ $limit: Constants.DB_RECORD_COUNT_CEIL });
}
// Count Records
- const tagsImportCountMDB = await global.database.getCollection<any>(tenantID, 'importtags')
+ const tagsImportCountMDB = await global.database.getCollection<any>(tenantID, 'importedtags')
.aggregate([...aggregation, { $count: 'count' }], { allowDiskUse: true })
.toArray();
// Check if only the total count is requested
@@ -148,7 +179,7 @@ export default class TagStorage {
// Project
DatabaseUtils.projectFields(aggregation, projectFields);
// Read DB
- const tagsImportMDB = await global.database.getCollection<any>(tenantID, 'importtags')
+ const tagsImportMDB = await global.database.getCollection<any>(tenantID, 'importedtags')
.aggregate(aggregation, {
allowDiskUse: true
})

MODIFY
src/storage/mongodb/UserStorage.ts
src/storage/mongodb/UserStorage.ts
@@ -285,7 +285,7 @@ export default class UserStorage {
importedOn: Utils.convertToDate(importedUserToSave.importedOn),
importedBy: Utils.convertToObjectID(importedUserToSave.importedBy)
};
- await global.database.getCollection<any>(tenantID, 'importusers').findOneAndUpdate(
+ await global.database.getCollection<any>(tenantID, 'importedusers').findOneAndUpdate(
{ _id: userMDB._id },
{ $set: userMDB },
{ upsert: true, returnOriginal: false }
@@ -295,13 +295,35 @@ export default class UserStorage {
return userMDB._id.toHexString();
}
+ public static async saveImportedUsers(tenantID: string, importedUsersToSave: ImportedUser[]): Promise<number> {
+ const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'saveImportedUsers');
+ const importedUsersToSaveMDB: any = importedUsersToSave.map((importedUserToSave) => ({
+ _id: importedUserToSave.id ? Utils.convertToObjectID(importedUserToSave.id) : new ObjectID(),
+ email: importedUserToSave.email,
+ firstName: importedUserToSave.firstName,
+ name: importedUserToSave.name,
+ status: importedUserToSave.status,
+ errorDescription: importedUserToSave.errorDescription,
+ importedOn: Utils.convertToDate(importedUserToSave.importedOn),
+ importedBy: Utils.convertToObjectID(importedUserToSave.importedBy)
+ }));
+ // Insert all at once
+ const result = await global.database.getCollection<any>(tenantID, 'importedusers').insertMany(
+ importedUsersToSaveMDB,
+ { ordered: false }
+ );
+ // Debug
+ await Logging.traceEnd(tenantID, MODULE_NAME, 'saveImportedUsers', uniqueTimerID);
+ return result.insertedCount;
+ }
+
public static async deleteImportedUser(tenantID: string, importedUserID: string): Promise<void> {
// Debug
const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'deleteImportedUser');
// Check Tenant
await DatabaseUtils.checkTenant(tenantID);
// Delete
- await global.database.getCollection<any>(tenantID, 'importusers').deleteOne(
+ await global.database.getCollection<any>(tenantID, 'importedusers').deleteOne(
{
'_id': Utils.convertToObjectID(importedUserID),
});
@@ -309,6 +331,17 @@ export default class UserStorage {
await Logging.traceEnd(tenantID, MODULE_NAME, 'deleteImportedUser', uniqueTimerID, { id: importedUserID });
}
+ public static async deleteImportedUsers(tenantID: string): Promise<void> {
+ // Debug
+ const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'deleteImportedUsers');
+ // Check Tenant
+ await DatabaseUtils.checkTenant(tenantID);
+ // Delete
+ await global.database.getCollection<any>(tenantID, 'importedusers').deleteMany({});
+ // Debug
+ await Logging.traceEnd(tenantID, MODULE_NAME, 'deleteImportedUsers', uniqueTimerID);
+ }
+
public static async saveUserPassword(tenantID: string, userID: string,
params: {
password?: string; passwordResetHash?: string; passwordWrongNbrTrials?: number;
@@ -720,7 +753,7 @@ export default class UserStorage {
aggregation.push({ $limit: Constants.DB_RECORD_COUNT_CEIL });
}
// Count Records
- const usersImportCountMDB = await global.database.getCollection<any>(tenantID, 'importusers')
+ const usersImportCountMDB = await global.database.getCollection<any>(tenantID, 'importedusers')
.aggregate([...aggregation, { $count: 'count' }], { allowDiskUse: true })
.toArray();
// Check if only the total count is requested
@@ -758,7 +791,7 @@ export default class UserStorage {
// Project
DatabaseUtils.projectFields(aggregation, projectFields);
// Read DB
- const usersImportMDB = await global.database.getCollection<any>(tenantID, 'importusers')
+ const usersImportMDB = await global.database.getCollection<any>(tenantID, 'importedusers')
.aggregate(aggregation, {
allowDiskUse: true
})

MODIFY
src/types/HTTPError.ts
src/types/HTTPError.ts
@@ -65,6 +65,7 @@ export enum HTTPError {
CANNOT_ACQUIRE_LOCK = 510, // FIXME: override an already defined HTTP status code: Not extended (RFC 2774)
INVALID_FILE_FORMAT = 583,
+ INVALID_FILE_CSV_HEADER_FORMAT = 584,
CHARGE_POINT_NOT_VALID = 584,
}

MODIFY
src/utils/Constants.ts
src/utils/Constants.ts
@@ -7,7 +7,7 @@ import Tenant from '../types/Tenant';
export default class Constants {
public static readonly BOOT_NOTIFICATION_WAIT_TIME = 60;
- public static readonly CSV_SEPARATOR = '\t'; // Cannot store regex in enum
+ public static readonly CSV_SEPARATOR = ','; // Cannot store regex in enum
public static readonly PERF_MAX_DATA_VOLUME_KB = 128;
public static readonly PERF_MAX_RESPONSE_TIME_MILLIS = 500;
@@ -32,6 +32,7 @@ export default class Constants {
public static readonly EXPORT_PAGE_SIZE = 1000;
public static readonly EXPORT_RECORD_MAX_COUNT = 100000;
public static readonly IMPORT_PAGE_SIZE = 1000;
+ public static readonly IMPORT_BATCH_INSERT_SIZE = 250;
public static readonly DEFAULT_TENANT = 'default';
public static readonly DEFAULT_TENANT_OBJECT = Object.freeze({

MODIFY
src/utils/Logging.ts
src/utils/Logging.ts
@@ -48,7 +48,7 @@ export default class Logging {
// Debug DB
public static async traceEnd(tenantID: string, module: string, method: string, key: string, data: any = {}): Promise<void> {
- if (Utils.isDevelopmentEnv()) {
+ if (!Utils.isDevelopmentEnv()) {
// Compute duration if provided
let executionDurationMillis: number;
let found = false;

x10 perfs by processing blocks of 250 records. Handled HTTP connection is closed during the upload. Used constants for CSV separator. Returned specific HTTP error code for missing CSV required header. Return to front-end the number of successful/failed import. Improve error handling.