delete imported users / add error code / sync import with pagination
MODIFY
src/assets/server/rest/v1/schemas/tag/tag-create-req.json
src/assets/server/rest/v1/schemas/tag/tag-create-req.json
"type": "object",
"properties": {
"id": {
- "type": "string"
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 100,
+ "pattern": "^[a-zA-Z0-9]*$"
},
"description": {
"type": "string"
- },
- "issuer": {
- "type": "boolean"
- },
- "deleted": {
- "type": "boolean"
- },
- "active": {
- "type": "boolean"
- },
- "createdOn": {
- "format": "date"
- },
- "createdBy": {
- "type": "object"
}
},
"required": [
"id",
- "issuer",
- "active"
+ "description"
]
}

MODIFY
src/assets/server/rest/v1/schemas/user/user-create-req.json
src/assets/server/rest/v1/schemas/user/user-create-req.json
"minLength": 1,
"maxLength": 150
},
- "createdOn": {
- "format": "date"
- },
- "createdBy": {
- "type": "object"
- },
"locale": {
"type": "string"
},

MODIFY
src/migration/tasks/AddDescriptionToTagsTask.ts
src/migration/tasks/AddDescriptionToTagsTask.ts
@@ -33,13 +33,13 @@ export default class AddDescriptionToTagsTask extends MigrationTask {
for (const tagMDB of tagsMDB) {
await global.database.getCollection<any>(tenant.id, 'tags').findOneAndUpdate(
{ _id: tagMDB._id },
- { $set: { description: `Tag ID '${tagMDB._id}'` } });
+ { $set: { description: `Badge ID '${tagMDB._id}'` } });
updated++;
}
}
// Log in the default tenant
if (updated > 0) {
- Logging.logDebug({
+ await Logging.logDebug({
tenantID: Constants.DEFAULT_TENANT,
module: MODULE_NAME, method: 'migrateTenant',
action: ServerAction.MIGRATION,
@@ -49,7 +49,7 @@ export default class AddDescriptionToTagsTask extends MigrationTask {
}
getVersion(): string {
- return '1.0';
+ return '1.1';
}
getName(): string {

MODIFY
src/scheduler/tasks/SynchronizeTagsImportTask.ts
src/scheduler/tasks/SynchronizeTagsImportTask.ts
import Constants from '../../utils/Constants';
-import { ImportStatus } from '../../types/GlobalType';
+import DbParams from '../../types/database/DbParams';
+import { HTTPError } from '../../types/HTTPError';
import { LockEntity } from '../../types/Locking';
import LockingManager from '../../locking/LockingManager';
import Logging from '../../utils/Logging';
@@ -7,7 +8,6 @@ import SchedulerTask from '../SchedulerTask';
import { ServerAction } from '../../types/Server';
import Tag from '../../types/Tag';
import TagStorage from '../../storage/mongodb/TagStorage';
-import TagValidator from '../../server/rest/v1/validator/TagValidation';
import { TaskConfig } from '../../types/TaskConfig';
import Tenant from '../../types/Tenant';
@@ -18,13 +18,22 @@ export default class SynchronizeTagsImportTask extends SchedulerTask {
const synchronizeTagsImport = LockingManager.createExclusiveLock(tenant.id, LockEntity.TAGS, 'synchronize-tags-import');
if (await LockingManager.acquire(synchronizeTagsImport)) {
try {
- const importedTags = await TagStorage.getImportedTags(tenant.id, { statuses: [ImportStatus.UNKNOWN] }, Constants.DB_PARAMS_MAX_LIMIT);
- if (importedTags.count !== 0) {
+ const dbParams: DbParams = { limit: Constants.EXPORT_PAGE_SIZE, skip: 0, onlyRecordCount: true };
+ let importedTags = await TagStorage.getImportedTags(tenant.id, { withNoError: true }, dbParams);
+ let count = importedTags.count;
+ delete dbParams.onlyRecordCount;
+ let skip = 0;
+ // Limit the number of records
+ if (count > Constants.EXPORT_RECORD_MAX_COUNT) {
+ count = Constants.EXPORT_RECORD_MAX_COUNT;
+ }
+ do {
+ importedTags = await TagStorage.getImportedTags(tenant.id, { withNoError: true }, dbParams);
for (const importedTag of importedTags.result) {
const foundTag = await TagStorage.getTag(tenant.id, importedTag.id);
if (foundTag) {
- importedTag.status = ImportStatus.ERROR;
- importedTag.error = 'Tag id already exists';
+ importedTag.errorCode = HTTPError.TAG_ALREADY_EXIST_ERROR;
+ importedTag.errorDescription = `Tag with ID '${foundTag.id}' already exists`;
await TagStorage.saveImportedTag(tenant.id, importedTag);
} else {
const tag: Tag = {
@@ -32,15 +41,13 @@ export default class SynchronizeTagsImportTask extends SchedulerTask {
description: importedTag.description,
issuer: true,
deleted: false,
- active: true,
+ active: false,
createdBy: { id: importedTag.importedBy },
- createdOn: new Date()
+ createdOn: importedTag.importedOn
};
try {
- TagValidator.getInstance().validateTagCreation(tag);
await TagStorage.saveTag(tenant.id, tag);
- importedTag.status = ImportStatus.IMPORTED;
- await TagStorage.saveImportedTag(tenant.id, importedTag);
+ await TagStorage.deleteImportedTag(tenant.id, importedTag.id);
await Logging.logDebug({
tenantID: tenant.id,
action: ServerAction.SYNCHRONIZE_TAGS,
@@ -48,8 +55,8 @@ export default class SynchronizeTagsImportTask extends SchedulerTask {
message: `Tag with id: ${importedTag.id} have been created in Tenant ${tenant.name}`
});
} catch (error) {
- importedTag.status = ImportStatus.ERROR;
- importedTag.error = error.message;
+ importedTag.errorCode = HTTPError.GENERAL_ERROR;
+ importedTag.errorDescription = error.message;
await TagStorage.saveImportedTag(tenant.id, importedTag);
// Error
await Logging.logError({
@@ -62,7 +69,8 @@ export default class SynchronizeTagsImportTask extends SchedulerTask {
}
}
}
- }
+ skip += Constants.EXPORT_PAGE_SIZE;
+ } while (skip < count);
} catch (error) {
// Log error
await Logging.logActionExceptionMessage(tenant.id, ServerAction.SYNCHRONIZE_TAGS, error);

MODIFY
src/scheduler/tasks/SynchronizeUsersImportTask.ts
src/scheduler/tasks/SynchronizeUsersImportTask.ts
import User, { UserRole, UserStatus } from '../../types/User';
import Constants from '../../utils/Constants';
-import { ImportStatus } from '../../types/GlobalType';
+import DbParams from '../../types/database/DbParams';
+import { HTTPError } from '../../types/HTTPError';
import { LockEntity } from '../../types/Locking';
import LockingManager from '../../locking/LockingManager';
import Logging from '../../utils/Logging';
@@ -10,7 +11,6 @@ import { ServerAction } from '../../types/Server';
import { TaskConfig } from '../../types/TaskConfig';
import Tenant from '../../types/Tenant';
import UserStorage from '../../storage/mongodb/UserStorage';
-import UserValidator from '../../server/rest/v1/validator/UserValidation';
const MODULE_NAME = 'SynchronizeUsersImportTask';
@@ -19,13 +19,22 @@ export default class SynchronizeUsersImportTask extends SchedulerTask {
const synchronizeUsersImport = LockingManager.createExclusiveLock(tenant.id, LockEntity.USER, 'synchronize-users-import');
if (await LockingManager.acquire(synchronizeUsersImport)) {
try {
- const importedUsers = await UserStorage.getImportedUsers(tenant.id, { statuses: [ImportStatus.UNKNOWN] }, Constants.DB_PARAMS_MAX_LIMIT);
- if (importedUsers.count !== 0) {
+ const dbParams: DbParams = { limit: Constants.EXPORT_PAGE_SIZE, skip: 0, onlyRecordCount: true };
+ let importedUsers = await UserStorage.getImportedUsers(tenant.id, { withNoError: true }, dbParams);
+ let count = importedUsers.count;
+ delete dbParams.onlyRecordCount;
+ let skip = 0;
+ // Limit the number of records
+ if (count > Constants.EXPORT_RECORD_MAX_COUNT) {
+ count = Constants.EXPORT_RECORD_MAX_COUNT;
+ }
+ do {
+ importedUsers = await UserStorage.getImportedUsers(tenant.id, { withNoError: true }, dbParams);
for (const importedUser of importedUsers.result) {
const foundUser = await UserStorage.getUserByEmail(tenant.id, importedUser.email);
if (foundUser) {
- importedUser.status = ImportStatus.ERROR;
- importedUser.error = 'Email already exists';
+ importedUser.errorCode = HTTPError.USER_EMAIL_ALREADY_EXIST_ERROR;
+ importedUser.errorDescription = 'Email already exists';
await UserStorage.saveImportedUser(tenant.id, importedUser);
} else {
const user: Partial<User> = {
@@ -42,14 +51,12 @@ export default class SynchronizeUsersImportTask extends SchedulerTask {
notificationsActive: true
};
try {
- UserValidator.getInstance().validateUserCreation(user);
user.id = await UserStorage.saveUser(tenant.id, user);
// Role need to be set separately
await UserStorage.saveUserRole(tenant.id, user.id, UserRole.BASIC);
// Status need to be set separately
await UserStorage.saveUserStatus(tenant.id, user.id, UserStatus.ACTIVE);
- importedUser.status = ImportStatus.IMPORTED;
- await UserStorage.saveImportedUser(tenant.id, importedUser);
+ await UserStorage.deleteImportedUser(tenant.id, importedUser.id);
await Logging.logDebug({
tenantID: tenant.id,
action: ServerAction.SYNCHRONIZE_USERS,
@@ -57,8 +64,8 @@ export default class SynchronizeUsersImportTask extends SchedulerTask {
message: `User with email: ${importedUser.email} have been created in Tenant ${tenant.name}`
});
} catch (error) {
- importedUser.status = ImportStatus.ERROR;
- importedUser.error = error.message;
+ importedUser.errorCode = HTTPError.GENERAL_ERROR;
+ importedUser.errorDescription = error.message;
await UserStorage.saveImportedUser(tenant.id, importedUser);
// Error
await Logging.logError({
@@ -71,7 +78,8 @@ export default class SynchronizeUsersImportTask extends SchedulerTask {
}
}
}
- }
+ skip += Constants.EXPORT_PAGE_SIZE;
+ } while (skip < count);
} catch (error) {
// Log error
await Logging.logActionExceptionMessage(tenant.id, ServerAction.SYNCHRONIZE_USERS, error);

MODIFY
src/server/rest/v1/service/TagService.ts
src/server/rest/v1/service/TagService.ts
import { Action, Entity } from '../../../../types/Authorization';
-import { ActionsResponse, ImportStatus } from '../../../../types/GlobalType';
import { HTTPAuthError, HTTPError } from '../../../../types/HTTPError';
import { NextFunction, Request, Response } from 'express';
import { OCPITokenType, OCPITokenWhitelist } from '../../../../types/ocpi/OCPIToken';
import Tag, { ImportedTag } from '../../../../types/Tag';
+import { ActionsResponse } from '../../../../types/GlobalType';
import AppAuthError from '../../../../exception/AppAuthError';
import AppError from '../../../../exception/AppError';
import AuthorizationService from './AuthorizationService';
@@ -20,6 +20,7 @@ import { ServerAction } from '../../../../types/Server';
import { StatusCodes } from 'http-status-codes';
import TagSecurity from './security/TagSecurity';
import TagStorage from '../../../../storage/mongodb/TagStorage';
+import TagValidator from '../validator/TagValidation';
import TenantComponents from '../../../../types/TenantComponents';
import TenantStorage from '../../../../storage/mongodb/TenantStorage';
import UserStorage from '../../../../storage/mongodb/UserStorage';
@@ -641,10 +642,10 @@ export default class TagService {
try {
const newUploadedTag: ImportedTag = {
id: tag.id.toUpperCase(),
- description: tag.description ? tag.description : `Tag ID '${tag.id}'`,
- importedBy: req.user.id,
- status: ImportStatus.UNKNOWN
+ description: tag.description ? tag.description : `Badge ID '${tag.id}'`,
};
+ TagValidator.getInstance().validateTagCreation(newUploadedTag);
+ newUploadedTag.importedBy = req.user.id;
await TagStorage.saveImportedTag(req.user.tenantID, newUploadedTag);
} catch (error) {
await Logging.logError({

MODIFY
src/server/rest/v1/service/UserService.ts
src/server/rest/v1/service/UserService.ts
@@ -19,7 +19,6 @@ import Cypher from '../../../../utils/Cypher';
import { DataResult } from '../../../../types/DataResult';
import EmspOCPIClient from '../../../../client/ocpi/EmspOCPIClient';
import I18nManager from '../../../../utils/I18nManager';
-import { ImportStatus } from '../../../../types/GlobalType';
import JSONStream from 'JSONStream';
import Logging from '../../../../utils/Logging';
import NotificationHandler from '../../../../notification/NotificationHandler';
@@ -37,6 +36,7 @@ import { UserInErrorType } from '../../../../types/InError';
import UserNotifications from '../../../../types/UserNotifications';
import UserSecurity from './security/UserSecurity';
import UserStorage from '../../../../storage/mongodb/UserStorage';
+import UserValidator from '../validator/UserValidation';
import Utils from '../../../../utils/Utils';
import UtilsService from './UtilsService';
import csvToJson from 'csvtojson/v2';
@@ -1278,16 +1278,16 @@ export default class UserService {
try {
const newUploadedUser: ImportedUser = {
name: user.Name,
- firstName: user.First_Name,
+ firstName: user.FirstName,
email: user.Email,
- importedBy: req.user.id,
- status: ImportStatus.UNKNOWN
};
+ UserValidator.getInstance().validateUserCreation(newUploadedUser);
+ newUploadedUser.importedBy = req.user.id;
await UserStorage.saveImportedUser(req.user.tenantID, newUploadedUser);
} catch (error) {
await Logging.logError({
tenantID: req.user.tenantID,
- module: MODULE_NAME, method: 'handleUpdloadUsersFile',
+ module: MODULE_NAME, method: 'handleUploadUsersFile',
action: action,
message: 'User cannot be imported',
detailedMessages: { error: error.message, stack: error.stack }

MODIFY
src/storage/mongodb/TagStorage.ts
src/storage/mongodb/TagStorage.ts
import Tag, { ImportedTag } from '../../types/Tag';
-import global, { FilterParams, ImportStatus } from '../../types/GlobalType';
+import global, { FilterParams } from '../../types/GlobalType';
import Constants from '../../utils/Constants';
import { DataResult } from '../../types/DataResult';
@@ -39,15 +39,15 @@ export default class TagStorage {
await Logging.traceEnd(tenantID, MODULE_NAME, 'saveTag', uniqueTimerID, tagMDB);
}
- public static async saveImportedTag(tenantID: string, importedTagToSave: ImportedTag): Promise<void> {
+ public static async saveImportedTag(tenantID: string, importedTagToSave: ImportedTag): Promise<string> {
const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'saveImportedTag');
const tagMDB = {
_id: importedTagToSave.id,
description: importedTagToSave.description,
importedOn: new Date(),
importedBy: Utils.convertToObjectID(importedTagToSave.importedBy),
- status: importedTagToSave.status,
- error: importedTagToSave.error
+ errorCode: importedTagToSave.errorCode,
+ errorDescription: importedTagToSave.errorDescription
};
await global.database.getCollection<any>(tenantID, 'tagsImport').findOneAndUpdate(
{ _id: tagMDB._id },
@@ -56,11 +56,26 @@ export default class TagStorage {
);
// Debug
await Logging.traceEnd(tenantID, MODULE_NAME, 'saveImportedTag', uniqueTimerID, tagMDB);
+ return tagMDB._id;
+ }
+
+ public static async deleteImportedTag(tenantID: string, importedTagID: string): Promise<void> {
+ // Debug
+ const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'deleteImportedTag');
+ // Check Tenant
+ await DatabaseUtils.checkTenant(tenantID);
+ // Delete
+ await global.database.getCollection<any>(tenantID, 'tagsImport').deleteOne(
+ {
+ '_id': importedTagID,
+ });
+ // Debug
+ await Logging.traceEnd(tenantID, MODULE_NAME, 'deleteImportedTag', uniqueTimerID, { id: importedTagID });
}
public static async getImportedTags(tenantID: string,
params: {
- statuses?: ImportStatus[]; search?: string
+ withNoError?: boolean; search?: string
},
dbParams: DbParams, projectFields?: string[]): Promise<DataResult<ImportedTag>> {
// Debug
@@ -83,10 +98,12 @@ export default class TagStorage {
{ 'description': { $regex: params.search, $options: 'i' } }
];
}
-
- // Status (Previously getUsersInError)
- if (params.statuses && params.statuses.length > 0) {
- filters.status = { $in: params.statuses };
+ // Only entries with no error
+ if (params.withNoError) {
+ filters.$or = [
+ { 'errorCode': { '$exists': false } },
+ { 'errorCode': null }
+ ];
}
// Add filters
aggregation.push({

MODIFY
src/storage/mongodb/UserStorage.ts
src/storage/mongodb/UserStorage.ts
import Site, { SiteUser } from '../../types/Site';
import User, { ImportedUser, UserRole, UserStatus } from '../../types/User';
import { UserInError, UserInErrorType } from '../../types/InError';
-import global, { FilterParams, Image, ImportStatus } from '../../types/GlobalType';
+import global, { FilterParams, Image } from '../../types/GlobalType';
import BackendError from '../../exception/BackendError';
import { BillingUserData } from '../../types/Billing';
@@ -272,15 +272,15 @@ export default class UserStorage {
return userMDB._id.toHexString();
}
- public static async saveImportedUser(tenantID: string, importedUserToSave: ImportedUser): Promise<void> {
+ public static async saveImportedUser(tenantID: string, importedUserToSave: ImportedUser): Promise<string> {
const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'saveImportedUser');
const userMDB = {
_id: importedUserToSave.id ? Utils.convertToObjectID(importedUserToSave.id) : new ObjectID(),
email: importedUserToSave.email,
firstName: importedUserToSave.firstName,
name: importedUserToSave.name,
- status: importedUserToSave.status,
- error: importedUserToSave.error,
+ errorCode: importedUserToSave.errorCode,
+ errorDescription: importedUserToSave.errorDescription,
importedOn: new Date(),
importedBy: Utils.convertToObjectID(importedUserToSave.importedBy)
};
@@ -291,6 +291,21 @@ export default class UserStorage {
);
// Debug
await Logging.traceEnd(tenantID, MODULE_NAME, 'saveImportedUser', uniqueTimerID, userMDB);
+ return userMDB._id.toHexString();
+ }
+
+ public static async deleteImportedUser(tenantID: string, importedUserID: string): Promise<void> {
+ // Debug
+ const uniqueTimerID = Logging.traceStart(tenantID, MODULE_NAME, 'deleteImportedUser');
+ // Check Tenant
+ await DatabaseUtils.checkTenant(tenantID);
+ // Delete
+ await global.database.getCollection<any>(tenantID, 'usersImport').deleteOne(
+ {
+ '_id': importedUserID,
+ });
+ // Debug
+ await Logging.traceEnd(tenantID, MODULE_NAME, 'deleteImportedUser', uniqueTimerID, { id: importedUserID });
}
public static async saveUserPassword(tenantID: string, userID: string,
@@ -668,7 +683,7 @@ export default class UserStorage {
public static async getImportedUsers(tenantID: string,
params: {
- statuses?: ImportStatus[]; search?: string
+ withNoError?: boolean; search?: string
},
dbParams: DbParams, projectFields?: string[]): Promise<DataResult<ImportedUser>> {
// Debug
@@ -692,12 +707,15 @@ export default class UserStorage {
{ 'email': { $regex: params.search, $options: 'i' } }
];
}
+ // Only entries with no error
+ if (params.withNoError) {
+ filters.$or = [
+ { 'errorCode': { '$exists': false } },
+ { 'errorCode': null }
+ ];
+ }
// Remove deleted
filters.deleted = { '$ne': true };
- // Status (Previously getUsersInError)
- if (params.statuses && params.statuses.length > 0) {
- filters.status = { $in: params.statuses };
- }
// Add filters
aggregation.push({
$match: filters

MODIFY
src/types/GlobalType.ts
src/types/GlobalType.ts
@@ -21,12 +21,6 @@ export interface Image {
image: string;
}
-export enum ImportStatus {
- UNKNOWN = 'U',
- IMPORTED = 'I',
- ERROR = 'E',
-}
-
export interface FilterParams {
[param: string]: any | string[];
}

MODIFY
src/types/Tag.ts
src/types/Tag.ts
import CreatedUpdatedProps from './CreatedUpdatedProps';
-import { ImportStatus } from './GlobalType';
+import { HTTPError } from './HTTPError';
import { OCPIToken } from './ocpi/OCPIToken';
+import { StatusCodes } from 'http-status-codes';
import User from './User';
export default interface Tag extends CreatedUpdatedProps {
@@ -17,10 +18,10 @@ export default interface Tag extends CreatedUpdatedProps {
}
export interface ImportedTag {
- id?: string;
- description?: string;
- importedBy: string;
+ id: string;
+ description: string;
+ importedBy?: string;
importedOn?: Date;
- error?: string;
- status?: ImportStatus
+ errorCode?: HTTPError | StatusCodes;
+ errorDescription?: string;
}

MODIFY
src/types/User.ts
src/types/User.ts
@@ -2,7 +2,8 @@ import Address from './Address';
import { AuthorizationActions } from './Authorization';
import { BillingUserData } from './Billing';
import CreatedUpdatedProps from './CreatedUpdatedProps';
-import { ImportStatus } from './GlobalType';
+import { HTTPError } from './HTTPError';
+import { StatusCodes } from 'http-status-codes';
import UserNotifications from './UserNotifications';
export default interface User extends CreatedUpdatedProps, AuthorizationActions {
@@ -62,10 +63,10 @@ export interface ImportedUser {
name: string;
firstName: string;
email: string;
- importedBy: string;
+ importedBy?: string;
importedOn?: Date;
- error?: string;
- status?: ImportStatus
+ errorCode?: HTTPError | StatusCodes;
+ errorDescription?: string;
}
export enum UserStatus {

delete imported users / add error code / sync import with pagination