using csv to json and busboy
MODIFY
package.json
package.json
"cfenv": "^1.2.3",
"cors": "^2.8.5",
"country-language": "^0.1.7",
+ "csvtojson": "^2.0.10",
"ejs": "^3.1.5",
"emailjs": "^3.3.0",
"event-stream": "^4.0.1",

MODIFY
src/authorization/Authorizations.ts
src/authorization/Authorizations.ts
@@ -329,6 +329,10 @@ export default class Authorizations {
return Authorizations.canPerformAction(loggedUser, Entity.USER, Action.CREATE);
}
+ public static canImportUser(loggedUser: UserToken): boolean {
+ return Authorizations.canPerformAction(loggedUser, Entity.USERS, Action.IMPORT);
+ }
+
public static canUpdateUser(loggedUser: UserToken, userID: string): boolean {
return Authorizations.canPerformAction(loggedUser, Entity.USER, Action.UPDATE,
{ user: userID, owner: loggedUser.id });

MODIFY
src/authorization/AuthorizationsDefinition.ts
src/authorization/AuthorizationsDefinition.ts
@@ -25,7 +25,7 @@ const AUTHORIZATION_DEFINITION: AuthorizationDefinition = {
},
admin: {
grants: [
- { resource: Entity.USERS, action: [Action.LIST, Action.SYNCHRONIZE_BILLING_USERS], attributes: ['*'] },
+ { resource: Entity.USERS, action: [Action.LIST, Action.SYNCHRONIZE_BILLING_USERS, Action.IMPORT], attributes: ['*'] },
{ resource: Entity.USER, action: [Action.CREATE, Action.READ, Action.UPDATE, Action.SYNCHRONIZE_BILLING_USER], attributes: ['*'] },
{
resource: Entity.USER, action: Action.DELETE, attributes: ['*'],

MODIFY
src/server/rest/v1/service/UserService.ts
src/server/rest/v1/service/UserService.ts
import { Action, Entity } from '../../../../types/Authorization';
import { HTTPAuthError, HTTPError } from '../../../../types/HTTPError';
-import { NextFunction, Request, Response } from 'express';
+import { NextFunction, Request, Response, json } from 'express';
import { OCPITokenType, OCPITokenWhitelist } from '../../../../types/ocpi/OCPIToken';
import User, { UserStatus } from '../../../../types/User';
@@ -38,11 +38,14 @@ import UserSecurity from './security/UserSecurity';
import UserStorage from '../../../../storage/mongodb/UserStorage';
import Utils from '../../../../utils/Utils';
import UtilsService from './UtilsService';
+import csvStream from 'csv-stream';
+import csvToJson from 'csvtojson/v2';
import fs from 'fs';
import { inspect } from 'util';
import moment from 'moment';
import os from 'os';
import path from 'path';
+import { resolve } from 'bluebird';
import through from 'through';
const MODULE_NAME = 'UserService';
@@ -110,78 +113,6 @@ export default class UserService {
next();
}
- public static async import(action: ServerAction, req: Request, res: Response, next: NextFunction) {
-
- const jStream = JSONStream.parse('users.*');
-
- // Const upload = multer({ dest: 'uploads/' });
-
- const busboy = new Busboy({ headers: req.headers });
-
- // Stream.on('file', function(data) {
- // console.log('received:', data);
- // });
-
- res.send('success');
-
- jStream.on('data', async function(data) {
- // Await UserStorage.saveImportedUser(req.user.tenantID, data);
- return data.toString();
- });
-
-
- busboy.on('file', async function(fieldname, file, filename, encoding, mimetype) {
- file.pipe(jStream);
- });
- busboy.on('finish', function() {
- res.end('That\'s all folks!');
- });
-
- busboy.pipe(process.stdout);
-
- // Return req.pipe(busboy);
-
- // JStream.on(function(data) {
- // console.log('received:', data);
- // });
-
- // busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
- // console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
- // file.on('data', function(data) {
- // console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
- // console.log(data);
- // });
- // file.on('end', function() {
- // console.log('File [' + fieldname + '] Finished');
- // });
- // });
- // busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
- // console.log('Field [' + fieldname + ']: value: ' + val);
- // });
- // busboy.on('finish', function() {
- // console.log('Done parsing form!');
- // res.end();
- // });
- req.pipe(busboy);
-
-
- // Upload(req, res, function(err) {
- // if (err) {
- // res.redirect(req.headers.referer + '/error.html');
- // return;
- // }
-
- // if (!req) {
- // res.redirect(req.headers.referer + '/error.html');
-
- // } else {
- // // Implement your own logic if needed. Like moving the file, renaming the file, etc.
- // res.redirect(req.headers.referer);
- // }
- // }).single('test');
-
- }
-
public static async handleAssignSitesToUser(action: ServerAction, req: Request, res: Response, next: NextFunction): Promise<void> {
UtilsService.assertComponentIsActiveFromToken(req.user, TenantComponents.ORGANIZATION,
Action.UPDATE, Entity.SITES, 'SiteService', 'handleAssignSitesToUser');
@@ -898,6 +829,118 @@ export default class UserService {
next();
}
+ public static async import(action: ServerAction, req: Request, res: Response, next: NextFunction) {
+
+ const busboy = new Busboy({ headers: req.headers });
+
+ // Check auth
+ if (!Authorizations.canCreateUser(req.user)) {
+ throw new AppAuthError({
+ errorCode: HTTPAuthError.ERROR,
+ user: req.user,
+ action: Action.IMPORT, entity: Entity.USERS,
+ module: MODULE_NAME, method: 'handleImportUser'
+ });
+ }
+
+ req.pipe(busboy);
+
+ const converter = csvToJson({
+ trim:true,
+ delimiter:['\t'],
+ quote: 'off'
+ });
+
+ busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
+ file.pipe(converter);
+ file.on('end', function() {
+ console.log('File [' + fieldname + '] Finished');
+ });
+ });
+ busboy.on('finish', function() {
+ console.log('Done parsing form!');
+ res.send('success');
+ res.end();
+ });
+
+ converter.subscribe(async (json) => new Promise(async (resolve,reject) => {
+ await UserStorage.saveImportedUser(req.user.tenantID, json);
+ resolve();
+ }));
+
+
+ // Const cStream = new csvStream()
+ // const busboy = new Busboy({ headers: req.headers });
+ // const jStream = JSONStream.parse('users.*');
+
+ // busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
+ // file.pipe(cStream);
+ // });
+
+ // cStream.on('data', async function(data) {
+ // await UserStorage.saveImportedUser(req.user.tenantID, data);
+ // });
+
+ // jStream.on('data', async function(data) {
+ // await UserStorage.saveImportedUser(req.user.tenantID, data);
+ // });
+
+ // busboy.on('finish', function() {
+ // res.end('That\'s all folks!');
+ // });
+
+ // csvToJson().fromStream(req).subscribe(async (json) => new Promise((resolve, reject) => {
+ // console.log(json);
+ // }).then());
+
+ // req.pipe(busboy);
+
+ // res.json('uploading');
+ // next();
+
+
+ // Return req.pipe(busboy);
+
+ // JStream.on(function(data) {
+ // console.log('received:', data);
+ // });
+
+ // busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
+ // console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
+ // file.on('data', function(data) {
+ // console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
+ // console.log(data);
+ // });
+ // file.on('end', function() {
+ // console.log('File [' + fieldname + '] Finished');
+ // });
+ // });
+ // busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
+ // console.log('Field [' + fieldname + ']: value: ' + val);
+ // });
+ // busboy.on('finish', function() {
+ // console.log('Done parsing form!');
+ // res.end();
+ // });
+
+
+ // Upload(req, res, function(err) {
+ // if (err) {
+ // res.redirect(req.headers.referer + '/error.html');
+ // return;
+ // }
+
+ // if (!req) {
+ // res.redirect(req.headers.referer + '/error.html');
+
+ // } else {
+ // // Implement your own logic if needed. Like moving the file, renaming the file, etc.
+ // res.redirect(req.headers.referer);
+ // }
+ // }).single('test');
+
+ }
+
public static async handleCreateUser(action: ServerAction, req: Request, res: Response, next: NextFunction): Promise<void> {
// Check auth
if (!Authorizations.canCreateUser(req.user)) {

MODIFY
src/storage/mongodb/UserStorage.ts
src/storage/mongodb/UserStorage.ts
@@ -273,7 +273,7 @@ export default class UserStorage {
public static async saveImportedUser(tenantID: string, usersToSave): Promise<void> {
await global.database.getCollection<any>(tenantID, 'userImport3').insertOne(
usersToSave);
- // Console.log('saved:', usersToSave);
+ console.log('saved:', usersToSave);
}
public static async saveUserPassword(tenantID: string, userID: string,

MODIFY
src/types/Authorization.ts
src/types/Authorization.ts
@@ -121,6 +121,7 @@ export enum Action {
REGISTER = 'Register',
TRIGGER_JOB = 'TriggerJob',
DOWNLOAD = 'Download',
+ IMPORT = 'Import'
}
export interface AuthorizationContext {

using csv to json and busboy