@Test void ingest() { Map<String, Integer> imported = new HashMap<>(); imported.put("test", 1); when(report.getNrImportedEntitiesMap()).thenReturn(imported); when(fileRepositoryCollectionFactoryMock.createFileRepositoryCollection(f)) .thenReturn(fileRepositoryCollectionMock); when(importServiceFactoryMock.getImportService("test_data_only.xlsx")) .thenReturn(importServiceMock); when(importServiceMock.doImport( any(), eq(MetadataAction.UPSERT), eq(ADD_UPDATE_EXISTING), eq(null))) .thenReturn(report); amazonBucketIngester.ingest( "jobExecutionID", "targetEntityTypeName", "bucket", "key(.*)", null, "test", "test", "region1", true, progress); verify(importServiceFactoryMock).getImportService("test_data_only.xlsx"); verify(importServiceMock) .doImport(any(), eq(MetadataAction.UPSERT), eq(ADD_UPDATE_EXISTING), eq(null)); }
public FileMeta ingest( String jobExecutionID, String targetEntityTypeName, String bucket, String key, String extension, String accessKey, String secretKey, String region, boolean isExpression, Progress progress) { FileMeta fileMeta; try { progress.setProgressMax(3); progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'"); AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region); progress.progress(1, "downloading..."); File file = amazonBucketClient.downloadFile( client, fileStore, jobExecutionID, bucket, key, extension, isExpression, targetEntityTypeName); if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName())) { if (ExcelUtils.getNumberOfSheets(file) == 1) { ExcelUtils.renameSheet(targetEntityTypeName, file, 0); } else { throw new MolgenisDataException( "Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet"); } } progress.progress(2, "Importing..."); ImportService importService = importServiceFactory.getImportService(file.getName()); File renamed = new File( String.format( "%s%s%s.%s", file.getParent(), File.separatorChar, targetEntityTypeName, extension)); Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING); RepositoryCollection repositoryCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(renamed); EntityImportReport report = importService.doImport( repositoryCollection, MetadataAction.UPSERT, DataAction.ADD_UPDATE_EXISTING, null); progress.status("Download and import from Amazon Bucket done."); progress.progress( 3, "Successfully imported " + report.getNrImportedEntitiesMap().keySet().toString() + " entities."); fileMeta = createFileMeta(jobExecutionID, file); } catch (Exception e) { throw new MolgenisDataException(e); } return fileMeta; }
AmazonBucketIngester { public FileMeta ingest( String jobExecutionID, String targetEntityTypeName, String bucket, String key, String extension, String accessKey, String secretKey, String region, boolean isExpression, Progress progress) { FileMeta fileMeta; try { progress.setProgressMax(3); progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'"); AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region); progress.progress(1, "downloading..."); File file = amazonBucketClient.downloadFile( client, fileStore, jobExecutionID, bucket, key, extension, isExpression, targetEntityTypeName); if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName())) { if (ExcelUtils.getNumberOfSheets(file) == 1) { ExcelUtils.renameSheet(targetEntityTypeName, file, 0); } else { throw new MolgenisDataException( "Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet"); } } progress.progress(2, "Importing..."); ImportService importService = importServiceFactory.getImportService(file.getName()); File renamed = new File( String.format( "%s%s%s.%s", file.getParent(), File.separatorChar, targetEntityTypeName, extension)); Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING); RepositoryCollection repositoryCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(renamed); EntityImportReport report = importService.doImport( repositoryCollection, MetadataAction.UPSERT, DataAction.ADD_UPDATE_EXISTING, null); progress.status("Download and import from Amazon Bucket done."); progress.progress( 3, "Successfully imported " + report.getNrImportedEntitiesMap().keySet().toString() + " entities."); fileMeta = createFileMeta(jobExecutionID, file); } catch (Exception e) { throw new MolgenisDataException(e); } return fileMeta; } }
AmazonBucketIngester { public FileMeta ingest( String jobExecutionID, String targetEntityTypeName, String bucket, String key, String extension, String accessKey, String secretKey, String region, boolean isExpression, Progress progress) { FileMeta fileMeta; try { progress.setProgressMax(3); progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'"); AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region); progress.progress(1, "downloading..."); File file = amazonBucketClient.downloadFile( client, fileStore, jobExecutionID, bucket, key, extension, isExpression, targetEntityTypeName); if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName())) { if (ExcelUtils.getNumberOfSheets(file) == 1) { ExcelUtils.renameSheet(targetEntityTypeName, file, 0); } else { throw new MolgenisDataException( "Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet"); } } progress.progress(2, "Importing..."); ImportService importService = importServiceFactory.getImportService(file.getName()); File renamed = new File( String.format( "%s%s%s.%s", file.getParent(), File.separatorChar, targetEntityTypeName, extension)); Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING); RepositoryCollection repositoryCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(renamed); EntityImportReport report = importService.doImport( repositoryCollection, MetadataAction.UPSERT, DataAction.ADD_UPDATE_EXISTING, null); progress.status("Download and import from Amazon Bucket done."); progress.progress( 3, "Successfully imported " + report.getNrImportedEntitiesMap().keySet().toString() + " entities."); fileMeta = createFileMeta(jobExecutionID, file); } catch (Exception e) { throw new MolgenisDataException(e); } return fileMeta; } AmazonBucketIngester(
      ImportServiceFactory importServiceFactory,
      FileRepositoryCollectionFactory fileRepositoryCollectionFactory,
      FileMetaFactory fileMetaFactory,
      FileStore fileStore,
      AmazonBucketClient amazonBucketClient); }
AmazonBucketIngester { public FileMeta ingest( String jobExecutionID, String targetEntityTypeName, String bucket, String key, String extension, String accessKey, String secretKey, String region, boolean isExpression, Progress progress) { FileMeta fileMeta; try { progress.setProgressMax(3); progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'"); AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region); progress.progress(1, "downloading..."); File file = amazonBucketClient.downloadFile( client, fileStore, jobExecutionID, bucket, key, extension, isExpression, targetEntityTypeName); if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName())) { if (ExcelUtils.getNumberOfSheets(file) == 1) { ExcelUtils.renameSheet(targetEntityTypeName, file, 0); } else { throw new MolgenisDataException( "Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet"); } } progress.progress(2, "Importing..."); ImportService importService = importServiceFactory.getImportService(file.getName()); File renamed = new File( String.format( "%s%s%s.%s", file.getParent(), File.separatorChar, targetEntityTypeName, extension)); Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING); RepositoryCollection repositoryCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(renamed); EntityImportReport report = importService.doImport( repositoryCollection, MetadataAction.UPSERT, DataAction.ADD_UPDATE_EXISTING, null); progress.status("Download and import from Amazon Bucket done."); progress.progress( 3, "Successfully imported " + report.getNrImportedEntitiesMap().keySet().toString() + " entities."); fileMeta = createFileMeta(jobExecutionID, file); } catch (Exception e) { throw new MolgenisDataException(e); } return fileMeta; } AmazonBucketIngester(
      ImportServiceFactory importServiceFactory,
      FileRepositoryCollectionFactory fileRepositoryCollectionFactory,
      FileMetaFactory fileMetaFactory,
      FileStore fileStore,
      AmazonBucketClient amazonBucketClient); FileMeta ingest(
      String jobExecutionID,
      String targetEntityTypeName,
      String bucket,
      String key,
      String extension,
      String accessKey,
      String secretKey,
      String region,
      boolean isExpression,
      Progress progress); }
AmazonBucketIngester { public FileMeta ingest( String jobExecutionID, String targetEntityTypeName, String bucket, String key, String extension, String accessKey, String secretKey, String region, boolean isExpression, Progress progress) { FileMeta fileMeta; try { progress.setProgressMax(3); progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'"); AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region); progress.progress(1, "downloading..."); File file = amazonBucketClient.downloadFile( client, fileStore, jobExecutionID, bucket, key, extension, isExpression, targetEntityTypeName); if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName())) { if (ExcelUtils.getNumberOfSheets(file) == 1) { ExcelUtils.renameSheet(targetEntityTypeName, file, 0); } else { throw new MolgenisDataException( "Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet"); } } progress.progress(2, "Importing..."); ImportService importService = importServiceFactory.getImportService(file.getName()); File renamed = new File( String.format( "%s%s%s.%s", file.getParent(), File.separatorChar, targetEntityTypeName, extension)); Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING); RepositoryCollection repositoryCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(renamed); EntityImportReport report = importService.doImport( repositoryCollection, MetadataAction.UPSERT, DataAction.ADD_UPDATE_EXISTING, null); progress.status("Download and import from Amazon Bucket done."); progress.progress( 3, "Successfully imported " + report.getNrImportedEntitiesMap().keySet().toString() + " entities."); fileMeta = createFileMeta(jobExecutionID, file); } catch (Exception e) { throw new MolgenisDataException(e); } return fileMeta; } AmazonBucketIngester(
      ImportServiceFactory importServiceFactory,
      FileRepositoryCollectionFactory fileRepositoryCollectionFactory,
      FileMetaFactory fileMetaFactory,
      FileStore fileStore,
      AmazonBucketClient amazonBucketClient); FileMeta ingest(
      String jobExecutionID,
      String targetEntityTypeName,
      String bucket,
      String key,
      String extension,
      String accessKey,
      String secretKey,
      String region,
      boolean isExpression,
      Progress progress);  }