@Test public void testRun() throws IOException { LoadBigDataFilesTask task = new LoadBigDataFilesTask(); task.setCloudService(service); task.setBucket("swetodblp-fullrun-1"); task.setTable("ontologies.swetodblp1"); task.setEncode("true"); task.run(); }
@Override public void run() throws IOException { log.info("Processing import into big data table: " + table + ", with encode: " + encode); List<StorageObject> objects = this.getCloudService().listCloudStorageObjects(bucket); List<String> files = new ArrayList<>(); for(StorageObject object: objects) { String filename = object.getName(); if(filename.endsWith(Constants.PROCESSED_FILES)) { files.add(object.getUri()); } else { log.warn("Skipping file: " + filename); } } BigDataTable bigDataTable; if(Boolean.valueOf(encode)) { bigDataTable = TableUtils.getBigQueryEncodedTripleTable(table); } else { bigDataTable = TableUtils.getBigQueryTripleTable(table); } String jobId = this.getCloudService().loadCloudStorageFilesIntoBigData(files, bigDataTable, true); log.info("Successfully imported data into big table, completed jodId: " + jobId); }
LoadBigDataFilesTask extends CommonTask { @Override public void run() throws IOException { log.info("Processing import into big data table: " + table + ", with encode: " + encode); List<StorageObject> objects = this.getCloudService().listCloudStorageObjects(bucket); List<String> files = new ArrayList<>(); for(StorageObject object: objects) { String filename = object.getName(); if(filename.endsWith(Constants.PROCESSED_FILES)) { files.add(object.getUri()); } else { log.warn("Skipping file: " + filename); } } BigDataTable bigDataTable; if(Boolean.valueOf(encode)) { bigDataTable = TableUtils.getBigQueryEncodedTripleTable(table); } else { bigDataTable = TableUtils.getBigQueryTripleTable(table); } String jobId = this.getCloudService().loadCloudStorageFilesIntoBigData(files, bigDataTable, true); log.info("Successfully imported data into big table, completed jodId: " + jobId); } }
LoadBigDataFilesTask extends CommonTask { @Override public void run() throws IOException { log.info("Processing import into big data table: " + table + ", with encode: " + encode); List<StorageObject> objects = this.getCloudService().listCloudStorageObjects(bucket); List<String> files = new ArrayList<>(); for(StorageObject object: objects) { String filename = object.getName(); if(filename.endsWith(Constants.PROCESSED_FILES)) { files.add(object.getUri()); } else { log.warn("Skipping file: " + filename); } } BigDataTable bigDataTable; if(Boolean.valueOf(encode)) { bigDataTable = TableUtils.getBigQueryEncodedTripleTable(table); } else { bigDataTable = TableUtils.getBigQueryTripleTable(table); } String jobId = this.getCloudService().loadCloudStorageFilesIntoBigData(files, bigDataTable, true); log.info("Successfully imported data into big table, completed jodId: " + jobId); }  }
LoadBigDataFilesTask extends CommonTask { @Override public void run() throws IOException { log.info("Processing import into big data table: " + table + ", with encode: " + encode); List<StorageObject> objects = this.getCloudService().listCloudStorageObjects(bucket); List<String> files = new ArrayList<>(); for(StorageObject object: objects) { String filename = object.getName(); if(filename.endsWith(Constants.PROCESSED_FILES)) { files.add(object.getUri()); } else { log.warn("Skipping file: " + filename); } } BigDataTable bigDataTable; if(Boolean.valueOf(encode)) { bigDataTable = TableUtils.getBigQueryEncodedTripleTable(table); } else { bigDataTable = TableUtils.getBigQueryTripleTable(table); } String jobId = this.getCloudService().loadCloudStorageFilesIntoBigData(files, bigDataTable, true); log.info("Successfully imported data into big table, completed jodId: " + jobId); }  @Override void run(); String getBucket(); void setBucket(String bucket); String getTable(); void setTable(String table); String getEncode(); void setEncode(String encode); }
LoadBigDataFilesTask extends CommonTask { @Override public void run() throws IOException { log.info("Processing import into big data table: " + table + ", with encode: " + encode); List<StorageObject> objects = this.getCloudService().listCloudStorageObjects(bucket); List<String> files = new ArrayList<>(); for(StorageObject object: objects) { String filename = object.getName(); if(filename.endsWith(Constants.PROCESSED_FILES)) { files.add(object.getUri()); } else { log.warn("Skipping file: " + filename); } } BigDataTable bigDataTable; if(Boolean.valueOf(encode)) { bigDataTable = TableUtils.getBigQueryEncodedTripleTable(table); } else { bigDataTable = TableUtils.getBigQueryTripleTable(table); } String jobId = this.getCloudService().loadCloudStorageFilesIntoBigData(files, bigDataTable, true); log.info("Successfully imported data into big table, completed jodId: " + jobId); }  @Override void run(); String getBucket(); void setBucket(String bucket); String getTable(); void setTable(String table); String getEncode(); void setEncode(String encode);  }