@Test public void testExecute() throws Throwable { myLog.debug( "delimiter not yet set in environment" ); testException(); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_DELIMITER, ";" ); myLog.debug( "filename not yet set in environment" ); testException(); myLog.debug( "wrong filename in environment" ); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, "blabla" ); testException(); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, toFile( "CSV_testdata_noHeader.csv" ).getCanonicalPath() ); myLog.debug( "fieldnames not yet set in environment" ); testException(); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES, "Screenname,Name,Middlename,Prename,email,birthday,entrydate" ); myLog.debug( "fieldnames now set in environment" ); testSuccess( 7, 2 ); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "2" ); myLog.debug( "skip >= rowsize. Ignored because headerRow is false" ); testSuccess( 7, 2 ); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "true" ); myLog.debug( "skip >= rowsize. HeaderRow=true -> Exception" ); testException(); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "0" ); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, toFile( "CSV_testdata_withHeader0.csv" ).getCanonicalPath() ); TestUtils.removeFromEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES ); myLog.debug( "CSV_testdata_withHeader0.csv success" ); testSuccess( 7, 2 ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "2" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, toFile( "CSV_testdata_withHeader2.csv" ).getCanonicalPath() ); myLog.debug( "CSV_testdata_withHeader2.csv success" ); testSuccess( 7, 2 ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "5" ); myLog.debug( "skip >= rowsize. HeaderRow=true -> Exception" ); testException(); TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES, "Screenname,Name,Middlename,Prename,email,birthday" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "0" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "false" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, toFile( "CSV_testdata_noHeader.csv" ).getCanonicalPath() ); myLog.debug( "CSV-File with too many entries -> Exception" ); testException(); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES, "Screenname,Name,Middlename,Prename,email,birthday,entrydate" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "0" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "false" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, toFile( "CSV_testdata_noHeader_empty_fields.csv" ).getCanonicalPath() ); myLog.debug( "loading avatar picture (jpg hex dump)" ); testSuccess( 7, 2 ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "true" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "true" ); TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, toFile( "CSV_testdata_withHeader0_Avatar.csv" ).getCanonicalPath() ); myLog.debug( "empty field content -> Empty String or null (if date)" ); testSuccess( 8, 2 ); }
@Override public CollectorResult execute() throws SdiException { init(); InputStream is = openInputStream(); List<List<String>> parsed = myParser.parse( is, myDelimiter, myEncoding, myLineFilters ); myFieldnames = evaluateFieldNames( parsed, myHeaderRow ); myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV-Fieldnames", myFieldnames ) ); int toSkip = myHeaderRow ? (mySkip + 1) : 0; myLog.debug( "Skipping first " + toSkip + " rows" ); if ( parsed.size() < toSkip ) { throw new SdiException( "No data found in CSV file "+ myInputFileName, SdiException.EXIT_CODE_PARSE_ERROR ); } Collection<Dataset> myRows = new ArrayList<>(); Collection<Dataset> myRowsFiltered = new ArrayList<>(); List<FieldConverter<?>> converters = myConverterFactory.getFieldConverters( myFieldnames ); ROW_LOOP: for ( int i = toSkip; i < parsed.size(); i++ ) { List<String> row = parsed.get( i ); myLog.trace( "row " + (i-toSkip) + ": " + row ); try { Collection<Object> converted = convertFields( row, converters ); Dataset dataset = Dataset.create( myFieldnames, converted ); for ( CollectFilter<?> filter : myCollectFilters ) { if ( filter.isFiltered( dataset ) ) { myLog.debug( "Given row is filtered: " + row ); myRowsFiltered.add( dataset ); continue ROW_LOOP; } } myRows.add( dataset ); } catch ( Exception t ) { myLog.error( "row " + (i-toSkip) + " cannot be converted" ); throw t; } } myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV parsed lines", myRows ) ); myLog.info( new ReportMsg( ReportMsg.ReportType.POSTPARSE_FILTER, "CSV Filtered datasets", myRowsFiltered ) ); return new CollectorResult() { @Override public Collection<String> getFieldnames() { return myFieldnames; } @Override public Collection<Dataset> getRows() { return myRows; } }; }
CsvCollector implements InputCollector { @Override public CollectorResult execute() throws SdiException { init(); InputStream is = openInputStream(); List<List<String>> parsed = myParser.parse( is, myDelimiter, myEncoding, myLineFilters ); myFieldnames = evaluateFieldNames( parsed, myHeaderRow ); myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV-Fieldnames", myFieldnames ) ); int toSkip = myHeaderRow ? (mySkip + 1) : 0; myLog.debug( "Skipping first " + toSkip + " rows" ); if ( parsed.size() < toSkip ) { throw new SdiException( "No data found in CSV file "+ myInputFileName, SdiException.EXIT_CODE_PARSE_ERROR ); } Collection<Dataset> myRows = new ArrayList<>(); Collection<Dataset> myRowsFiltered = new ArrayList<>(); List<FieldConverter<?>> converters = myConverterFactory.getFieldConverters( myFieldnames ); ROW_LOOP: for ( int i = toSkip; i < parsed.size(); i++ ) { List<String> row = parsed.get( i ); myLog.trace( "row " + (i-toSkip) + ": " + row ); try { Collection<Object> converted = convertFields( row, converters ); Dataset dataset = Dataset.create( myFieldnames, converted ); for ( CollectFilter<?> filter : myCollectFilters ) { if ( filter.isFiltered( dataset ) ) { myLog.debug( "Given row is filtered: " + row ); myRowsFiltered.add( dataset ); continue ROW_LOOP; } } myRows.add( dataset ); } catch ( Exception t ) { myLog.error( "row " + (i-toSkip) + " cannot be converted" ); throw t; } } myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV parsed lines", myRows ) ); myLog.info( new ReportMsg( ReportMsg.ReportType.POSTPARSE_FILTER, "CSV Filtered datasets", myRowsFiltered ) ); return new CollectorResult() { @Override public Collection<String> getFieldnames() { return myFieldnames; } @Override public Collection<Dataset> getRows() { return myRows; } }; } }
CsvCollector implements InputCollector { @Override public CollectorResult execute() throws SdiException { init(); InputStream is = openInputStream(); List<List<String>> parsed = myParser.parse( is, myDelimiter, myEncoding, myLineFilters ); myFieldnames = evaluateFieldNames( parsed, myHeaderRow ); myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV-Fieldnames", myFieldnames ) ); int toSkip = myHeaderRow ? (mySkip + 1) : 0; myLog.debug( "Skipping first " + toSkip + " rows" ); if ( parsed.size() < toSkip ) { throw new SdiException( "No data found in CSV file "+ myInputFileName, SdiException.EXIT_CODE_PARSE_ERROR ); } Collection<Dataset> myRows = new ArrayList<>(); Collection<Dataset> myRowsFiltered = new ArrayList<>(); List<FieldConverter<?>> converters = myConverterFactory.getFieldConverters( myFieldnames ); ROW_LOOP: for ( int i = toSkip; i < parsed.size(); i++ ) { List<String> row = parsed.get( i ); myLog.trace( "row " + (i-toSkip) + ": " + row ); try { Collection<Object> converted = convertFields( row, converters ); Dataset dataset = Dataset.create( myFieldnames, converted ); for ( CollectFilter<?> filter : myCollectFilters ) { if ( filter.isFiltered( dataset ) ) { myLog.debug( "Given row is filtered: " + row ); myRowsFiltered.add( dataset ); continue ROW_LOOP; } } myRows.add( dataset ); } catch ( Exception t ) { myLog.error( "row " + (i-toSkip) + " cannot be converted" ); throw t; } } myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV parsed lines", myRows ) ); myLog.info( new ReportMsg( ReportMsg.ReportType.POSTPARSE_FILTER, "CSV Filtered datasets", myRowsFiltered ) ); return new CollectorResult() { @Override public Collection<String> getFieldnames() { return myFieldnames; } @Override public Collection<Dataset> getRows() { return myRows; } }; } CsvCollector(); }
CsvCollector implements InputCollector { @Override public CollectorResult execute() throws SdiException { init(); InputStream is = openInputStream(); List<List<String>> parsed = myParser.parse( is, myDelimiter, myEncoding, myLineFilters ); myFieldnames = evaluateFieldNames( parsed, myHeaderRow ); myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV-Fieldnames", myFieldnames ) ); int toSkip = myHeaderRow ? (mySkip + 1) : 0; myLog.debug( "Skipping first " + toSkip + " rows" ); if ( parsed.size() < toSkip ) { throw new SdiException( "No data found in CSV file "+ myInputFileName, SdiException.EXIT_CODE_PARSE_ERROR ); } Collection<Dataset> myRows = new ArrayList<>(); Collection<Dataset> myRowsFiltered = new ArrayList<>(); List<FieldConverter<?>> converters = myConverterFactory.getFieldConverters( myFieldnames ); ROW_LOOP: for ( int i = toSkip; i < parsed.size(); i++ ) { List<String> row = parsed.get( i ); myLog.trace( "row " + (i-toSkip) + ": " + row ); try { Collection<Object> converted = convertFields( row, converters ); Dataset dataset = Dataset.create( myFieldnames, converted ); for ( CollectFilter<?> filter : myCollectFilters ) { if ( filter.isFiltered( dataset ) ) { myLog.debug( "Given row is filtered: " + row ); myRowsFiltered.add( dataset ); continue ROW_LOOP; } } myRows.add( dataset ); } catch ( Exception t ) { myLog.error( "row " + (i-toSkip) + " cannot be converted" ); throw t; } } myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV parsed lines", myRows ) ); myLog.info( new ReportMsg( ReportMsg.ReportType.POSTPARSE_FILTER, "CSV Filtered datasets", myRowsFiltered ) ); return new CollectorResult() { @Override public Collection<String> getFieldnames() { return myFieldnames; } @Override public Collection<Dataset> getRows() { return myRows; } }; } CsvCollector(); @Override CollectorResult execute(); }
CsvCollector implements InputCollector { @Override public CollectorResult execute() throws SdiException { init(); InputStream is = openInputStream(); List<List<String>> parsed = myParser.parse( is, myDelimiter, myEncoding, myLineFilters ); myFieldnames = evaluateFieldNames( parsed, myHeaderRow ); myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV-Fieldnames", myFieldnames ) ); int toSkip = myHeaderRow ? (mySkip + 1) : 0; myLog.debug( "Skipping first " + toSkip + " rows" ); if ( parsed.size() < toSkip ) { throw new SdiException( "No data found in CSV file "+ myInputFileName, SdiException.EXIT_CODE_PARSE_ERROR ); } Collection<Dataset> myRows = new ArrayList<>(); Collection<Dataset> myRowsFiltered = new ArrayList<>(); List<FieldConverter<?>> converters = myConverterFactory.getFieldConverters( myFieldnames ); ROW_LOOP: for ( int i = toSkip; i < parsed.size(); i++ ) { List<String> row = parsed.get( i ); myLog.trace( "row " + (i-toSkip) + ": " + row ); try { Collection<Object> converted = convertFields( row, converters ); Dataset dataset = Dataset.create( myFieldnames, converted ); for ( CollectFilter<?> filter : myCollectFilters ) { if ( filter.isFiltered( dataset ) ) { myLog.debug( "Given row is filtered: " + row ); myRowsFiltered.add( dataset ); continue ROW_LOOP; } } myRows.add( dataset ); } catch ( Exception t ) { myLog.error( "row " + (i-toSkip) + " cannot be converted" ); throw t; } } myLog.info( new ReportMsg( ReportMsg.ReportType.COLLECTOR, "CSV parsed lines", myRows ) ); myLog.info( new ReportMsg( ReportMsg.ReportType.POSTPARSE_FILTER, "CSV Filtered datasets", myRowsFiltered ) ); return new CollectorResult() { @Override public Collection<String> getFieldnames() { return myFieldnames; } @Override public Collection<Dataset> getRows() { return myRows; } }; } CsvCollector(); @Override CollectorResult execute();  }