@Test public void testOpenNextFile_010_ignoreEmpty() { stepMetaInterface.setIgnoreEmptyFile( true ); stepInputFiles.addFile( getFile( "input0.txt" ) ); stepInputFiles.addFile( getFile( "input1.txt" ) ); stepInputFiles.addFile( getFile( "input0.txt" ) ); assertTrue( stepLoadFileInput.openNextFile() ); assertFalse( stepLoadFileInput.openNextFile() ); }
boolean openNextFile() { try { if ( meta.getIsInFields() ) { data.readrow = getRow(); if ( data.readrow == null ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } if ( first ) { first = false; data.inputRowMeta = getInputRowMeta(); data.outputRowMeta = data.inputRowMeta.clone(); meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore ); data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); if ( meta.getIsInFields() ) { if ( Utils.isEmpty( meta.getDynamicFilenameField() ) ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); } if ( data.indexOfFilenameField < 0 ) { data.indexOfFilenameField = data.inputRowMeta.indexOfValue( meta.getDynamicFilenameField() ); if ( data.indexOfFilenameField < 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.ErrorFindingField" ) + "[" + meta.getDynamicFilenameField() + "]" ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Exception.CouldnotFindField", meta.getDynamicFilenameField() ) ); } } data.totalpreviousfields = data.inputRowMeta.size(); } } String Fieldvalue = data.inputRowMeta.getString( data.readrow, data.indexOfFilenameField ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue ) ); } FileObject file = null; try { data.file = KettleVFS.getFileObject( Fieldvalue ); } catch ( Exception e ) { throw new KettleException( e ); } finally { try { if ( file != null ) { file.close(); } } catch ( Exception e ) { } } } else { if ( data.filenr >= data.files.nrOfFiles() ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } data.last_file = ( data.filenr == data.files.nrOfFiles() - 1 ); data.file = data.files.getFile( data.filenr ); } if ( meta.isIgnoreMissingPath() && !data.file.exists() ) { logBasic( BaseMessages.getString( PKG, "LoadFileInput.Error.FileNotExists", "" + data.file.getName() ) ); return openNextFile(); } data.fileSize = data.file.getContent().getSize(); data.filenr++; if ( meta.isIgnoreEmptyFile() && data.fileSize == 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName() ) ); return openNextFile(); } else { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.OpeningFile", data.file.toString() ) ); } data.filename = KettleVFS.getFilename( data.file ); if ( meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0 ) { data.shortFilename = data.file.getName().getBaseName(); } if ( meta.getPathField() != null && meta.getPathField().length() > 0 ) { data.path = KettleVFS.getFilename( data.file.getParent() ); } if ( meta.isHiddenField() != null && meta.isHiddenField().length() > 0 ) { data.hidden = data.file.isHidden(); } if ( meta.getExtensionField() != null && meta.getExtensionField().length() > 0 ) { data.extension = data.file.getName().getExtension(); } if ( meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0 ) { data.lastModificationDateTime = new Date( data.file.getContent().getLastModifiedTime() ); } if ( meta.getUriField() != null && meta.getUriField().length() > 0 ) { data.uriName = data.file.getName().getURI(); } if ( meta.getRootUriField() != null && meta.getRootUriField().length() > 0 ) { data.rootUriName = data.file.getName().getRootURI(); } getFileContent(); addFileToResultFilesname( data.file ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FileOpened", data.file.toString() ) ); } } } catch ( Exception e ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.UnableToOpenFile", "" + data.filenr, data.file .toString(), e.toString() ) ); stopAll(); setErrors( 1 ); return false; } return true; }
LoadFileInput extends BaseStep implements StepInterface { boolean openNextFile() { try { if ( meta.getIsInFields() ) { data.readrow = getRow(); if ( data.readrow == null ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } if ( first ) { first = false; data.inputRowMeta = getInputRowMeta(); data.outputRowMeta = data.inputRowMeta.clone(); meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore ); data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); if ( meta.getIsInFields() ) { if ( Utils.isEmpty( meta.getDynamicFilenameField() ) ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); } if ( data.indexOfFilenameField < 0 ) { data.indexOfFilenameField = data.inputRowMeta.indexOfValue( meta.getDynamicFilenameField() ); if ( data.indexOfFilenameField < 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.ErrorFindingField" ) + "[" + meta.getDynamicFilenameField() + "]" ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Exception.CouldnotFindField", meta.getDynamicFilenameField() ) ); } } data.totalpreviousfields = data.inputRowMeta.size(); } } String Fieldvalue = data.inputRowMeta.getString( data.readrow, data.indexOfFilenameField ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue ) ); } FileObject file = null; try { data.file = KettleVFS.getFileObject( Fieldvalue ); } catch ( Exception e ) { throw new KettleException( e ); } finally { try { if ( file != null ) { file.close(); } } catch ( Exception e ) { } } } else { if ( data.filenr >= data.files.nrOfFiles() ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } data.last_file = ( data.filenr == data.files.nrOfFiles() - 1 ); data.file = data.files.getFile( data.filenr ); } if ( meta.isIgnoreMissingPath() && !data.file.exists() ) { logBasic( BaseMessages.getString( PKG, "LoadFileInput.Error.FileNotExists", "" + data.file.getName() ) ); return openNextFile(); } data.fileSize = data.file.getContent().getSize(); data.filenr++; if ( meta.isIgnoreEmptyFile() && data.fileSize == 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName() ) ); return openNextFile(); } else { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.OpeningFile", data.file.toString() ) ); } data.filename = KettleVFS.getFilename( data.file ); if ( meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0 ) { data.shortFilename = data.file.getName().getBaseName(); } if ( meta.getPathField() != null && meta.getPathField().length() > 0 ) { data.path = KettleVFS.getFilename( data.file.getParent() ); } if ( meta.isHiddenField() != null && meta.isHiddenField().length() > 0 ) { data.hidden = data.file.isHidden(); } if ( meta.getExtensionField() != null && meta.getExtensionField().length() > 0 ) { data.extension = data.file.getName().getExtension(); } if ( meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0 ) { data.lastModificationDateTime = new Date( data.file.getContent().getLastModifiedTime() ); } if ( meta.getUriField() != null && meta.getUriField().length() > 0 ) { data.uriName = data.file.getName().getURI(); } if ( meta.getRootUriField() != null && meta.getRootUriField().length() > 0 ) { data.rootUriName = data.file.getName().getRootURI(); } getFileContent(); addFileToResultFilesname( data.file ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FileOpened", data.file.toString() ) ); } } } catch ( Exception e ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.UnableToOpenFile", "" + data.filenr, data.file .toString(), e.toString() ) ); stopAll(); setErrors( 1 ); return false; } return true; } }
LoadFileInput extends BaseStep implements StepInterface { boolean openNextFile() { try { if ( meta.getIsInFields() ) { data.readrow = getRow(); if ( data.readrow == null ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } if ( first ) { first = false; data.inputRowMeta = getInputRowMeta(); data.outputRowMeta = data.inputRowMeta.clone(); meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore ); data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); if ( meta.getIsInFields() ) { if ( Utils.isEmpty( meta.getDynamicFilenameField() ) ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); } if ( data.indexOfFilenameField < 0 ) { data.indexOfFilenameField = data.inputRowMeta.indexOfValue( meta.getDynamicFilenameField() ); if ( data.indexOfFilenameField < 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.ErrorFindingField" ) + "[" + meta.getDynamicFilenameField() + "]" ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Exception.CouldnotFindField", meta.getDynamicFilenameField() ) ); } } data.totalpreviousfields = data.inputRowMeta.size(); } } String Fieldvalue = data.inputRowMeta.getString( data.readrow, data.indexOfFilenameField ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue ) ); } FileObject file = null; try { data.file = KettleVFS.getFileObject( Fieldvalue ); } catch ( Exception e ) { throw new KettleException( e ); } finally { try { if ( file != null ) { file.close(); } } catch ( Exception e ) { } } } else { if ( data.filenr >= data.files.nrOfFiles() ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } data.last_file = ( data.filenr == data.files.nrOfFiles() - 1 ); data.file = data.files.getFile( data.filenr ); } if ( meta.isIgnoreMissingPath() && !data.file.exists() ) { logBasic( BaseMessages.getString( PKG, "LoadFileInput.Error.FileNotExists", "" + data.file.getName() ) ); return openNextFile(); } data.fileSize = data.file.getContent().getSize(); data.filenr++; if ( meta.isIgnoreEmptyFile() && data.fileSize == 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName() ) ); return openNextFile(); } else { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.OpeningFile", data.file.toString() ) ); } data.filename = KettleVFS.getFilename( data.file ); if ( meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0 ) { data.shortFilename = data.file.getName().getBaseName(); } if ( meta.getPathField() != null && meta.getPathField().length() > 0 ) { data.path = KettleVFS.getFilename( data.file.getParent() ); } if ( meta.isHiddenField() != null && meta.isHiddenField().length() > 0 ) { data.hidden = data.file.isHidden(); } if ( meta.getExtensionField() != null && meta.getExtensionField().length() > 0 ) { data.extension = data.file.getName().getExtension(); } if ( meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0 ) { data.lastModificationDateTime = new Date( data.file.getContent().getLastModifiedTime() ); } if ( meta.getUriField() != null && meta.getUriField().length() > 0 ) { data.uriName = data.file.getName().getURI(); } if ( meta.getRootUriField() != null && meta.getRootUriField().length() > 0 ) { data.rootUriName = data.file.getName().getRootURI(); } getFileContent(); addFileToResultFilesname( data.file ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FileOpened", data.file.toString() ) ); } } } catch ( Exception e ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.UnableToOpenFile", "" + data.filenr, data.file .toString(), e.toString() ) ); stopAll(); setErrors( 1 ); return false; } return true; } LoadFileInput( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
    Trans trans ); }
LoadFileInput extends BaseStep implements StepInterface { boolean openNextFile() { try { if ( meta.getIsInFields() ) { data.readrow = getRow(); if ( data.readrow == null ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } if ( first ) { first = false; data.inputRowMeta = getInputRowMeta(); data.outputRowMeta = data.inputRowMeta.clone(); meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore ); data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); if ( meta.getIsInFields() ) { if ( Utils.isEmpty( meta.getDynamicFilenameField() ) ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); } if ( data.indexOfFilenameField < 0 ) { data.indexOfFilenameField = data.inputRowMeta.indexOfValue( meta.getDynamicFilenameField() ); if ( data.indexOfFilenameField < 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.ErrorFindingField" ) + "[" + meta.getDynamicFilenameField() + "]" ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Exception.CouldnotFindField", meta.getDynamicFilenameField() ) ); } } data.totalpreviousfields = data.inputRowMeta.size(); } } String Fieldvalue = data.inputRowMeta.getString( data.readrow, data.indexOfFilenameField ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue ) ); } FileObject file = null; try { data.file = KettleVFS.getFileObject( Fieldvalue ); } catch ( Exception e ) { throw new KettleException( e ); } finally { try { if ( file != null ) { file.close(); } } catch ( Exception e ) { } } } else { if ( data.filenr >= data.files.nrOfFiles() ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } data.last_file = ( data.filenr == data.files.nrOfFiles() - 1 ); data.file = data.files.getFile( data.filenr ); } if ( meta.isIgnoreMissingPath() && !data.file.exists() ) { logBasic( BaseMessages.getString( PKG, "LoadFileInput.Error.FileNotExists", "" + data.file.getName() ) ); return openNextFile(); } data.fileSize = data.file.getContent().getSize(); data.filenr++; if ( meta.isIgnoreEmptyFile() && data.fileSize == 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName() ) ); return openNextFile(); } else { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.OpeningFile", data.file.toString() ) ); } data.filename = KettleVFS.getFilename( data.file ); if ( meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0 ) { data.shortFilename = data.file.getName().getBaseName(); } if ( meta.getPathField() != null && meta.getPathField().length() > 0 ) { data.path = KettleVFS.getFilename( data.file.getParent() ); } if ( meta.isHiddenField() != null && meta.isHiddenField().length() > 0 ) { data.hidden = data.file.isHidden(); } if ( meta.getExtensionField() != null && meta.getExtensionField().length() > 0 ) { data.extension = data.file.getName().getExtension(); } if ( meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0 ) { data.lastModificationDateTime = new Date( data.file.getContent().getLastModifiedTime() ); } if ( meta.getUriField() != null && meta.getUriField().length() > 0 ) { data.uriName = data.file.getName().getURI(); } if ( meta.getRootUriField() != null && meta.getRootUriField().length() > 0 ) { data.rootUriName = data.file.getName().getRootURI(); } getFileContent(); addFileToResultFilesname( data.file ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FileOpened", data.file.toString() ) ); } } } catch ( Exception e ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.UnableToOpenFile", "" + data.filenr, data.file .toString(), e.toString() ) ); stopAll(); setErrors( 1 ); return false; } return true; } LoadFileInput( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
    Trans trans ); boolean processRow( StepMetaInterface smi, StepDataInterface sdi ); static String getTextFileContent( String vfsFilename, String encoding ); boolean init( StepMetaInterface smi, StepDataInterface sdi ); void dispose( StepMetaInterface smi, StepDataInterface sdi ); }
LoadFileInput extends BaseStep implements StepInterface { boolean openNextFile() { try { if ( meta.getIsInFields() ) { data.readrow = getRow(); if ( data.readrow == null ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } if ( first ) { first = false; data.inputRowMeta = getInputRowMeta(); data.outputRowMeta = data.inputRowMeta.clone(); meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore ); data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); if ( meta.getIsInFields() ) { if ( Utils.isEmpty( meta.getDynamicFilenameField() ) ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Log.NoField" ) ); } if ( data.indexOfFilenameField < 0 ) { data.indexOfFilenameField = data.inputRowMeta.indexOfValue( meta.getDynamicFilenameField() ); if ( data.indexOfFilenameField < 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.ErrorFindingField" ) + "[" + meta.getDynamicFilenameField() + "]" ); throw new KettleException( BaseMessages.getString( PKG, "LoadFileInput.Exception.CouldnotFindField", meta.getDynamicFilenameField() ) ); } } data.totalpreviousfields = data.inputRowMeta.size(); } } String Fieldvalue = data.inputRowMeta.getString( data.readrow, data.indexOfFilenameField ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue ) ); } FileObject file = null; try { data.file = KettleVFS.getFileObject( Fieldvalue ); } catch ( Exception e ) { throw new KettleException( e ); } finally { try { if ( file != null ) { file.close(); } } catch ( Exception e ) { } } } else { if ( data.filenr >= data.files.nrOfFiles() ) { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FinishedProcessing" ) ); } return false; } data.last_file = ( data.filenr == data.files.nrOfFiles() - 1 ); data.file = data.files.getFile( data.filenr ); } if ( meta.isIgnoreMissingPath() && !data.file.exists() ) { logBasic( BaseMessages.getString( PKG, "LoadFileInput.Error.FileNotExists", "" + data.file.getName() ) ); return openNextFile(); } data.fileSize = data.file.getContent().getSize(); data.filenr++; if ( meta.isIgnoreEmptyFile() && data.fileSize == 0 ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName() ) ); return openNextFile(); } else { if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.OpeningFile", data.file.toString() ) ); } data.filename = KettleVFS.getFilename( data.file ); if ( meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0 ) { data.shortFilename = data.file.getName().getBaseName(); } if ( meta.getPathField() != null && meta.getPathField().length() > 0 ) { data.path = KettleVFS.getFilename( data.file.getParent() ); } if ( meta.isHiddenField() != null && meta.isHiddenField().length() > 0 ) { data.hidden = data.file.isHidden(); } if ( meta.getExtensionField() != null && meta.getExtensionField().length() > 0 ) { data.extension = data.file.getName().getExtension(); } if ( meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0 ) { data.lastModificationDateTime = new Date( data.file.getContent().getLastModifiedTime() ); } if ( meta.getUriField() != null && meta.getUriField().length() > 0 ) { data.uriName = data.file.getName().getURI(); } if ( meta.getRootUriField() != null && meta.getRootUriField().length() > 0 ) { data.rootUriName = data.file.getName().getRootURI(); } getFileContent(); addFileToResultFilesname( data.file ); if ( isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "LoadFileInput.Log.FileOpened", data.file.toString() ) ); } } } catch ( Exception e ) { logError( BaseMessages.getString( PKG, "LoadFileInput.Log.UnableToOpenFile", "" + data.filenr, data.file .toString(), e.toString() ) ); stopAll(); setErrors( 1 ); return false; } return true; } LoadFileInput( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
    Trans trans ); boolean processRow( StepMetaInterface smi, StepDataInterface sdi ); static String getTextFileContent( String vfsFilename, String encoding ); boolean init( StepMetaInterface smi, StepDataInterface sdi ); void dispose( StepMetaInterface smi, StepDataInterface sdi );  }