@Test public void testGetCmds() throws IOException { JobEntrySparkSubmit ss = new JobEntrySparkSubmit(); ss.setScriptPath( "scriptPath" ); ss.setMaster( "master_url" ); ss.setJobType( JOB_TYPE_JAVA_SCALA ); ss.setJar( "jar_path" ); ss.setArgs( "arg1 arg2" ); ss.setClassName( "class_name" ); ss.setDriverMemory( "driverMemory" ); ss.setExecutorMemory( "executorMemory" ); List<String> configParams = new ArrayList<String>(); configParams.add( "name1=value1" ); configParams.add( "name2=value 2" ); ss.setConfigParams( configParams ); Map<String, String> libs = new LinkedHashMap<>(); libs.put("file: libs.put("/path/to/lib2", "<Static>"); ss.setLibs( libs ); String[] expected = new String[] { "scriptPath", "--master", "master_url", "--conf", "name1=value1", "--conf", "name2=value 2", "--driver-memory", "driverMemory", "--executor-memory", "executorMemory", "--class", "class_name", "--jars", "file: Assert.assertArrayEquals( expected, ss.getCmds().toArray() ); ss.setJobType( JOB_TYPE_PYTHON ); ss.setPyFile( "pyFile-path" ); expected = new String[] { "scriptPath", "--master", "master_url", "--conf", "name1=value1", "--conf", "name2=value 2", "--driver-memory", "driverMemory", "--executor-memory", "executorMemory", "--py-files", "file: Assert.assertArrayEquals( expected, ss.getCmds().toArray() ); }
public List<String> getCmds() throws IOException { List<String> cmds = new ArrayList<String>(); cmds.add( environmentSubstitute( scriptPath ) ); cmds.add( "--master" ); cmds.add( environmentSubstitute( master ) ); for ( String confParam : configParams ) { cmds.add( "--conf" ); cmds.add( environmentSubstitute( confParam ) ); } if ( !Const.isEmpty( driverMemory ) ) { cmds.add( "--driver-memory" ); cmds.add( environmentSubstitute( driverMemory ) ); } if ( !Const.isEmpty( executorMemory ) ) { cmds.add( "--executor-memory" ); cmds.add( environmentSubstitute( executorMemory ) ); } switch ( jobType ) { case JOB_TYPE_JAVA_SCALA: { if ( !Const.isEmpty( className ) ) { cmds.add( "--class" ); cmds.add( environmentSubstitute( className ) ); } if ( !libs.isEmpty() ) { cmds.add( "--jars" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( resolvePath( environmentSubstitute( jar ) ) ); break; } case JOB_TYPE_PYTHON: { if ( !libs.isEmpty() ) { cmds.add( "--py-files" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( environmentSubstitute( pyFile ) ); break; } } if ( !Const.isEmpty( args ) ) { List<String> argArray = parseCommandLine( args ); for ( String anArg : argArray ) { if ( !Const.isEmpty( anArg ) ) { if ( anArg.startsWith( HADOOP_CLUSTER_PREFIX ) ) { anArg = resolvePath( environmentSubstitute( anArg ) ); } cmds.add( anArg ); } } } return cmds; }
JobEntrySparkSubmit extends JobEntryBase implements Cloneable, JobEntryInterface, JobEntryListener { public List<String> getCmds() throws IOException { List<String> cmds = new ArrayList<String>(); cmds.add( environmentSubstitute( scriptPath ) ); cmds.add( "--master" ); cmds.add( environmentSubstitute( master ) ); for ( String confParam : configParams ) { cmds.add( "--conf" ); cmds.add( environmentSubstitute( confParam ) ); } if ( !Const.isEmpty( driverMemory ) ) { cmds.add( "--driver-memory" ); cmds.add( environmentSubstitute( driverMemory ) ); } if ( !Const.isEmpty( executorMemory ) ) { cmds.add( "--executor-memory" ); cmds.add( environmentSubstitute( executorMemory ) ); } switch ( jobType ) { case JOB_TYPE_JAVA_SCALA: { if ( !Const.isEmpty( className ) ) { cmds.add( "--class" ); cmds.add( environmentSubstitute( className ) ); } if ( !libs.isEmpty() ) { cmds.add( "--jars" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( resolvePath( environmentSubstitute( jar ) ) ); break; } case JOB_TYPE_PYTHON: { if ( !libs.isEmpty() ) { cmds.add( "--py-files" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( environmentSubstitute( pyFile ) ); break; } } if ( !Const.isEmpty( args ) ) { List<String> argArray = parseCommandLine( args ); for ( String anArg : argArray ) { if ( !Const.isEmpty( anArg ) ) { if ( anArg.startsWith( HADOOP_CLUSTER_PREFIX ) ) { anArg = resolvePath( environmentSubstitute( anArg ) ); } cmds.add( anArg ); } } } return cmds; } }
JobEntrySparkSubmit extends JobEntryBase implements Cloneable, JobEntryInterface, JobEntryListener { public List<String> getCmds() throws IOException { List<String> cmds = new ArrayList<String>(); cmds.add( environmentSubstitute( scriptPath ) ); cmds.add( "--master" ); cmds.add( environmentSubstitute( master ) ); for ( String confParam : configParams ) { cmds.add( "--conf" ); cmds.add( environmentSubstitute( confParam ) ); } if ( !Const.isEmpty( driverMemory ) ) { cmds.add( "--driver-memory" ); cmds.add( environmentSubstitute( driverMemory ) ); } if ( !Const.isEmpty( executorMemory ) ) { cmds.add( "--executor-memory" ); cmds.add( environmentSubstitute( executorMemory ) ); } switch ( jobType ) { case JOB_TYPE_JAVA_SCALA: { if ( !Const.isEmpty( className ) ) { cmds.add( "--class" ); cmds.add( environmentSubstitute( className ) ); } if ( !libs.isEmpty() ) { cmds.add( "--jars" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( resolvePath( environmentSubstitute( jar ) ) ); break; } case JOB_TYPE_PYTHON: { if ( !libs.isEmpty() ) { cmds.add( "--py-files" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( environmentSubstitute( pyFile ) ); break; } } if ( !Const.isEmpty( args ) ) { List<String> argArray = parseCommandLine( args ); for ( String anArg : argArray ) { if ( !Const.isEmpty( anArg ) ) { if ( anArg.startsWith( HADOOP_CLUSTER_PREFIX ) ) { anArg = resolvePath( environmentSubstitute( anArg ) ); } cmds.add( anArg ); } } } return cmds; } JobEntrySparkSubmit( String n );  JobEntrySparkSubmit(); }
JobEntrySparkSubmit extends JobEntryBase implements Cloneable, JobEntryInterface, JobEntryListener { public List<String> getCmds() throws IOException { List<String> cmds = new ArrayList<String>(); cmds.add( environmentSubstitute( scriptPath ) ); cmds.add( "--master" ); cmds.add( environmentSubstitute( master ) ); for ( String confParam : configParams ) { cmds.add( "--conf" ); cmds.add( environmentSubstitute( confParam ) ); } if ( !Const.isEmpty( driverMemory ) ) { cmds.add( "--driver-memory" ); cmds.add( environmentSubstitute( driverMemory ) ); } if ( !Const.isEmpty( executorMemory ) ) { cmds.add( "--executor-memory" ); cmds.add( environmentSubstitute( executorMemory ) ); } switch ( jobType ) { case JOB_TYPE_JAVA_SCALA: { if ( !Const.isEmpty( className ) ) { cmds.add( "--class" ); cmds.add( environmentSubstitute( className ) ); } if ( !libs.isEmpty() ) { cmds.add( "--jars" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( resolvePath( environmentSubstitute( jar ) ) ); break; } case JOB_TYPE_PYTHON: { if ( !libs.isEmpty() ) { cmds.add( "--py-files" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( environmentSubstitute( pyFile ) ); break; } } if ( !Const.isEmpty( args ) ) { List<String> argArray = parseCommandLine( args ); for ( String anArg : argArray ) { if ( !Const.isEmpty( anArg ) ) { if ( anArg.startsWith( HADOOP_CLUSTER_PREFIX ) ) { anArg = resolvePath( environmentSubstitute( anArg ) ); } cmds.add( anArg ); } } } return cmds; } JobEntrySparkSubmit( String n );  JobEntrySparkSubmit(); Object clone(); String getXML(); void loadXML( Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers, Repository rep,
                       IMetaStore metaStore ); void loadRep( Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List<DatabaseMeta> databases,
                       List<SlaveServer> slaveServers ); void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_job ); String getScriptPath(); void setScriptPath( String scriptPath ); String getMaster(); void setMaster( String master ); List<String> getConfigParams(); void setConfigParams( List<String> configParams ); Map<String, String> getLibs(); void setLibs( Map<String, String> docs ); String getJar(); void setJar( String jar ); String getClassName(); void setClassName( String className ); String getArgs(); void setArgs( String args ); String getExecutorMemory(); void setExecutorMemory( String executorMemory ); String getDriverMemory(); void setDriverMemory( String driverMemory ); boolean isBlockExecution(); void setBlockExecution( boolean blockExecution ); String getJobType(); void setJobType( String jobType ); String getPyFile(); void setPyFile( String pyFile ); List<String> getCmds(); Result execute( Result result, int nr ); boolean evaluates(); @Override void check( List<CheckResultInterface> remarks, JobMeta jobMeta, VariableSpace space, Repository repository,
                     IMetaStore metaStore ); List<String> parseCommandLine( String commandLineString ); @Override void afterExecution( Job arg0, JobEntryCopy arg1, JobEntryInterface arg2, Result arg3 ); @Override void beforeExecution( Job arg0, JobEntryCopy arg1, JobEntryInterface arg2 ); }
JobEntrySparkSubmit extends JobEntryBase implements Cloneable, JobEntryInterface, JobEntryListener { public List<String> getCmds() throws IOException { List<String> cmds = new ArrayList<String>(); cmds.add( environmentSubstitute( scriptPath ) ); cmds.add( "--master" ); cmds.add( environmentSubstitute( master ) ); for ( String confParam : configParams ) { cmds.add( "--conf" ); cmds.add( environmentSubstitute( confParam ) ); } if ( !Const.isEmpty( driverMemory ) ) { cmds.add( "--driver-memory" ); cmds.add( environmentSubstitute( driverMemory ) ); } if ( !Const.isEmpty( executorMemory ) ) { cmds.add( "--executor-memory" ); cmds.add( environmentSubstitute( executorMemory ) ); } switch ( jobType ) { case JOB_TYPE_JAVA_SCALA: { if ( !Const.isEmpty( className ) ) { cmds.add( "--class" ); cmds.add( environmentSubstitute( className ) ); } if ( !libs.isEmpty() ) { cmds.add( "--jars" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( resolvePath( environmentSubstitute( jar ) ) ); break; } case JOB_TYPE_PYTHON: { if ( !libs.isEmpty() ) { cmds.add( "--py-files" ); cmds.add( environmentSubstitute( Joiner.on( ',' ).join( libs.keySet() ) ) ); } cmds.add( environmentSubstitute( pyFile ) ); break; } } if ( !Const.isEmpty( args ) ) { List<String> argArray = parseCommandLine( args ); for ( String anArg : argArray ) { if ( !Const.isEmpty( anArg ) ) { if ( anArg.startsWith( HADOOP_CLUSTER_PREFIX ) ) { anArg = resolvePath( environmentSubstitute( anArg ) ); } cmds.add( anArg ); } } } return cmds; } JobEntrySparkSubmit( String n );  JobEntrySparkSubmit(); Object clone(); String getXML(); void loadXML( Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers, Repository rep,
                       IMetaStore metaStore ); void loadRep( Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List<DatabaseMeta> databases,
                       List<SlaveServer> slaveServers ); void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_job ); String getScriptPath(); void setScriptPath( String scriptPath ); String getMaster(); void setMaster( String master ); List<String> getConfigParams(); void setConfigParams( List<String> configParams ); Map<String, String> getLibs(); void setLibs( Map<String, String> docs ); String getJar(); void setJar( String jar ); String getClassName(); void setClassName( String className ); String getArgs(); void setArgs( String args ); String getExecutorMemory(); void setExecutorMemory( String executorMemory ); String getDriverMemory(); void setDriverMemory( String driverMemory ); boolean isBlockExecution(); void setBlockExecution( boolean blockExecution ); String getJobType(); void setJobType( String jobType ); String getPyFile(); void setPyFile( String pyFile ); List<String> getCmds(); Result execute( Result result, int nr ); boolean evaluates(); @Override void check( List<CheckResultInterface> remarks, JobMeta jobMeta, VariableSpace space, Repository repository,
                     IMetaStore metaStore ); List<String> parseCommandLine( String commandLineString ); @Override void afterExecution( Job arg0, JobEntryCopy arg1, JobEntryInterface arg2, Result arg3 ); @Override void beforeExecution( Job arg0, JobEntryCopy arg1, JobEntryInterface arg2 ); static final String JOB_TYPE_JAVA_SCALA; static final String JOB_TYPE_PYTHON; static final String HADOOP_CLUSTER_PREFIX; }