@Test public void testMainArguments() { final Props jobProps = new Props(); jobProps.put(TonyJobArg.HDFS_CLASSPATH.azPropName, "hdfs: jobProps.put(TonyJob.WORKER_ENV_PREFIX + "E1", "e1"); jobProps.put(TonyJob.WORKER_ENV_PREFIX + "E2", "e2"); final TonyJob tonyJob = new TonyJob("test_tony_job", new Props(), jobProps, log) { @Override public String getWorkingDirectory() { return System.getProperty("java.io.tmpdir"); } }; String args = tonyJob.getMainArguments(); Assert.assertTrue(args.contains(TonyJobArg.HDFS_CLASSPATH.tonyParamName + " hdfs: Assert.assertTrue(args.contains(TonyJobArg.SHELL_ENV.tonyParamName + " E2=e2")); Assert.assertTrue(args.contains(TonyJobArg.SHELL_ENV.tonyParamName + " E1=e1")); }
@Override protected String getMainArguments() { StringBuilder args = new StringBuilder(super.getMainArguments()); info("All job props: " + getJobProps()); info("All sys props: " + getSysProps()); String srcDir = getJobProps().getString(TonyJobArg.SRC_DIR.azPropName, "src"); args.append(" " + TonyJobArg.SRC_DIR.tonyParamName + " " + srcDir); String hdfsClasspath = getJobProps().getString(TonyJobArg.HDFS_CLASSPATH.azPropName, null); if (hdfsClasspath != null) { args.append(" " + TonyJobArg.HDFS_CLASSPATH.tonyParamName + " " + hdfsClasspath); } Map<String, String> workerEnvs = getJobProps().getMapByPrefix(WORKER_ENV_PREFIX); for (Map.Entry<String, String> workerEnv : workerEnvs.entrySet()) { args.append(" " + TonyJobArg.SHELL_ENV.tonyParamName + " " + workerEnv.getKey() + "=" + workerEnv.getValue()); } String taskParams = getJobProps().getString(TonyJobArg.TASK_PARAMS.azPropName, null); if (taskParams != null) { args.append(" " + TonyJobArg.TASK_PARAMS.tonyParamName + " '" + taskParams + "'"); } String pythonBinaryPath = getJobProps().getString(TonyJobArg.PYTHON_BINARY_PATH.azPropName, null); if (pythonBinaryPath != null) { args.append(" " + TonyJobArg.PYTHON_BINARY_PATH.tonyParamName + " " + pythonBinaryPath); } String pythonVenv = getJobProps().getString(TonyJobArg.PYTHON_VENV.azPropName, null); if (pythonVenv != null) { args.append(" " + TonyJobArg.PYTHON_VENV.tonyParamName + " " + pythonVenv); } String executes = getJobProps().getString(TonyJobArg.EXECUTES.azPropName, null); if (executes != null) { args.append(" " + TonyJobArg.EXECUTES.tonyParamName + " " + executes); } info("Complete main arguments: " + args); return args.toString(); }
TonyJob extends HadoopJavaJob { @Override protected String getMainArguments() { StringBuilder args = new StringBuilder(super.getMainArguments()); info("All job props: " + getJobProps()); info("All sys props: " + getSysProps()); String srcDir = getJobProps().getString(TonyJobArg.SRC_DIR.azPropName, "src"); args.append(" " + TonyJobArg.SRC_DIR.tonyParamName + " " + srcDir); String hdfsClasspath = getJobProps().getString(TonyJobArg.HDFS_CLASSPATH.azPropName, null); if (hdfsClasspath != null) { args.append(" " + TonyJobArg.HDFS_CLASSPATH.tonyParamName + " " + hdfsClasspath); } Map<String, String> workerEnvs = getJobProps().getMapByPrefix(WORKER_ENV_PREFIX); for (Map.Entry<String, String> workerEnv : workerEnvs.entrySet()) { args.append(" " + TonyJobArg.SHELL_ENV.tonyParamName + " " + workerEnv.getKey() + "=" + workerEnv.getValue()); } String taskParams = getJobProps().getString(TonyJobArg.TASK_PARAMS.azPropName, null); if (taskParams != null) { args.append(" " + TonyJobArg.TASK_PARAMS.tonyParamName + " '" + taskParams + "'"); } String pythonBinaryPath = getJobProps().getString(TonyJobArg.PYTHON_BINARY_PATH.azPropName, null); if (pythonBinaryPath != null) { args.append(" " + TonyJobArg.PYTHON_BINARY_PATH.tonyParamName + " " + pythonBinaryPath); } String pythonVenv = getJobProps().getString(TonyJobArg.PYTHON_VENV.azPropName, null); if (pythonVenv != null) { args.append(" " + TonyJobArg.PYTHON_VENV.tonyParamName + " " + pythonVenv); } String executes = getJobProps().getString(TonyJobArg.EXECUTES.azPropName, null); if (executes != null) { args.append(" " + TonyJobArg.EXECUTES.tonyParamName + " " + executes); } info("Complete main arguments: " + args); return args.toString(); } }
TonyJob extends HadoopJavaJob { @Override protected String getMainArguments() { StringBuilder args = new StringBuilder(super.getMainArguments()); info("All job props: " + getJobProps()); info("All sys props: " + getSysProps()); String srcDir = getJobProps().getString(TonyJobArg.SRC_DIR.azPropName, "src"); args.append(" " + TonyJobArg.SRC_DIR.tonyParamName + " " + srcDir); String hdfsClasspath = getJobProps().getString(TonyJobArg.HDFS_CLASSPATH.azPropName, null); if (hdfsClasspath != null) { args.append(" " + TonyJobArg.HDFS_CLASSPATH.tonyParamName + " " + hdfsClasspath); } Map<String, String> workerEnvs = getJobProps().getMapByPrefix(WORKER_ENV_PREFIX); for (Map.Entry<String, String> workerEnv : workerEnvs.entrySet()) { args.append(" " + TonyJobArg.SHELL_ENV.tonyParamName + " " + workerEnv.getKey() + "=" + workerEnv.getValue()); } String taskParams = getJobProps().getString(TonyJobArg.TASK_PARAMS.azPropName, null); if (taskParams != null) { args.append(" " + TonyJobArg.TASK_PARAMS.tonyParamName + " '" + taskParams + "'"); } String pythonBinaryPath = getJobProps().getString(TonyJobArg.PYTHON_BINARY_PATH.azPropName, null); if (pythonBinaryPath != null) { args.append(" " + TonyJobArg.PYTHON_BINARY_PATH.tonyParamName + " " + pythonBinaryPath); } String pythonVenv = getJobProps().getString(TonyJobArg.PYTHON_VENV.azPropName, null); if (pythonVenv != null) { args.append(" " + TonyJobArg.PYTHON_VENV.tonyParamName + " " + pythonVenv); } String executes = getJobProps().getString(TonyJobArg.EXECUTES.azPropName, null); if (executes != null) { args.append(" " + TonyJobArg.EXECUTES.tonyParamName + " " + executes); } info("Complete main arguments: " + args); return args.toString(); } TonyJob(String jobid, Props sysProps, Props jobProps, Logger log); }
TonyJob extends HadoopJavaJob { @Override protected String getMainArguments() { StringBuilder args = new StringBuilder(super.getMainArguments()); info("All job props: " + getJobProps()); info("All sys props: " + getSysProps()); String srcDir = getJobProps().getString(TonyJobArg.SRC_DIR.azPropName, "src"); args.append(" " + TonyJobArg.SRC_DIR.tonyParamName + " " + srcDir); String hdfsClasspath = getJobProps().getString(TonyJobArg.HDFS_CLASSPATH.azPropName, null); if (hdfsClasspath != null) { args.append(" " + TonyJobArg.HDFS_CLASSPATH.tonyParamName + " " + hdfsClasspath); } Map<String, String> workerEnvs = getJobProps().getMapByPrefix(WORKER_ENV_PREFIX); for (Map.Entry<String, String> workerEnv : workerEnvs.entrySet()) { args.append(" " + TonyJobArg.SHELL_ENV.tonyParamName + " " + workerEnv.getKey() + "=" + workerEnv.getValue()); } String taskParams = getJobProps().getString(TonyJobArg.TASK_PARAMS.azPropName, null); if (taskParams != null) { args.append(" " + TonyJobArg.TASK_PARAMS.tonyParamName + " '" + taskParams + "'"); } String pythonBinaryPath = getJobProps().getString(TonyJobArg.PYTHON_BINARY_PATH.azPropName, null); if (pythonBinaryPath != null) { args.append(" " + TonyJobArg.PYTHON_BINARY_PATH.tonyParamName + " " + pythonBinaryPath); } String pythonVenv = getJobProps().getString(TonyJobArg.PYTHON_VENV.azPropName, null); if (pythonVenv != null) { args.append(" " + TonyJobArg.PYTHON_VENV.tonyParamName + " " + pythonVenv); } String executes = getJobProps().getString(TonyJobArg.EXECUTES.azPropName, null); if (executes != null) { args.append(" " + TonyJobArg.EXECUTES.tonyParamName + " " + executes); } info("Complete main arguments: " + args); return args.toString(); } TonyJob(String jobid, Props sysProps, Props jobProps, Logger log); Configuration getTonyJobConf(); @Override void run(); @Override String getJavaClass(); }
TonyJob extends HadoopJavaJob { @Override protected String getMainArguments() { StringBuilder args = new StringBuilder(super.getMainArguments()); info("All job props: " + getJobProps()); info("All sys props: " + getSysProps()); String srcDir = getJobProps().getString(TonyJobArg.SRC_DIR.azPropName, "src"); args.append(" " + TonyJobArg.SRC_DIR.tonyParamName + " " + srcDir); String hdfsClasspath = getJobProps().getString(TonyJobArg.HDFS_CLASSPATH.azPropName, null); if (hdfsClasspath != null) { args.append(" " + TonyJobArg.HDFS_CLASSPATH.tonyParamName + " " + hdfsClasspath); } Map<String, String> workerEnvs = getJobProps().getMapByPrefix(WORKER_ENV_PREFIX); for (Map.Entry<String, String> workerEnv : workerEnvs.entrySet()) { args.append(" " + TonyJobArg.SHELL_ENV.tonyParamName + " " + workerEnv.getKey() + "=" + workerEnv.getValue()); } String taskParams = getJobProps().getString(TonyJobArg.TASK_PARAMS.azPropName, null); if (taskParams != null) { args.append(" " + TonyJobArg.TASK_PARAMS.tonyParamName + " '" + taskParams + "'"); } String pythonBinaryPath = getJobProps().getString(TonyJobArg.PYTHON_BINARY_PATH.azPropName, null); if (pythonBinaryPath != null) { args.append(" " + TonyJobArg.PYTHON_BINARY_PATH.tonyParamName + " " + pythonBinaryPath); } String pythonVenv = getJobProps().getString(TonyJobArg.PYTHON_VENV.azPropName, null); if (pythonVenv != null) { args.append(" " + TonyJobArg.PYTHON_VENV.tonyParamName + " " + pythonVenv); } String executes = getJobProps().getString(TonyJobArg.EXECUTES.azPropName, null); if (executes != null) { args.append(" " + TonyJobArg.EXECUTES.tonyParamName + " " + executes); } info("Complete main arguments: " + args); return args.toString(); } TonyJob(String jobid, Props sysProps, Props jobProps, Logger log); Configuration getTonyJobConf(); @Override void run(); @Override String getJavaClass(); static final String HADOOP_OPTS; static final String HADOOP_GLOBAL_OPTS; static final String WORKER_ENV_PREFIX; static final String TONY_APPLICATION_TAGS; }