@Test public void testExecuteShell() throws IOException, InterruptedException { assertEquals(Utils.executeShell("foo", 0, null), 127); }
public static int executeShell(String taskCommand, long timeout, Map<String, String> env) throws IOException, InterruptedException { LOG.info("Executing command: " + taskCommand); String executablePath = taskCommand.trim().split(" ")[0]; File executable = new File(executablePath); if (!executable.canExecute()) { if (!executable.setExecutable(true)) { LOG.warn("Failed to make " + executable + " executable"); } } if (System.getenv(Constants.SKIP_HADOOP_PATH) == null) { taskCommand = Constants.HADOOP_CLASSPATH_COMMAND + taskCommand; } ProcessBuilder taskProcessBuilder = new ProcessBuilder("bash", "-c", taskCommand); taskProcessBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.environment().remove("MALLOC_ARENA_MAX"); if (env != null) { taskProcessBuilder.environment().putAll(env); } Process taskProcess = taskProcessBuilder.start(); if (timeout > 0) { taskProcess.waitFor(timeout, TimeUnit.MILLISECONDS); } else { taskProcess.waitFor(); } return taskProcess.exitValue(); }
Utils { public static int executeShell(String taskCommand, long timeout, Map<String, String> env) throws IOException, InterruptedException { LOG.info("Executing command: " + taskCommand); String executablePath = taskCommand.trim().split(" ")[0]; File executable = new File(executablePath); if (!executable.canExecute()) { if (!executable.setExecutable(true)) { LOG.warn("Failed to make " + executable + " executable"); } } if (System.getenv(Constants.SKIP_HADOOP_PATH) == null) { taskCommand = Constants.HADOOP_CLASSPATH_COMMAND + taskCommand; } ProcessBuilder taskProcessBuilder = new ProcessBuilder("bash", "-c", taskCommand); taskProcessBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.environment().remove("MALLOC_ARENA_MAX"); if (env != null) { taskProcessBuilder.environment().putAll(env); } Process taskProcess = taskProcessBuilder.start(); if (timeout > 0) { taskProcess.waitFor(timeout, TimeUnit.MILLISECONDS); } else { taskProcess.waitFor(); } return taskProcess.exitValue(); } }
Utils { public static int executeShell(String taskCommand, long timeout, Map<String, String> env) throws IOException, InterruptedException { LOG.info("Executing command: " + taskCommand); String executablePath = taskCommand.trim().split(" ")[0]; File executable = new File(executablePath); if (!executable.canExecute()) { if (!executable.setExecutable(true)) { LOG.warn("Failed to make " + executable + " executable"); } } if (System.getenv(Constants.SKIP_HADOOP_PATH) == null) { taskCommand = Constants.HADOOP_CLASSPATH_COMMAND + taskCommand; } ProcessBuilder taskProcessBuilder = new ProcessBuilder("bash", "-c", taskCommand); taskProcessBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.environment().remove("MALLOC_ARENA_MAX"); if (env != null) { taskProcessBuilder.environment().putAll(env); } Process taskProcess = taskProcessBuilder.start(); if (timeout > 0) { taskProcess.waitFor(timeout, TimeUnit.MILLISECONDS); } else { taskProcess.waitFor(); } return taskProcess.exitValue(); } private  Utils(); }
Utils { public static int executeShell(String taskCommand, long timeout, Map<String, String> env) throws IOException, InterruptedException { LOG.info("Executing command: " + taskCommand); String executablePath = taskCommand.trim().split(" ")[0]; File executable = new File(executablePath); if (!executable.canExecute()) { if (!executable.setExecutable(true)) { LOG.warn("Failed to make " + executable + " executable"); } } if (System.getenv(Constants.SKIP_HADOOP_PATH) == null) { taskCommand = Constants.HADOOP_CLASSPATH_COMMAND + taskCommand; } ProcessBuilder taskProcessBuilder = new ProcessBuilder("bash", "-c", taskCommand); taskProcessBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.environment().remove("MALLOC_ARENA_MAX"); if (env != null) { taskProcessBuilder.environment().putAll(env); } Process taskProcess = taskProcessBuilder.start(); if (timeout > 0) { taskProcess.waitFor(timeout, TimeUnit.MILLISECONDS); } else { taskProcess.waitFor(); } return taskProcess.exitValue(); } private  Utils(); static boolean poll(Callable<Boolean> func, int interval, int timeout); static T pollTillNonNull(Callable<T> func, int interval, int timeout); static String parseMemoryString(String memory); static void zipFolder(java.nio.file.Path sourceFolderPath, java.nio.file.Path zipPath); static void unzipArchive(String src, String dst); static void setCapabilityGPU(Resource resource, int gpuCount); static String constructUrl(String urlString); static String constructContainerUrl(Container container); static String constructContainerUrl(String nodeAddress, ContainerId containerId); static void printTaskUrl(TaskInfo taskInfo, Log log); static void printTonyPortalUrl(String portalUrl, String appId, Log log); static String getTonySrcZipName(String appId); static Map<String, String> parseKeyValue(String[] keyValues); static Options getCommonOptions(); static int executeShell(String taskCommand, long timeout, Map<String, String> env); static String getCurrentHostName(); static String getHostNameOrIpFromTokenConf(Configuration conf); static void addEnvironmentForResource(LocalResource resource, FileSystem fs, String envPrefix,
                                               Map<String, String> env); static Map<String, JobContainerRequest> parseContainerRequests(Configuration conf); static Set<String> getAllJobTypes(Configuration conf); static int getNumTotalTasks(Configuration conf); static String getTaskType(String confKey); static boolean isArchive(String path); static boolean renameFile(String oldName, String newName); static String constructTFConfig(String clusterSpec, String jobName, int taskIndex); static String getClientResourcesPath(String appId, String fileName); static void cleanupHDFSPath(Configuration hdfsConf, Path path); static void addResources(String[] resources, Map<String, LocalResource> resourcesMap, FileSystem fs); static void addResource(String path, Map<String, LocalResource> resourcesMap, FileSystem fs); static String buildRMUrl(Configuration yarnConf, String appId); static void printCompletedTrackedTasks(int completedTrackedTasks, int totalTrackedTasks); static String parseClusterSpecForPytorch(String clusterSpec); static String[] parseClusterSpecForMXNet(String clusterSpec); static String resolveNameToIpAddress(String hostname); static void createDirIfNotExists(FileSystem fs, Path dir, FsPermission permission); static String[] getUntrackedJobTypes(Configuration conf); static boolean isJobTypeTracked(String taskName, Configuration tonyConf); static String[] getStopOnFailureJobTypes(Configuration conf); static void uploadFileAndSetConfResources(Path hdfsPath, Path filePath, String fileName,
                                                   Configuration tonyConf, FileSystem fs,
                                                   LocalResourceType resourceType, String resourceKey); static String[] splitAddressPort(String hostname); static void appendConfResources(String key, String resource, Configuration tonyConf); static void initYarnConf(Configuration yarnConf); static void initHdfsConf(Configuration hdfsConf); static void extractResources(String appId); static Map<String, String> getContainerEnvForDocker(Configuration tonyConf, String jobType); static Map<String, String> linksToBeDisplayedOnPage(String jobId); }
Utils { public static int executeShell(String taskCommand, long timeout, Map<String, String> env) throws IOException, InterruptedException { LOG.info("Executing command: " + taskCommand); String executablePath = taskCommand.trim().split(" ")[0]; File executable = new File(executablePath); if (!executable.canExecute()) { if (!executable.setExecutable(true)) { LOG.warn("Failed to make " + executable + " executable"); } } if (System.getenv(Constants.SKIP_HADOOP_PATH) == null) { taskCommand = Constants.HADOOP_CLASSPATH_COMMAND + taskCommand; } ProcessBuilder taskProcessBuilder = new ProcessBuilder("bash", "-c", taskCommand); taskProcessBuilder.redirectError(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT); taskProcessBuilder.environment().remove("MALLOC_ARENA_MAX"); if (env != null) { taskProcessBuilder.environment().putAll(env); } Process taskProcess = taskProcessBuilder.start(); if (timeout > 0) { taskProcess.waitFor(timeout, TimeUnit.MILLISECONDS); } else { taskProcess.waitFor(); } return taskProcess.exitValue(); } private  Utils(); static boolean poll(Callable<Boolean> func, int interval, int timeout); static T pollTillNonNull(Callable<T> func, int interval, int timeout); static String parseMemoryString(String memory); static void zipFolder(java.nio.file.Path sourceFolderPath, java.nio.file.Path zipPath); static void unzipArchive(String src, String dst); static void setCapabilityGPU(Resource resource, int gpuCount); static String constructUrl(String urlString); static String constructContainerUrl(Container container); static String constructContainerUrl(String nodeAddress, ContainerId containerId); static void printTaskUrl(TaskInfo taskInfo, Log log); static void printTonyPortalUrl(String portalUrl, String appId, Log log); static String getTonySrcZipName(String appId); static Map<String, String> parseKeyValue(String[] keyValues); static Options getCommonOptions(); static int executeShell(String taskCommand, long timeout, Map<String, String> env); static String getCurrentHostName(); static String getHostNameOrIpFromTokenConf(Configuration conf); static void addEnvironmentForResource(LocalResource resource, FileSystem fs, String envPrefix,
                                               Map<String, String> env); static Map<String, JobContainerRequest> parseContainerRequests(Configuration conf); static Set<String> getAllJobTypes(Configuration conf); static int getNumTotalTasks(Configuration conf); static String getTaskType(String confKey); static boolean isArchive(String path); static boolean renameFile(String oldName, String newName); static String constructTFConfig(String clusterSpec, String jobName, int taskIndex); static String getClientResourcesPath(String appId, String fileName); static void cleanupHDFSPath(Configuration hdfsConf, Path path); static void addResources(String[] resources, Map<String, LocalResource> resourcesMap, FileSystem fs); static void addResource(String path, Map<String, LocalResource> resourcesMap, FileSystem fs); static String buildRMUrl(Configuration yarnConf, String appId); static void printCompletedTrackedTasks(int completedTrackedTasks, int totalTrackedTasks); static String parseClusterSpecForPytorch(String clusterSpec); static String[] parseClusterSpecForMXNet(String clusterSpec); static String resolveNameToIpAddress(String hostname); static void createDirIfNotExists(FileSystem fs, Path dir, FsPermission permission); static String[] getUntrackedJobTypes(Configuration conf); static boolean isJobTypeTracked(String taskName, Configuration tonyConf); static String[] getStopOnFailureJobTypes(Configuration conf); static void uploadFileAndSetConfResources(Path hdfsPath, Path filePath, String fileName,
                                                   Configuration tonyConf, FileSystem fs,
                                                   LocalResourceType resourceType, String resourceKey); static String[] splitAddressPort(String hostname); static void appendConfResources(String key, String resource, Configuration tonyConf); static void initYarnConf(Configuration yarnConf); static void initHdfsConf(Configuration hdfsConf); static void extractResources(String appId); static Map<String, String> getContainerEnvForDocker(Configuration tonyConf, String jobType); static Map<String, String> linksToBeDisplayedOnPage(String jobId);  }