@Test public void testBuildClusterMasterCommand() { final String script = CloudInitScriptBuilder.buildClusterMasterCommand(this.clusterName, this.nfsParentDir, this.sinkLogRootDir, this.awsConfDir, this.appWorkDir, this.masterS3ConfUri, this.masterS3ConfFiles, this.masterS3JarsUri, this.masterS3JarFiles, this.masterJarsDir, this.masterJvmMemory, Optional.<String>absent(), gobblinVersion); final String decodedScript = new String(Base64.decodeBase64(script)); Assert.assertEquals(decodedScript, this.expectedMasterCloudInitScript, "Master launcher cloud-init script not built as expected"); }
public static String buildClusterMasterCommand(String clusterName, String nfsParentDir, String sinkLogRootDir, String awsConfDir, String appWorkDir, String masterS3ConfUri, String masterS3ConfFiles, String masterS3JarsUri, String masterS3JarsFiles, String masterJarsDir, String masterJvmMemory, Optional<String> masterJvmArgs, Optional<String> gobblinVersion) { final StringBuilder cloudInitCmds = new StringBuilder().append(BASH).append("\n"); final String clusterMasterClassName = GobblinAWSClusterManager.class.getSimpleName(); final String nfsDir = nfsParentDir + clusterName; final String nfsShareDirCmd = String.format("echo '%s %s(%s)' | tee --append %s", nfsDir, NFS_SHARE_ALL_IPS, NFS_SHARE_DEFAULT_OPTS, NFS_CONF_FILE); cloudInitCmds.append("mkdir -p ").append(nfsDir).append(File.separator).append("1").append("\n"); cloudInitCmds.append(NFS_SERVER_INSTALL_CMD).append("\n"); cloudInitCmds.append(nfsShareDirCmd).append("\n"); cloudInitCmds.append(NFS_SERVER_START_CMD).append("\n"); cloudInitCmds.append(NFS_EXPORT_FS_CMD).append("\n"); cloudInitCmds.append("mkdir -p ").append(sinkLogRootDir).append("\n"); cloudInitCmds.append("chown -R ec2-user:ec2-user /home/ec2-user/*").append("\n"); if (gobblinVersion.isPresent()) { cloudInitCmds.append("vr=").append(gobblinVersion.get()).append("\n"); } cloudInitCmds.append("cgS3=").append(masterS3ConfUri).append("\n"); cloudInitCmds.append("cg=").append(awsConfDir).append("\n"); cloudInitCmds.append("jrS3=").append(masterS3JarsUri).append("\n"); cloudInitCmds.append("jr=").append(masterJarsDir).append("\n"); final StringBuilder classpath = new StringBuilder(); final List<String> awsConfs = SPLITTER.splitToList(masterS3ConfFiles); for (String awsConf : awsConfs) { cloudInitCmds.append(String.format("wget -P \"${cg}\" \"${cgS3}\"%s", awsConf)).append("\n"); } classpath.append(awsConfDir); final List<String> awsJars = SPLITTER.splitToList(masterS3JarsFiles); for (String awsJar : awsJars) { cloudInitCmds.append(String.format("wget -P \"${jr}\" \"${jrS3}\"%s", awsJar)).append("\n"); } classpath.append(":").append(masterJarsDir).append("*"); final StringBuilder launchGobblinClusterMasterCmd = new StringBuilder() .append("java") .append(" -cp ").append(classpath) .append(" -Xmx").append(masterJvmMemory) .append(" ").append(JvmUtils.formatJvmArguments(masterJvmArgs)) .append(" ").append(GobblinAWSClusterManager.class.getName()) .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) .append(" ").append(clusterName) .append(" --").append(GobblinAWSConfigurationKeys.APP_WORK_DIR) .append(" ").append(appWorkDir) .append(" 1>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDOUT) .append(" 2>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDERR); cloudInitCmds.append(launchGobblinClusterMasterCmd).append("\n"); final String cloudInitScript = cloudInitCmds.toString(); LOGGER.info("Cloud-init script for master node: " + cloudInitScript); return encodeBase64(cloudInitScript); }
CloudInitScriptBuilder { public static String buildClusterMasterCommand(String clusterName, String nfsParentDir, String sinkLogRootDir, String awsConfDir, String appWorkDir, String masterS3ConfUri, String masterS3ConfFiles, String masterS3JarsUri, String masterS3JarsFiles, String masterJarsDir, String masterJvmMemory, Optional<String> masterJvmArgs, Optional<String> gobblinVersion) { final StringBuilder cloudInitCmds = new StringBuilder().append(BASH).append("\n"); final String clusterMasterClassName = GobblinAWSClusterManager.class.getSimpleName(); final String nfsDir = nfsParentDir + clusterName; final String nfsShareDirCmd = String.format("echo '%s %s(%s)' | tee --append %s", nfsDir, NFS_SHARE_ALL_IPS, NFS_SHARE_DEFAULT_OPTS, NFS_CONF_FILE); cloudInitCmds.append("mkdir -p ").append(nfsDir).append(File.separator).append("1").append("\n"); cloudInitCmds.append(NFS_SERVER_INSTALL_CMD).append("\n"); cloudInitCmds.append(nfsShareDirCmd).append("\n"); cloudInitCmds.append(NFS_SERVER_START_CMD).append("\n"); cloudInitCmds.append(NFS_EXPORT_FS_CMD).append("\n"); cloudInitCmds.append("mkdir -p ").append(sinkLogRootDir).append("\n"); cloudInitCmds.append("chown -R ec2-user:ec2-user /home/ec2-user/*").append("\n"); if (gobblinVersion.isPresent()) { cloudInitCmds.append("vr=").append(gobblinVersion.get()).append("\n"); } cloudInitCmds.append("cgS3=").append(masterS3ConfUri).append("\n"); cloudInitCmds.append("cg=").append(awsConfDir).append("\n"); cloudInitCmds.append("jrS3=").append(masterS3JarsUri).append("\n"); cloudInitCmds.append("jr=").append(masterJarsDir).append("\n"); final StringBuilder classpath = new StringBuilder(); final List<String> awsConfs = SPLITTER.splitToList(masterS3ConfFiles); for (String awsConf : awsConfs) { cloudInitCmds.append(String.format("wget -P \"${cg}\" \"${cgS3}\"%s", awsConf)).append("\n"); } classpath.append(awsConfDir); final List<String> awsJars = SPLITTER.splitToList(masterS3JarsFiles); for (String awsJar : awsJars) { cloudInitCmds.append(String.format("wget -P \"${jr}\" \"${jrS3}\"%s", awsJar)).append("\n"); } classpath.append(":").append(masterJarsDir).append("*"); final StringBuilder launchGobblinClusterMasterCmd = new StringBuilder() .append("java") .append(" -cp ").append(classpath) .append(" -Xmx").append(masterJvmMemory) .append(" ").append(JvmUtils.formatJvmArguments(masterJvmArgs)) .append(" ").append(GobblinAWSClusterManager.class.getName()) .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) .append(" ").append(clusterName) .append(" --").append(GobblinAWSConfigurationKeys.APP_WORK_DIR) .append(" ").append(appWorkDir) .append(" 1>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDOUT) .append(" 2>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDERR); cloudInitCmds.append(launchGobblinClusterMasterCmd).append("\n"); final String cloudInitScript = cloudInitCmds.toString(); LOGGER.info("Cloud-init script for master node: " + cloudInitScript); return encodeBase64(cloudInitScript); } }
CloudInitScriptBuilder { public static String buildClusterMasterCommand(String clusterName, String nfsParentDir, String sinkLogRootDir, String awsConfDir, String appWorkDir, String masterS3ConfUri, String masterS3ConfFiles, String masterS3JarsUri, String masterS3JarsFiles, String masterJarsDir, String masterJvmMemory, Optional<String> masterJvmArgs, Optional<String> gobblinVersion) { final StringBuilder cloudInitCmds = new StringBuilder().append(BASH).append("\n"); final String clusterMasterClassName = GobblinAWSClusterManager.class.getSimpleName(); final String nfsDir = nfsParentDir + clusterName; final String nfsShareDirCmd = String.format("echo '%s %s(%s)' | tee --append %s", nfsDir, NFS_SHARE_ALL_IPS, NFS_SHARE_DEFAULT_OPTS, NFS_CONF_FILE); cloudInitCmds.append("mkdir -p ").append(nfsDir).append(File.separator).append("1").append("\n"); cloudInitCmds.append(NFS_SERVER_INSTALL_CMD).append("\n"); cloudInitCmds.append(nfsShareDirCmd).append("\n"); cloudInitCmds.append(NFS_SERVER_START_CMD).append("\n"); cloudInitCmds.append(NFS_EXPORT_FS_CMD).append("\n"); cloudInitCmds.append("mkdir -p ").append(sinkLogRootDir).append("\n"); cloudInitCmds.append("chown -R ec2-user:ec2-user /home/ec2-user/*").append("\n"); if (gobblinVersion.isPresent()) { cloudInitCmds.append("vr=").append(gobblinVersion.get()).append("\n"); } cloudInitCmds.append("cgS3=").append(masterS3ConfUri).append("\n"); cloudInitCmds.append("cg=").append(awsConfDir).append("\n"); cloudInitCmds.append("jrS3=").append(masterS3JarsUri).append("\n"); cloudInitCmds.append("jr=").append(masterJarsDir).append("\n"); final StringBuilder classpath = new StringBuilder(); final List<String> awsConfs = SPLITTER.splitToList(masterS3ConfFiles); for (String awsConf : awsConfs) { cloudInitCmds.append(String.format("wget -P \"${cg}\" \"${cgS3}\"%s", awsConf)).append("\n"); } classpath.append(awsConfDir); final List<String> awsJars = SPLITTER.splitToList(masterS3JarsFiles); for (String awsJar : awsJars) { cloudInitCmds.append(String.format("wget -P \"${jr}\" \"${jrS3}\"%s", awsJar)).append("\n"); } classpath.append(":").append(masterJarsDir).append("*"); final StringBuilder launchGobblinClusterMasterCmd = new StringBuilder() .append("java") .append(" -cp ").append(classpath) .append(" -Xmx").append(masterJvmMemory) .append(" ").append(JvmUtils.formatJvmArguments(masterJvmArgs)) .append(" ").append(GobblinAWSClusterManager.class.getName()) .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) .append(" ").append(clusterName) .append(" --").append(GobblinAWSConfigurationKeys.APP_WORK_DIR) .append(" ").append(appWorkDir) .append(" 1>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDOUT) .append(" 2>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDERR); cloudInitCmds.append(launchGobblinClusterMasterCmd).append("\n"); final String cloudInitScript = cloudInitCmds.toString(); LOGGER.info("Cloud-init script for master node: " + cloudInitScript); return encodeBase64(cloudInitScript); }  }
CloudInitScriptBuilder { public static String buildClusterMasterCommand(String clusterName, String nfsParentDir, String sinkLogRootDir, String awsConfDir, String appWorkDir, String masterS3ConfUri, String masterS3ConfFiles, String masterS3JarsUri, String masterS3JarsFiles, String masterJarsDir, String masterJvmMemory, Optional<String> masterJvmArgs, Optional<String> gobblinVersion) { final StringBuilder cloudInitCmds = new StringBuilder().append(BASH).append("\n"); final String clusterMasterClassName = GobblinAWSClusterManager.class.getSimpleName(); final String nfsDir = nfsParentDir + clusterName; final String nfsShareDirCmd = String.format("echo '%s %s(%s)' | tee --append %s", nfsDir, NFS_SHARE_ALL_IPS, NFS_SHARE_DEFAULT_OPTS, NFS_CONF_FILE); cloudInitCmds.append("mkdir -p ").append(nfsDir).append(File.separator).append("1").append("\n"); cloudInitCmds.append(NFS_SERVER_INSTALL_CMD).append("\n"); cloudInitCmds.append(nfsShareDirCmd).append("\n"); cloudInitCmds.append(NFS_SERVER_START_CMD).append("\n"); cloudInitCmds.append(NFS_EXPORT_FS_CMD).append("\n"); cloudInitCmds.append("mkdir -p ").append(sinkLogRootDir).append("\n"); cloudInitCmds.append("chown -R ec2-user:ec2-user /home/ec2-user/*").append("\n"); if (gobblinVersion.isPresent()) { cloudInitCmds.append("vr=").append(gobblinVersion.get()).append("\n"); } cloudInitCmds.append("cgS3=").append(masterS3ConfUri).append("\n"); cloudInitCmds.append("cg=").append(awsConfDir).append("\n"); cloudInitCmds.append("jrS3=").append(masterS3JarsUri).append("\n"); cloudInitCmds.append("jr=").append(masterJarsDir).append("\n"); final StringBuilder classpath = new StringBuilder(); final List<String> awsConfs = SPLITTER.splitToList(masterS3ConfFiles); for (String awsConf : awsConfs) { cloudInitCmds.append(String.format("wget -P \"${cg}\" \"${cgS3}\"%s", awsConf)).append("\n"); } classpath.append(awsConfDir); final List<String> awsJars = SPLITTER.splitToList(masterS3JarsFiles); for (String awsJar : awsJars) { cloudInitCmds.append(String.format("wget -P \"${jr}\" \"${jrS3}\"%s", awsJar)).append("\n"); } classpath.append(":").append(masterJarsDir).append("*"); final StringBuilder launchGobblinClusterMasterCmd = new StringBuilder() .append("java") .append(" -cp ").append(classpath) .append(" -Xmx").append(masterJvmMemory) .append(" ").append(JvmUtils.formatJvmArguments(masterJvmArgs)) .append(" ").append(GobblinAWSClusterManager.class.getName()) .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) .append(" ").append(clusterName) .append(" --").append(GobblinAWSConfigurationKeys.APP_WORK_DIR) .append(" ").append(appWorkDir) .append(" 1>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDOUT) .append(" 2>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDERR); cloudInitCmds.append(launchGobblinClusterMasterCmd).append("\n"); final String cloudInitScript = cloudInitCmds.toString(); LOGGER.info("Cloud-init script for master node: " + cloudInitScript); return encodeBase64(cloudInitScript); }  static String buildClusterMasterCommand(String clusterName, String nfsParentDir, String sinkLogRootDir,
      String awsConfDir, String appWorkDir,
      String masterS3ConfUri, String masterS3ConfFiles,
      String masterS3JarsUri, String masterS3JarsFiles, String masterJarsDir,
      String masterJvmMemory, Optional<String> masterJvmArgs, Optional<String> gobblinVersion); static String buildClusterWorkerCommand(String clusterName, String nfsParentDir, String sinkLogRootDir,
      String awsConfDir, String appWorkDir, String masterPublicIp,
      String workerS3ConfUri, String workerS3ConfFiles,
      String workerS3JarsUri, String workerS3JarsFiles, String workerJarsDir,
      String workerJvmMemory, Optional<String> workerJvmArgs, Optional<String> gobblinVersion); }
CloudInitScriptBuilder { public static String buildClusterMasterCommand(String clusterName, String nfsParentDir, String sinkLogRootDir, String awsConfDir, String appWorkDir, String masterS3ConfUri, String masterS3ConfFiles, String masterS3JarsUri, String masterS3JarsFiles, String masterJarsDir, String masterJvmMemory, Optional<String> masterJvmArgs, Optional<String> gobblinVersion) { final StringBuilder cloudInitCmds = new StringBuilder().append(BASH).append("\n"); final String clusterMasterClassName = GobblinAWSClusterManager.class.getSimpleName(); final String nfsDir = nfsParentDir + clusterName; final String nfsShareDirCmd = String.format("echo '%s %s(%s)' | tee --append %s", nfsDir, NFS_SHARE_ALL_IPS, NFS_SHARE_DEFAULT_OPTS, NFS_CONF_FILE); cloudInitCmds.append("mkdir -p ").append(nfsDir).append(File.separator).append("1").append("\n"); cloudInitCmds.append(NFS_SERVER_INSTALL_CMD).append("\n"); cloudInitCmds.append(nfsShareDirCmd).append("\n"); cloudInitCmds.append(NFS_SERVER_START_CMD).append("\n"); cloudInitCmds.append(NFS_EXPORT_FS_CMD).append("\n"); cloudInitCmds.append("mkdir -p ").append(sinkLogRootDir).append("\n"); cloudInitCmds.append("chown -R ec2-user:ec2-user /home/ec2-user/*").append("\n"); if (gobblinVersion.isPresent()) { cloudInitCmds.append("vr=").append(gobblinVersion.get()).append("\n"); } cloudInitCmds.append("cgS3=").append(masterS3ConfUri).append("\n"); cloudInitCmds.append("cg=").append(awsConfDir).append("\n"); cloudInitCmds.append("jrS3=").append(masterS3JarsUri).append("\n"); cloudInitCmds.append("jr=").append(masterJarsDir).append("\n"); final StringBuilder classpath = new StringBuilder(); final List<String> awsConfs = SPLITTER.splitToList(masterS3ConfFiles); for (String awsConf : awsConfs) { cloudInitCmds.append(String.format("wget -P \"${cg}\" \"${cgS3}\"%s", awsConf)).append("\n"); } classpath.append(awsConfDir); final List<String> awsJars = SPLITTER.splitToList(masterS3JarsFiles); for (String awsJar : awsJars) { cloudInitCmds.append(String.format("wget -P \"${jr}\" \"${jrS3}\"%s", awsJar)).append("\n"); } classpath.append(":").append(masterJarsDir).append("*"); final StringBuilder launchGobblinClusterMasterCmd = new StringBuilder() .append("java") .append(" -cp ").append(classpath) .append(" -Xmx").append(masterJvmMemory) .append(" ").append(JvmUtils.formatJvmArguments(masterJvmArgs)) .append(" ").append(GobblinAWSClusterManager.class.getName()) .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) .append(" ").append(clusterName) .append(" --").append(GobblinAWSConfigurationKeys.APP_WORK_DIR) .append(" ").append(appWorkDir) .append(" 1>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDOUT) .append(" 2>").append(sinkLogRootDir) .append(clusterMasterClassName).append(".") .append("master").append(".") .append(CloudInitScriptBuilder.STDERR); cloudInitCmds.append(launchGobblinClusterMasterCmd).append("\n"); final String cloudInitScript = cloudInitCmds.toString(); LOGGER.info("Cloud-init script for master node: " + cloudInitScript); return encodeBase64(cloudInitScript); }  static String buildClusterMasterCommand(String clusterName, String nfsParentDir, String sinkLogRootDir,
      String awsConfDir, String appWorkDir,
      String masterS3ConfUri, String masterS3ConfFiles,
      String masterS3JarsUri, String masterS3JarsFiles, String masterJarsDir,
      String masterJvmMemory, Optional<String> masterJvmArgs, Optional<String> gobblinVersion); static String buildClusterWorkerCommand(String clusterName, String nfsParentDir, String sinkLogRootDir,
      String awsConfDir, String appWorkDir, String masterPublicIp,
      String workerS3ConfUri, String workerS3ConfFiles,
      String workerS3JarsUri, String workerS3JarsFiles, String workerJarsDir,
      String workerJvmMemory, Optional<String> workerJvmArgs, Optional<String> gobblinVersion); static final String BASH; }