ServiceApiUtil { @VisibleForTesting public static void validateAndResolveService(Service service, SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws IOException { boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED, RegistryConstants.DEFAULT_DNS_ENABLED); if (dnsEnabled) { if (RegistryUtils.currentUser().length() > RegistryConstants.MAX_FQDN_LABEL_LENGTH) { throw new IllegalArgumentException( RestApiErrorMessages.ERROR_USER_NAME_INVALID); } userNamePattern.validate(RegistryUtils.currentUser()); } if (StringUtils.isEmpty(service.getName())) { throw new IllegalArgumentException( RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); } if (StringUtils.isEmpty(service.getVersion())) { throw new IllegalArgumentException(String.format( RestApiErrorMessages.ERROR_APPLICATION_VERSION_INVALID, service.getName())); } validateNameFormat(service.getName(), conf); if (!hasComponent(service)) { throw new IllegalArgumentException( "No component specified for " + service.getName()); } if (UserGroupInformation.isSecurityEnabled()) { validateKerberosPrincipal(service.getKerberosPrincipal()); } try { validateDockerClientConfiguration(service, conf); } catch (IOException e) { throw new IllegalArgumentException(e); } Configuration globalConf = service.getConfiguration(); Set<String> componentNames = new HashSet<>(); List<Component> componentsToRemove = new ArrayList<>(); List<Component> componentsToAdd = new ArrayList<>(); for (Component comp : service.getComponents()) { int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH; maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length(); if (dnsEnabled && comp.getName().length() > maxCompLength) { throw new IllegalArgumentException(String.format(RestApiErrorMessages .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName())); } if (service.getName().equals(comp.getName())) { throw new IllegalArgumentException(String.format(RestApiErrorMessages .ERROR_COMPONENT_NAME_CONFLICTS_WITH_SERVICE_NAME, comp.getName(), service.getName())); } if (componentNames.contains(comp.getName())) { throw new IllegalArgumentException("Component name collision: " + comp.getName()); } if (comp.getArtifact() != null && comp.getArtifact().getType() == Artifact.TypeEnum.SERVICE) { if (StringUtils.isEmpty(comp.getArtifact().getId())) { throw new IllegalArgumentException( RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); } LOG.info("Marking {} for removal", comp.getName()); componentsToRemove.add(comp); List<Component> externalComponents = getComponents(fs, comp.getArtifact().getId()); for (Component c : externalComponents) { Component override = service.getComponent(c.getName()); if (override != null && override.getArtifact() == null) { override.mergeFrom(c); LOG.info("Merging external component {} from external {}", c .getName(), comp.getName()); } else { if (componentNames.contains(c.getName())) { throw new IllegalArgumentException("Component name collision: " + c.getName()); } componentNames.add(c.getName()); componentsToAdd.add(c); LOG.info("Adding component {} from external {}", c.getName(), comp.getName()); } } } else { componentNames.add(comp.getName()); comp.getConfiguration().mergeFrom(globalConf); } } service.getComponents().removeAll(componentsToRemove); service.getComponents().addAll(componentsToAdd); Artifact globalArtifact = service.getArtifact(); Resource globalResource = service.getResource(); for (Component comp : service.getComponents()) { if (comp.getArtifact() == null && service.getArtifact() != null && service.getArtifact().getType() != Artifact.TypeEnum .SERVICE) { comp.setArtifact(globalArtifact); } if (comp.getResource() == null) { comp.setResource(globalResource); } if (comp.getDependencies() != null) { for (String dependency : comp.getDependencies()) { if (!componentNames.contains(dependency)) { throw new IllegalArgumentException(String.format( RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency, comp.getName())); } } } validateComponent(comp, fs.getFileSystem(), conf); } validatePlacementPolicy(service.getComponents(), componentNames); sortByDependencies(service.getComponents()); if (service.getLifetime() == null) { service.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); } } static  String(String s); @VisibleForTesting static void setJsonSerDeser(JsonSerDeser jsd); @VisibleForTesting static void validateAndResolveService(Service service,
      SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf); static void validateJvmOpts(String jvmOpts); static void validateKerberosPrincipal(
      KerberosPrincipal kerberosPrincipal); static void validateNameFormat(String name,
      org.apache.hadoop.conf.Configuration conf); @VisibleForTesting static List<Component> getComponents(SliderFileSystem
      fs, String serviceName); static Service loadService(SliderFileSystem fs, String
      serviceName); static Service loadServiceUpgrade(SliderFileSystem fs,
      String serviceName, String version); static Service loadServiceFrom(SliderFileSystem fs,
      Path appDefPath); static Path getServiceJsonPath(SliderFileSystem fs, String serviceName); static void validateCompResourceSize(
      org.apache.hadoop.yarn.api.records.Resource maxResource,
      Service service); static Collection<Component> sortByDependencies(List<Component>
      components); static void createDirAndPersistApp(SliderFileSystem fs, Path appDir,
      Service service); static Path writeAppDefinition(SliderFileSystem fs, Path appDir,
      Service service); static Path writeAppDefinition(SliderFileSystem fs, Service service); static List<Container> getLiveContainers(Service service,
      List<String> componentInstances); static void validateInstancesUpgrade(List<Container>
      liveContainers); static boolean isUpgradable(Container container); static List<Container> validateAndResolveCompsUpgrade(
      Service liveService, Collection<String> compNames); static List<Container> validateAndResolveCompsStable(
      Service liveService, Collection<String> compNames); static String getHostnameSuffix(String serviceName, org.apache
      .hadoop.conf.Configuration conf); static String parseAndValidateComponentInstanceName(String
      instanceOrHostname, String serviceName, org.apache.hadoop.conf
      .Configuration conf); static String parseComponentName(String componentInstanceName); static List<String> resolveCompsDependency(Service service); static void checkServiceDependencySatisified(Service service); static JsonSerDeser<Service> jsonSerDeser; static final JsonSerDeser<Container[]> CONTAINER_JSON_SERDE; static final JsonSerDeser<ComponentContainers[]> COMP_CONTAINERS_JSON_SERDE; static final JsonSerDeser<Component[]> COMP_JSON_SERDE; }
@Test(timeout = 90000) public void testResourceValidation() throws Exception { assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR .length()); SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service app = new Service(); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with no name"); } catch (IllegalArgumentException e) { assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); } app.setName("test"); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + " service with no version"); } catch (IllegalArgumentException e) { assertEquals(String.format(ERROR_APPLICATION_VERSION_INVALID, app.getName()), e.getMessage()); } app.setVersion("v1"); String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR}; for (String badName : badNames) { app.setName(badName); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName); } catch (IllegalArgumentException e) { } } app.setName(LEN_64_STR); Component comp = new Component().name("comp1"); app.addComponent(comp); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DEFAULT_DNS); Assert.fail(EXCEPTION_PREFIX + "service with no launch command"); } catch (IllegalArgumentException e) { assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND, e.getMessage()); } app.setName(LEN_64_STR.substring(0, RegistryConstants .MAX_FQDN_LABEL_LENGTH)); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with no launch command"); } catch (IllegalArgumentException e) { assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND, e.getMessage()); } comp.setLaunchCommand("sleep 1"); Resource res = new Resource(); app.setResource(res); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with no memory"); } catch (IllegalArgumentException e) { assertEquals(String.format( RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, comp.getName()), e.getMessage()); } res.setMemory("100mb"); res.setCpus(-2); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail( EXCEPTION_PREFIX + "service with invalid no of cpus"); } catch (IllegalArgumentException e) { assertEquals(String.format( RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, comp.getName()), e.getMessage()); } res.setCpus(2); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with no container count"); } catch (IllegalArgumentException e) { Assert.assertTrue(e.getMessage() .contains(ERROR_CONTAINERS_COUNT_INVALID)); } res.setProfile("hbase_finance_large"); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with resource profile along with cpus/memory"); } catch (IllegalArgumentException e) { assertEquals(String.format(RestApiErrorMessages .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, comp.getName()), e.getMessage()); } res.setCpus(null); res.setMemory(null); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with resource profile only"); } catch (IllegalArgumentException e) { assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET, e.getMessage()); } res.setProfile(null); res.setCpus(2); res.setMemory("2gb"); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "null number of containers"); } catch (IllegalArgumentException e) { Assert.assertTrue(e.getMessage() .startsWith(ERROR_CONTAINERS_COUNT_INVALID)); } }
@Test public void testArtifacts() throws IOException { SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service app = new Service(); app.setName("service1"); app.setVersion("v1"); Resource res = new Resource(); app.setResource(res); res.setMemory("512M"); Artifact artifact = new Artifact(); app.setArtifact(artifact); String compName = "comp1"; Component comp = ServiceTestUtils.createComponent(compName); app.setComponents(Collections.singletonList(comp)); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with no artifact id"); } catch (IllegalArgumentException e) { assertEquals(String.format(ERROR_ARTIFACT_ID_FOR_COMP_INVALID, compName), e.getMessage()); } artifact.setType(Artifact.TypeEnum.SERVICE); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with no artifact id"); } catch (IllegalArgumentException e) { assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); } artifact.setType(Artifact.TypeEnum.TARBALL); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with no artifact id"); } catch (IllegalArgumentException e) { assertEquals(String.format(ERROR_ARTIFACT_ID_FOR_COMP_INVALID, compName), e.getMessage()); } artifact.setType(Artifact.TypeEnum.DOCKER); artifact.setId("docker.io/centos:centos7"); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); } catch (IllegalArgumentException e) { LOG.error("service attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); } assertThat(app.getLifetime()).isEqualTo(DEFAULT_UNLIMITED_LIFETIME); }
@Test public void testDuplicateComponents() throws IOException { SliderFileSystem sfs = ServiceTestUtils.initMockFs(); String compName = "comp1"; Service app = createValidApplication(compName); app.addComponent(createValidComponent(compName)); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "service with component collision"); } catch (IllegalArgumentException e) { assertEquals("Component name collision: " + compName, e.getMessage()); } }
@Test public void testComponentNameSameAsServiceName() throws IOException { SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service app = new Service(); app.setName("test"); app.setVersion("v1"); app.addComponent(createValidComponent("test")); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); Assert.fail(EXCEPTION_PREFIX + "component name matches service name"); } catch (IllegalArgumentException e) { assertEquals("Component name test must not be same as service name test", e.getMessage()); } }
@Test public void testExternalDuplicateComponent() throws IOException { Service ext = createValidApplication("comp1"); SliderFileSystem sfs = ServiceTestUtils.initMockFs(ext); Service app = createValidApplication("comp1"); Artifact artifact = new Artifact(); artifact.setType(Artifact.TypeEnum.SERVICE); artifact.setId("id"); app.getComponent("comp1").setArtifact(artifact); try { ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); } catch (IllegalArgumentException e) { Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); } }
@Test public void testDependencySorting() throws IOException { Component a = ServiceTestUtils.createComponent("a"); Component b = ServiceTestUtils.createComponent("b"); Component c = ServiceTestUtils.createComponent("c"); Component d = ServiceTestUtils.createComponent("d").dependencies(Arrays.asList("c")); Component e = ServiceTestUtils.createComponent("e") .dependencies(Arrays.asList("b", "d")); verifyDependencySorting(Arrays.asList(a, b, c), a, b, c); verifyDependencySorting(Arrays.asList(c, a, b), c, a, b); verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e); verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e); c.setDependencies(Arrays.asList("e")); try { verifyDependencySorting(Arrays.asList(a, b, c, d, e)); Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle"); } catch (IllegalArgumentException ex) { assertEquals(String.format( RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d, e)), ex.getMessage()); } SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service service = createValidApplication(null); service.setComponents(Arrays.asList(c, d, e)); try { ServiceApiUtil.validateAndResolveService(service, sfs, CONF_DEFAULT_DNS); Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies"); } catch (IllegalArgumentException ex) { assertEquals(String.format( RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex .getMessage()); } }