AMSPropertyProvider extends MetricsPropertyProvider { static URIBuilder getAMSUriBuilder(String hostname, int port, boolean httpsEnabled) { URIBuilder uriBuilder = new URIBuilder(); uriBuilder.setScheme(httpsEnabled ? "https" : "http"); uriBuilder.setHost(hostname); uriBuilder.setPort(port); uriBuilder.setPath("/ws/v1/timeline/metrics"); return uriBuilder; } AMSPropertyProvider(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap,
                             URLStreamProvider streamProvider,
                             ComponentSSLConfiguration configuration,
                             TimelineMetricCacheProvider cacheProvider,
                             MetricHostProvider hostProvider,
                             String clusterNamePropertyId,
                             String hostNamePropertyId,
                             String componentNamePropertyId); @Override Set<String> checkPropertyIds(Set<String> propertyIds); @Override Set<Resource> populateResourcesWithProperties(Set<Resource> resources,
               Request request, Set<String> propertyIds); @Override Map<String, Map<String, PropertyInfo>> getComponentMetrics();  }
@Test public void testPopulateResourcesForSingleHostMetric() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(SINGLE_HOST_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Host); AMSPropertyProvider propertyProvider = new AMSHostPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID ); Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID1, new TemporalInfoImpl(1416445244800L, 1416448936474L, 15L)); Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID1), temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "cpu_user"); uriBuilder.addParameter("hostname", "h1"); uriBuilder.addParameter("appId", "HOST"); uriBuilder.addParameter("startTime", "1416445244800"); uriBuilder.addParameter("endTime", "1416448936474"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Number[][] val = (Number[][]) res.getPropertyValue(PROPERTY_ID1); Assert.assertNotNull("No value for property " + PROPERTY_ID1, val); Assert.assertEquals(111, val.length); }
@Test public void testPopulateResourcesForSingleHostMetricPointInTime() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(SINGLE_HOST_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Host); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); AMSPropertyProvider propertyProvider = new AMSHostPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID ); Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); Map<String, TemporalInfo> temporalInfoMap = Collections.emptyMap(); Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID1), temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(res); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "cpu_user"); uriBuilder.addParameter("hostname", "h1"); uriBuilder.addParameter("appId", "HOST"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Double val = (Double) res.getPropertyValue(PROPERTY_ID1); Assert.assertEquals(41.088, val, 0.001); }
@Test public void testPopulateResourcesForMultipleHostMetricscPointInTime() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(MULTIPLE_HOST_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Host); AMSPropertyProvider propertyProvider = new AMSHostPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID ); Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); Map<String, TemporalInfo> temporalInfoMap = Collections.emptyMap(); Request request = PropertyHelper.getReadRequest( new HashSet<String>() {{ add(PROPERTY_ID1); add(PROPERTY_ID2); }}, temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "cpu_user,mem_free"); uriBuilder.addParameter("hostname", "h1"); uriBuilder.addParameter("appId", "HOST"); URIBuilder uriBuilder2 = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder2.addParameter("metricNames", "mem_free,cpu_user"); uriBuilder2.addParameter("hostname", "h1"); uriBuilder2.addParameter("appId", "HOST"); Assert.assertTrue(uriBuilder.toString().equals(streamProvider.getLastSpec()) || uriBuilder2.toString().equals(streamProvider.getLastSpec())); Double val1 = (Double) res.getPropertyValue(PROPERTY_ID1); Assert.assertNotNull("No value for property " + PROPERTY_ID1, val1); Assert.assertEquals(41.088, val1, 0.001); Double val2 = (Double) res.getPropertyValue(PROPERTY_ID2); Assert.assertNotNull("No value for property " + PROPERTY_ID2, val2); Assert.assertEquals(2.47025664E8, val2, 0.1); }
@Test public void testPopulateResourcesForMultipleHostMetrics() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(MULTIPLE_HOST_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Host); AMSPropertyProvider propertyProvider = new AMSHostPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID ); Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID1, new TemporalInfoImpl(1416445244701L, 1416448936564L, 15L)); temporalInfoMap.put(PROPERTY_ID2, new TemporalInfoImpl(1416445244701L, 1416448936564L, 15L)); Request request = PropertyHelper.getReadRequest( new HashSet<String>() {{ add(PROPERTY_ID1); add(PROPERTY_ID2); add("params/padding/NONE"); }}, temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder1 = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder1.addParameter("metricNames", "cpu_user,mem_free"); uriBuilder1.addParameter("hostname", "h1"); uriBuilder1.addParameter("appId", "HOST"); uriBuilder1.addParameter("startTime", "1416445244701"); uriBuilder1.addParameter("endTime", "1416448936564"); URIBuilder uriBuilder2 = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder2.addParameter("metricNames", "mem_free,cpu_user"); uriBuilder2.addParameter("hostname", "h1"); uriBuilder2.addParameter("appId", "HOST"); uriBuilder2.addParameter("startTime", "1416445244701"); uriBuilder2.addParameter("endTime", "1416448936564"); List<String> allSpecs = new ArrayList<String>(streamProvider.getAllSpecs()); Assert.assertEquals(1, allSpecs.size()); Assert.assertTrue(uriBuilder1.toString().equals(allSpecs.get(0)) || uriBuilder2.toString().equals(allSpecs.get(0))); Number[][] val = (Number[][]) res.getPropertyValue(PROPERTY_ID1); Assert.assertEquals(111, val.length); val = (Number[][]) res.getPropertyValue(PROPERTY_ID2); Assert.assertEquals(86, val.length); }
@Test public void testPopulateResourcesForRegexpMetrics() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(MULTIPLE_COMPONENT_REGEXP_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = new HashMap<String, Map<String, PropertyInfo>>() {{ put("RESOURCEMANAGER", new HashMap<String, PropertyInfo>() {{ put("metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB", new PropertyInfo("yarn.QueueMetrics.Queue=(.+).AvailableMB", true, false)); }}); }}; AMSPropertyProvider propertyProvider = new AMSComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); String propertyId1 = "metrics/yarn/Queue/root/AvailableMB"; Resource resource = new ResourceImpl(Resource.Type.Component); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "RESOURCEMANAGER"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(propertyId1, new TemporalInfoImpl(1416528759233L, 1416531129231L, 1L)); Request request = PropertyHelper.getReadRequest( Collections.singleton(propertyId1), temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "yarn.QueueMetrics.Queue=root.AvailableMB"); uriBuilder.addParameter("appId", "RESOURCEMANAGER"); uriBuilder.addParameter("startTime", "1416528759233"); uriBuilder.addParameter("endTime", "1416531129231"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Number[][] val = (Number[][]) res.getPropertyValue("metrics/yarn/Queue/root/AvailableMB"); Assert.assertNotNull("No value for property metrics/yarn/Queue/root/AvailableMB", val); Assert.assertEquals(238, val.length); }
@Test public void testPopulateResourcesForSingleComponentMetric() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(SINGLE_COMPONENT_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Component); AMSPropertyProvider propertyProvider = new AMSComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); String propertyId = PropertyHelper.getPropertyId("metrics/rpc", "RpcQueueTime_avg_time"); Resource resource = new ResourceImpl(Resource.Type.Component); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "NAMENODE"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416528759233L, 1416531129231L, 1L)); Request request = PropertyHelper.getReadRequest( Collections.singleton(propertyId), temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "rpc.rpc.RpcQueueTimeAvgTime"); uriBuilder.addParameter("appId", "NAMENODE"); uriBuilder.addParameter("startTime", "1416528759233"); uriBuilder.addParameter("endTime", "1416531129231"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Number[][] val = (Number[][]) res.getPropertyValue(propertyId); Assert.assertNotNull("No value for property " + propertyId, val); Assert.assertEquals(238, val.length); }
@Test public void testPopulateResourcesForMultipleComponentsMetric() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(SINGLE_COMPONENT_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Component); AMSPropertyProvider propertyProvider = new AMSComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); Set<String> requestedPropertyIds = new HashSet<>(Arrays.asList("metrics/hbase/master", "metrics/cpu/cpu_wio")); Resource resource = new ResourceImpl(Resource.Type.Component); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "METRICS_COLLECTOR"); Resource namenodeResource = new ResourceImpl(Resource.Type.Component); namenodeResource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); namenodeResource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); namenodeResource.setProperty(COMPONENT_NAME_PROPERTY_ID, "NAMENODE"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); for (String propertyId : requestedPropertyIds) { temporalInfoMap.put(propertyId, new TemporalInfoImpl(1416528759233L, 1416531129231L, 1L)); } Request request = PropertyHelper.getReadRequest( requestedPropertyIds, temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(new HashSet<>(Arrays.asList(resource, namenodeResource)), request, null); Assert.assertEquals(2, resources.size()); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "cpu_wio"); uriBuilder.addParameter("appId", "NAMENODE"); uriBuilder.addParameter("startTime", "1416528759233"); uriBuilder.addParameter("endTime", "1416531129231"); Assert.assertTrue(streamProvider.getAllSpecs().contains(uriBuilder.toString())); List<String> allSpecs = new ArrayList<>(streamProvider.getAllSpecs()); Assert.assertEquals(2, allSpecs.size()); }
@Test public void testPopulateMetricsForEmbeddedHBase() throws Exception { AmbariManagementController ams = createNiceMock(AmbariManagementController.class); PowerMock.mockStatic(AmbariServer.class); expect(AmbariServer.getController()).andReturn(ams).anyTimes(); AmbariMetaInfo ambariMetaInfo = createNiceMock(AmbariMetaInfo.class); Clusters clusters = createNiceMock(Clusters.class); Cluster cluster = createNiceMock(Cluster.class); ComponentInfo componentInfo = createNiceMock(ComponentInfo.class); expect(ams.getClusters()).andReturn(clusters).anyTimes(); expect(clusters.getCluster("HostRoles/cluster_name")).andReturn(cluster).anyTimes(); expect(cluster.getResourceId()).andReturn(2L).anyTimes(); StackId stackId = new StackId("HDP", "2.2"); try { expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster).anyTimes(); } catch (AmbariException e) { e.printStackTrace(); } expect(cluster.getCurrentStackVersion()).andReturn(stackId).anyTimes(); expect(ams.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); expect(ambariMetaInfo.getComponentToService("HDP", "2.2", "METRICS_COLLECTOR")).andReturn("AMS").anyTimes(); expect(ambariMetaInfo.getComponent("HDP", "2.2", "AMS", "METRICS_COLLECTOR")) .andReturn(componentInfo).anyTimes(); expect(componentInfo.getTimelineAppid()).andReturn("AMS-HBASE"); replay(ams, clusters, cluster, ambariMetaInfo, componentInfo); PowerMock.replayAll(); TestStreamProvider streamProvider = new TestStreamProvider(EMBEDDED_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Component); AMSPropertyProvider propertyProvider = new AMSComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); String propertyId = PropertyHelper.getPropertyId("metrics/hbase/regionserver", "requests"); Resource resource = new ResourceImpl(Resource.Type.Component); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "METRICS_COLLECTOR"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(propertyId, new TemporalInfoImpl(1421694000L, 1421697600L, 1L)); Request request = PropertyHelper.getReadRequest( Collections.singleton(propertyId), temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "regionserver.Server.totalRequestCount"); uriBuilder.addParameter("appId", "AMS-HBASE"); uriBuilder.addParameter("startTime", "1421694000"); uriBuilder.addParameter("endTime", "1421697600"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Number[][] val = (Number[][]) res.getPropertyValue(propertyId); Assert.assertEquals(189, val.length); }
@Test public void testAggregateFunctionForComponentMetrics() throws Exception { AmbariManagementController ams = createNiceMock(AmbariManagementController.class); PowerMock.mockStatic(AmbariServer.class); expect(AmbariServer.getController()).andReturn(ams).anyTimes(); AmbariMetaInfo ambariMetaInfo = createNiceMock(AmbariMetaInfo.class); Clusters clusters = createNiceMock(Clusters.class); Cluster cluster = createNiceMock(Cluster.class); ComponentInfo componentInfo = createNiceMock(ComponentInfo.class); StackId stackId = new StackId("HDP", "2.2"); expect(ams.getClusters()).andReturn(clusters).anyTimes(); expect(clusters.getCluster("HostRoles/cluster_name")).andReturn(cluster).anyTimes(); expect(cluster.getResourceId()).andReturn(2L).anyTimes(); try { expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster).anyTimes(); } catch (AmbariException e) { e.printStackTrace(); } expect(cluster.getCurrentStackVersion()).andReturn(stackId).anyTimes(); expect(ams.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); expect(ambariMetaInfo.getComponentToService("HDP", "2.2", "HBASE_REGIONSERVER")).andReturn("HBASE").anyTimes(); expect(ambariMetaInfo.getComponent("HDP", "2.2", "HBASE", "HBASE_REGIONSERVER")) .andReturn(componentInfo).anyTimes(); expect(componentInfo.getTimelineAppid()).andReturn("HBASE"); replay(ams, clusters, cluster, ambariMetaInfo, componentInfo); PowerMock.replayAll(); TestStreamProvider streamProvider = new TestStreamProvider(AGGREGATE_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Component); PropertyHelper.updateMetricsWithAggregateFunctionSupport(propertyIds); AMSComponentPropertyProvider propertyProvider = new AMSComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); String propertyId = PropertyHelper.getPropertyId("metrics/rpc", "NumOpenConnections._sum"); Resource resource = new ResourceImpl(Resource.Type.Component); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "HBASE_REGIONSERVER"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(propertyId, new TemporalInfoImpl(1429824611300L, 1429825241400L, 1L)); Request request = PropertyHelper.getReadRequest( Collections.singleton(propertyId), temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "rpc.rpc.NumOpenConnections._sum"); uriBuilder.addParameter("appId", "HBASE"); uriBuilder.addParameter("startTime", "1429824611300"); uriBuilder.addParameter("endTime", "1429825241400"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Number[][] val = (Number[][]) res.getPropertyValue(propertyId); Assert.assertEquals(32, val.length); }
@Test public void testFilterOutOfBandMetricData() throws Exception { setUpCommonMocks(); TestStreamProvider streamProvider = new TestStreamProvider(SINGLE_HOST_METRICS_FILE_PATH); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.Host); AMSPropertyProvider propertyProvider = new AMSHostPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID ); Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID1, new TemporalInfoImpl(1416446744801L, 1416447224801L, 1L)); Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID1), temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); URIBuilder uriBuilder = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder.addParameter("metricNames", "cpu_user"); uriBuilder.addParameter("hostname", "h1"); uriBuilder.addParameter("appId", "HOST"); uriBuilder.addParameter("startTime", "1416446744801"); uriBuilder.addParameter("endTime", "1416447224801"); Assert.assertEquals(uriBuilder.toString(), streamProvider.getLastSpec()); Number[][] val = (Number[][]) res.getPropertyValue(PROPERTY_ID1); Assert.assertNotNull("No value for property " + PROPERTY_ID1, val); Assert.assertEquals(25, val.length); }
@Test public void testPopulateResourcesForHostComponentHostMetrics() throws Exception { setUpCommonMocks(); TestStreamProviderForHostComponentHostMetricsTest streamProvider = new TestStreamProviderForHostComponentHostMetricsTest(null); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider(); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.HostComponent); AMSPropertyProvider propertyProvider = new AMSHostComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "DATANODE"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID1, new TemporalInfoImpl(1416445244801L, 1416448936464L, 1L)); temporalInfoMap.put(PROPERTY_ID3, new TemporalInfoImpl(1416445244801L, 1416448936464L, 1L)); Request request = PropertyHelper.getReadRequest( new HashSet<String>() {{ add(PROPERTY_ID1); add(PROPERTY_ID3); add("params/padding/NONE"); }}, temporalInfoMap); Set<Resource> resources = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources.size()); Resource res = resources.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources.iterator().next()); Assert.assertNotNull(properties); Set<String> specs = streamProvider.getAllSpecs(); Assert.assertEquals(2, specs.size()); String hostMetricSpec = null; String hostComponentMetricsSpec = null; for (String spec : specs) { if (spec.contains("HOST")) { hostMetricSpec = spec; } else { hostComponentMetricsSpec = spec; } } Assert.assertNotNull(hostMetricSpec); Assert.assertNotNull(hostComponentMetricsSpec); URIBuilder uriBuilder1 = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder1.addParameter("metricNames", "dfs.datanode.BlocksReplicated"); uriBuilder1.addParameter("hostname", "h1"); uriBuilder1.addParameter("appId", "DATANODE"); uriBuilder1.addParameter("startTime", "1416445244801"); uriBuilder1.addParameter("endTime", "1416448936464"); Assert.assertEquals(uriBuilder1.toString(), hostComponentMetricsSpec); URIBuilder uriBuilder2 = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); uriBuilder2.addParameter("metricNames", "cpu_user"); uriBuilder2.addParameter("hostname", "h1"); uriBuilder2.addParameter("appId", "HOST"); uriBuilder2.addParameter("startTime", "1416445244801"); uriBuilder2.addParameter("endTime", "1416448936464"); Assert.assertEquals(uriBuilder2.toString(), hostMetricSpec); Number[][] val = (Number[][]) res.getPropertyValue(PROPERTY_ID1); Assert.assertEquals(111, val.length); val = (Number[][]) res.getPropertyValue(PROPERTY_ID3); Assert.assertNotNull("No value for property " + PROPERTY_ID3, val); Assert.assertEquals(8, val.length); }
@Test public void testPopulateResourcesForHostComponentMetricsForMultipleHosts() throws Exception { setUpCommonMocks(); TestStreamProviderForHostComponentMultipleHostsMetricsTest streamProvider = new TestStreamProviderForHostComponentMultipleHostsMetricsTest(null); injectCacheEntryFactoryWithStreamProvider(streamProvider); TestMetricHostProvider metricHostProvider = new TestMetricHostProvider("h1"); ComponentSSLConfiguration sslConfiguration = mock(ComponentSSLConfiguration.class); TimelineMetricCacheProvider cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); TimelineMetricCache cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); Map<String, Map<String, PropertyInfo>> propertyIds = PropertyHelper.getMetricPropertyIds(Resource.Type.HostComponent); Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h1"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "DATANODE"); Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID4, new TemporalInfoImpl(1416445244801L, 1416448936464L, 1L)); Request request = PropertyHelper.getReadRequest( new HashSet<String>() {{ add(PROPERTY_ID4); add("params/padding/NONE"); }}, temporalInfoMap); AMSPropertyProvider propertyProvider = new AMSHostComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); Set<Resource> resources1 = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources1.size()); Resource res1 = resources1.iterator().next(); Map<String, Object> properties = PropertyHelper.getProperties(resources1.iterator().next()); Assert.assertNotNull(properties); metricHostProvider = new TestMetricHostProvider("h2"); cacheProviderMock = EasyMock.createMock(TimelineMetricCacheProvider.class); cacheMock = EasyMock.createMock(TimelineMetricCache.class); expect(cacheProviderMock.getTimelineMetricsCache()).andReturn(cacheMock).anyTimes(); resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(CLUSTER_NAME_PROPERTY_ID, "c1"); resource.setProperty(HOST_NAME_PROPERTY_ID, "h2"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "DATANODE"); temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID4, new TemporalInfoImpl(1416445244801L, 1416448936464L, 1L)); request = PropertyHelper.getReadRequest( new HashSet<String>() {{ add(PROPERTY_ID4); add("params/padding/NONE"); }}, temporalInfoMap); propertyProvider = new AMSHostComponentPropertyProvider( propertyIds, streamProvider, sslConfiguration, cacheProviderMock, metricHostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID ); resource.setProperty(HOST_NAME_PROPERTY_ID, "h2"); Set<Resource> resources2 = propertyProvider.populateResources(Collections.singleton(resource), request, null); Assert.assertEquals(1, resources2.size()); Resource res2 = resources2.iterator().next(); properties = PropertyHelper.getProperties(resources2.iterator().next()); Assert.assertNotNull(properties); Set<String> specs = streamProvider.getAllSpecs(); Assert.assertEquals(2, specs.size()); URIBuilder uriBuilder1 = AMSPropertyProvider.getAMSUriBuilder("localhost", 6188, false); Number[][] val; for (String spec : specs) { Assert.assertNotNull(spec); if (spec.contains("h2")) { uriBuilder1.setParameter("metricNames", "dfs.datanode.blocks_removed"); uriBuilder1.setParameter("hostname", "h2"); uriBuilder1.setParameter("appId", "DATANODE"); uriBuilder1.setParameter("startTime", "1416445244801"); uriBuilder1.setParameter("endTime", "1416448936464"); Assert.assertEquals(uriBuilder1.toString(), spec); val = (Number[][]) res2.getPropertyValue(PROPERTY_ID4); Assert.assertNotNull("No value for property " + PROPERTY_ID4, val); Assert.assertEquals(9, val.length); } else { uriBuilder1.setParameter("metricNames", "dfs.datanode.blocks_removed"); uriBuilder1.setParameter("hostname", "h1"); uriBuilder1.setParameter("appId", "DATANODE"); uriBuilder1.setParameter("startTime", "1416445244801"); uriBuilder1.setParameter("endTime", "1416448936464"); Assert.assertEquals(uriBuilder1.toString(), spec); val = (Number[][]) res1.getPropertyValue(PROPERTY_ID4); Assert.assertNotNull("No value for property " + PROPERTY_ID4, val); Assert.assertEquals(8, val.length); } } }