MetricsReporter extends NodeRepositoryMaintainer { @Override public boolean maintain() { NodeList nodes = nodeRepository().list(); ServiceModel serviceModel = serviceMonitor.getServiceModelSnapshot(); updateLockMetrics(); nodes.forEach(node -> updateNodeMetrics(node, serviceModel)); updateStateMetrics(nodes); updateMaintenanceMetrics(); updateDockerMetrics(nodes); updateTenantUsageMetrics(nodes); updateCacheMetrics(); return true; } MetricsReporter(NodeRepository nodeRepository,
                    Metric metric,
                    Orchestrator orchestrator,
                    ServiceMonitor serviceMonitor,
                    Supplier<Integer> pendingRedeploymentsSupplier,
                    Duration interval,
                    Clock clock); @Override boolean maintain();  }
@Test public void test_registered_metric() { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); Curator curator = new MockCurator(); NodeRepository nodeRepository = new NodeRepository(nodeFlavors, new EmptyProvisionServiceProvider().getHostResourcesCalculator(), curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), new InMemoryFlagSource(), true, false, 0, 1000); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(List.of(node), Agent.system); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); nodeRepository.addNodes(List.of(hostNode), Agent.system); Map<String, Number> expectedMetrics = new TreeMap<>(); expectedMetrics.put("hostedVespa.provisionedHosts", 1); expectedMetrics.put("hostedVespa.parkedHosts", 0); expectedMetrics.put("hostedVespa.readyHosts", 0); expectedMetrics.put("hostedVespa.reservedHosts", 0); expectedMetrics.put("hostedVespa.activeHosts", 0); expectedMetrics.put("hostedVespa.inactiveHosts", 0); expectedMetrics.put("hostedVespa.dirtyHosts", 0); expectedMetrics.put("hostedVespa.failedHosts", 0); expectedMetrics.put("hostedVespa.deprovisionedHosts", 0); expectedMetrics.put("hostedVespa.pendingRedeployments", 42); expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityCpu", 0.0); expectedMetrics.put("hostedVespa.docker.freeCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.freeCapacityMem", 0.0); expectedMetrics.put("hostedVespa.docker.freeCapacityCpu", 0.0); expectedMetrics.put("wantedRebootGeneration", 0L); expectedMetrics.put("currentRebootGeneration", 0L); expectedMetrics.put("wantToReboot", 0); expectedMetrics.put("wantToRetire", 0); expectedMetrics.put("wantToDeprovision", 0); expectedMetrics.put("failReport", 0); expectedMetrics.put("allowedToBeDown", 1); expectedMetrics.put("suspended", 1); expectedMetrics.put("suspendedSeconds", 123L); expectedMetrics.put("numberOfServices", 0L); expectedMetrics.put("cache.nodeObject.hitRate", 0.6D); expectedMetrics.put("cache.nodeObject.evictionCount", 0L); expectedMetrics.put("cache.nodeObject.size", 2L); nodeRepository.list(); expectedMetrics.put("cache.curator.hitRate", 0.5D); expectedMetrics.put("cache.curator.evictionCount", 0L); expectedMetrics.put("cache.curator.size", 11L); ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); when(orchestrator.getHostInfo(eq(reference), any())).thenReturn( HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1))); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter( nodeRepository, metric, orchestrator, serviceMonitor, () -> 42, Duration.ofMinutes(1), clock); metricsReporter.maintain(); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.acquire", 3); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.acquireFailed", 0); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.acquireTimedOut", 0); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.locked", 3); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.release", 3); verifyAndRemoveIntegerMetricSum(metric, "lockAttempt.releaseFailed", 0); metric.remove("lockAttempt.acquireLatency"); metric.remove("lockAttempt.acquireMaxActiveLatency"); metric.remove("lockAttempt.acquireHz"); metric.remove("lockAttempt.acquireLoad"); metric.remove("lockAttempt.lockedLatency"); metric.remove("lockAttempt.lockedMaxActiveLatency"); metric.remove("lockAttempt.lockedHz"); metric.remove("lockAttempt.lockedLoad"); assertEquals(expectedMetrics, new TreeMap<>(metric.values)); }
@Test public void docker_metrics() { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); Curator curator = new MockCurator(); NodeRepository nodeRepository = new NodeRepository(nodeFlavors, new EmptyProvisionServiceProvider().getHostResourcesCalculator(), curator, Clock.systemUTC(), Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), new InMemoryFlagSource(), true, false, 0, 1000); Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost", Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host, Optional.empty()); nodeRepository.addNodes(List.of(dockerHost), Agent.system); nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName()); nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName()); Node container1 = Node.createDockerNode(Set.of("::2"), "container1", "dockerHost", new NodeResources(1, 3, 2, 1), NodeType.tenant); container1 = container1.with(allocation(Optional.of("app1"), container1).get()); nodeRepository.addDockerNodes(new LockedNodeList(List.of(container1), nodeRepository.lockUnallocated())); Node container2 = Node.createDockerNode(Set.of("::3"), "container2", "dockerHost", new NodeResources(2, 4, 4, 1), NodeType.tenant); container2 = container2.with(allocation(Optional.of("app2"), container2).get()); nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated())); NestedTransaction transaction = new NestedTransaction(); nodeRepository.activate(nodeRepository.getNodes(NodeType.host), transaction); transaction.commit(); Orchestrator orchestrator = mock(Orchestrator.class); when(orchestrator.getHostInfo(eq(reference), any())).thenReturn(HostInfo.createNoRemarks()); TestMetric metric = new TestMetric(); ManualClock clock = new ManualClock(); MetricsReporter metricsReporter = new MetricsReporter( nodeRepository, metric, orchestrator, serviceMonitor, () -> 42, Duration.ofMinutes(1), clock); metricsReporter.maintain(); assertEquals(0, metric.values.get("hostedVespa.readyHosts")); assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); assertEquals(120.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); assertEquals(100.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); assertEquals( 7.0, metric.values.get("hostedVespa.docker.totalCapacityCpu")); assertEquals(114.0, metric.values.get("hostedVespa.docker.freeCapacityDisk")); assertEquals( 93.0, metric.values.get("hostedVespa.docker.freeCapacityMem")); assertEquals( 4.0, metric.values.get("hostedVespa.docker.freeCapacityCpu")); Metric.Context app1context = metric.createContext(Map.of("app", "test.default", "tenantName", "app1", "applicationId", "app1.test.default")); assertEquals(2.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityDisk", app1context), 0.01d); assertEquals(3.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityMem", app1context), 0.01d); assertEquals(1.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityCpu", app1context), 0.01d); Metric.Context app2context = metric.createContext(Map.of("app", "test.default", "tenantName", "app2", "applicationId", "app2.test.default")); assertEquals(4.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityDisk", app2context), 0.01d); assertEquals(4.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityMem", app2context), 0.01d); assertEquals(2.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityCpu", app2context), 0.01d); }