Record memory usage after garbage collection
Per conversation in
MODIFY
instrumentation/runtime-metrics/javaagent/src/test/groovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy
instrumentation/runtime-metrics/javaagent/src/test/groovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy
@@ -28,6 +28,7 @@ class RuntimeMetricsTest extends AgentInstrumentationSpecification {
assert getMetrics().any { it.name == "process.runtime.jvm.memory.usage" }
assert getMetrics().any { it.name == "process.runtime.jvm.memory.committed" }
assert getMetrics().any { it.name == "process.runtime.jvm.memory.limit" }
+ assert getMetrics().any { it.name == "process.runtime.jvm.memory.usage_after_last_gc" }
assert getMetrics().any { it.name == "process.runtime.jvm.threads.count" }
assert getMetrics().any { it.name == "process.runtime.jvm.buffer.limit" }
assert getMetrics().any { it.name == "process.runtime.jvm.buffer.count" }

MODIFY
instrumentation/runtime-metrics/library/README.md
instrumentation/runtime-metrics/library/README.md
@@ -53,28 +53,34 @@ The following lists attributes reported for a variety of garbage collectors. Not
* `process.runtime.jvm.memory.usage`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.committed`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.limit`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}
+ * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}
* G1 Garbage Collector
* `process.runtime.jvm.memory.init`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.usage`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.committed`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}
+ * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=G1 Old Gen,type=heap}
* Parallel Garbage Collector
* `process.runtime.jvm.memory.init`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.usage`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.committed`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}
+ * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}
* Serial Garbage Collector
* `process.runtime.jvm.memory.init`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.usage`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.committed`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
* `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}
+ * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}
* Shenandoah Garbage Collector
* `process.runtime.jvm.memory.init`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
* `process.runtime.jvm.memory.usage`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
* `process.runtime.jvm.memory.committed`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
* `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
+ * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Shenandoah,type=heap}
* Z Garbage Collector
* `process.runtime.jvm.memory.init`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
* `process.runtime.jvm.memory.usage`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
* `process.runtime.jvm.memory.committed`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
* `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
+ * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=ZHeap,type=heap}

MODIFY
instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java
instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java
@@ -34,7 +34,8 @@ import java.util.function.Function;
* process.runtime.jvm.memory.init{type="heap",pool="G1 Eden Space"} 1000000
* process.runtime.jvm.memory.usage{type="heap",pool="G1 Eden Space"} 2500000
* process.runtime.jvm.memory.committed{type="heap",pool="G1 Eden Space"} 3000000
- * process.runtime.jvm.memory.max{type="heap",pool="G1 Eden Space"} 4000000
+ * process.runtime.jvm.memory.limit{type="heap",pool="G1 Eden Space"} 4000000
+ * process.runtime.jvm.memory.usage_after_last_gc{type="heap",pool="G1 Eden Space"} 1500000
* process.runtime.jvm.memory.init{type="non_heap",pool="Metaspace"} 200
* process.runtime.jvm.memory.usage{type="non_heap",pool="Metaspace"} 400
* process.runtime.jvm.memory.committed{type="non_heap",pool="Metaspace"} 500
@@ -61,30 +62,41 @@ public final class MemoryPools {
.upDownCounterBuilder("process.runtime.jvm.memory.usage")
.setDescription("Measure of memory used")
.setUnit("By")
- .buildWithCallback(callback(poolBeans, MemoryUsage::getUsed));
+ .buildWithCallback(callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getUsed));
meter
.upDownCounterBuilder("process.runtime.jvm.memory.init")
.setDescription("Measure of initial memory requested")
.setUnit("By")
- .buildWithCallback(callback(poolBeans, MemoryUsage::getInit));
+ .buildWithCallback(callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getInit));
meter
.upDownCounterBuilder("process.runtime.jvm.memory.committed")
.setDescription("Measure of memory committed")
.setUnit("By")
- .buildWithCallback(callback(poolBeans, MemoryUsage::getCommitted));
+ .buildWithCallback(
+ callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getCommitted));
meter
.upDownCounterBuilder("process.runtime.jvm.memory.limit")
.setDescription("Measure of max obtainable memory")
.setUnit("By")
- .buildWithCallback(callback(poolBeans, MemoryUsage::getMax));
+ .buildWithCallback(callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getMax));
+
+ meter
+ .upDownCounterBuilder("process.runtime.jvm.memory.usage_after_last_gc")
+ .setDescription(
+ "Measure of memory used after the most recent garbage collection event on this pool")
+ .setUnit("By")
+ .buildWithCallback(
+ callback(poolBeans, MemoryPoolMXBean::getCollectionUsage, MemoryUsage::getUsed));
}
// Visible for testing
static Consumer<ObservableLongMeasurement> callback(
- List<MemoryPoolMXBean> poolBeans, Function<MemoryUsage, Long> extractor) {
+ List<MemoryPoolMXBean> poolBeans,
+ Function<MemoryPoolMXBean, MemoryUsage> memoryUsageExtractor,
+ Function<MemoryUsage, Long> valueExtractor) {
List<Attributes> attributeSets = new ArrayList<>(poolBeans.size());
for (MemoryPoolMXBean pool : poolBeans) {
attributeSets.add(
@@ -97,7 +109,13 @@ public final class MemoryPools {
return measurement -> {
for (int i = 0; i < poolBeans.size(); i++) {
Attributes attributes = attributeSets.get(i);
- long value = extractor.apply(poolBeans.get(i).getUsage());
+ MemoryUsage memoryUsage = memoryUsageExtractor.apply(poolBeans.get(i));
+ if (memoryUsage == null) {
+ // JVM may return null in special cases for MemoryPoolMXBean.getUsage() and
+ // MemoryPoolMXBean.getCollectionUsage()
+ continue;
+ }
+ long value = valueExtractor.apply(memoryUsage);
if (value != -1) {
measurement.record(value, attributes);
}

MODIFY
instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java
instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java
@@ -8,6 +8,7 @@ package io.opentelemetry.instrumentation.runtimemetrics;
import static io.opentelemetry.instrumentation.runtimemetrics.ScopeUtil.EXPECTED_SCOPE;
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@@ -22,6 +23,7 @@ import java.lang.management.MemoryPoolMXBean;
import java.lang.management.MemoryType;
import java.lang.management.MemoryUsage;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import org.junit.jupiter.api.BeforeEach;
@@ -31,8 +33,11 @@ import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
@ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.LENIENT)
class MemoryPoolsTest {
@RegisterExtension
@@ -45,6 +50,8 @@ class MemoryPoolsTest {
@Mock private MemoryUsage heapPoolUsage;
@Mock private MemoryUsage nonHeapUsage;
+ @Mock private MemoryUsage heapCollectionUsage;
+ @Mock private MemoryUsage nonHeapCollectionUsage;
private List<MemoryPoolMXBean> beans;
@@ -53,9 +60,11 @@ class MemoryPoolsTest {
when(heapPoolBean.getName()).thenReturn("heap_pool");
when(heapPoolBean.getType()).thenReturn(MemoryType.HEAP);
when(heapPoolBean.getUsage()).thenReturn(heapPoolUsage);
+ when(heapPoolBean.getCollectionUsage()).thenReturn(heapCollectionUsage);
when(nonHeapPoolBean.getName()).thenReturn("non_heap_pool");
when(nonHeapPoolBean.getType()).thenReturn(MemoryType.NON_HEAP);
when(nonHeapPoolBean.getUsage()).thenReturn(nonHeapUsage);
+ when(nonHeapPoolBean.getCollectionUsage()).thenReturn(nonHeapCollectionUsage);
beans = Arrays.asList(heapPoolBean, nonHeapPoolBean);
}
@@ -69,7 +78,8 @@ class MemoryPoolsTest {
when(nonHeapUsage.getUsed()).thenReturn(15L);
when(nonHeapUsage.getCommitted()).thenReturn(16L);
when(nonHeapUsage.getMax()).thenReturn(17L);
-
+ when(heapCollectionUsage.getUsed()).thenReturn(18L);
+ when(nonHeapCollectionUsage.getUsed()).thenReturn(19L);
MemoryPools.registerObservers(testing.getOpenTelemetry(), beans);
testing.waitAndAssertMetrics(
@@ -176,6 +186,33 @@ class MemoryPoolsTest {
AttributeKey.stringKey("pool"), "non_heap_pool")
.hasAttribute(
AttributeKey.stringKey("type"), "non_heap")))));
+ testing.waitAndAssertMetrics(
+ "io.opentelemetry.runtime-metrics",
+ "process.runtime.jvm.memory.usage_after_last_gc",
+ metrics ->
+ metrics.anySatisfy(
+ metricData ->
+ assertThat(metricData)
+ .hasInstrumentationScope(EXPECTED_SCOPE)
+ .hasDescription(
+ "Measure of memory used after the most recent garbage collection event on this pool")
+ .hasUnit("By")
+ .hasLongSumSatisfying(
+ sum ->
+ sum.hasPointsSatisfying(
+ point ->
+ point
+ .hasValue(18)
+ .hasAttribute(
+ AttributeKey.stringKey("pool"), "heap_pool")
+ .hasAttribute(AttributeKey.stringKey("type"), "heap"),
+ point ->
+ point
+ .hasValue(19)
+ .hasAttribute(
+ AttributeKey.stringKey("pool"), "non_heap_pool")
+ .hasAttribute(
+ AttributeKey.stringKey("type"), "non_heap")))));
}
@Test
@@ -184,7 +221,7 @@ class MemoryPoolsTest {
when(nonHeapUsage.getUsed()).thenReturn(2L);
Consumer<ObservableLongMeasurement> callback =
- MemoryPools.callback(beans, MemoryUsage::getUsed);
+ MemoryPools.callback(beans, MemoryPoolMXBean::getUsage, MemoryUsage::getUsed);
callback.accept(measurement);
verify(measurement)
@@ -199,11 +236,25 @@ class MemoryPoolsTest {
when(heapPoolUsage.getMax()).thenReturn(1L);
when(nonHeapUsage.getMax()).thenReturn(-1L);
- Consumer<ObservableLongMeasurement> callback = MemoryPools.callback(beans, MemoryUsage::getMax);
+ Consumer<ObservableLongMeasurement> callback =
+ MemoryPools.callback(beans, MemoryPoolMXBean::getUsage, MemoryUsage::getMax);
callback.accept(measurement);
verify(measurement)
.record(1, Attributes.builder().put("pool", "heap_pool").put("type", "heap").build());
verify(measurement, never()).record(eq(-1), any());
}
+
+ @Test
+ void callback_NullUsage() {
+ when(heapPoolBean.getCollectionUsage()).thenReturn(null);
+
+ Consumer<ObservableLongMeasurement> callback =
+ MemoryPools.callback(
+ Collections.singletonList(heapPoolBean),
+ MemoryPoolMXBean::getCollectionUsage,
+ MemoryUsage::getUsed);
+ callback.accept(measurement);
+ verify(measurement, never()).record(anyLong(), any());
+ }
}

Record memory usage after garbage collection (#6963)

Per conversation in #6362.