Update JMH benchmarks
Benchmark is run with `./gradlew :dd-java-agent:benchmark:jmh` -- after updating the absolute paths to the various agents in `ClassRetransformingBenchmark`.  Unfortunately I was unable to have it launch correctly with a relative path.
MODIFY
dd-java-agent/benchmark/benchmark.gradle
dd-java-agent/benchmark/benchmark.gradle
@@ -6,68 +6,44 @@ apply from: "${rootDir}/gradle/java.gradle"
dependencies {
jmh project(':dd-trace-api')
- jmh group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.7.6'
-
- // Add a bunch of dependencies so instrumentation is not disabled.
- jmh group: 'javax.jms', name: 'javax.jms-api', version: '2.0.1'
- jmh group: 'javax.servlet', name: 'javax.servlet-api', version: '3.0.1'
- jmh group: 'org.mongodb', name: 'mongo-java-driver', version: '3.4.2'
- jmh group: 'org.mongodb', name: 'mongodb-driver-async', version: '3.4.2'
- jmh(group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.119') {
- exclude(module: 'httpclient')
- exclude(module: 'jackson-databind')
- exclude(module: 'jackson-dataformat-cbor')
- }
- jmh group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.6.0'
- jmh group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
- jmh(group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version: '3.2.0')
+ jmh deps.bytebuddyagent
}
-configurations.testRuntimeClasspath.dependencies.clear()
-
-
jmh {
timeUnit = 'us' // Output time unit. Available time units are: [m, s, ms, us, ns].
- benchmarkMode = ['thrpt', 'avgt']
-// timeOnIteration = '5s'
- iterations = 5 // Number of measurement iterations to do.
+ benchmarkMode = ['avgt']
+ timeOnIteration = '20s'
+ iterations = 1 // Number of measurement iterations to do.
fork = 1 // How many times to forks a single benchmark. Use 0 to disable forking altogether
-// jvmArgs = ["-Dasdf=123"]
-// jvmArgs = ["-javaagent:${project(':dd-java-agent').shadowJar.archivePath}"]
+ jvmArgs = ["-Ddd.jmxfetch.enabled=false", "-Ddd.writer.type=LoggingWriter"]
+// jvmArgs += ["-XX:+UnlockDiagnosticVMOptions", "-XX:+DebugNonSafepoints", "-XX:StartFlightRecording=delay=5s,dumponexit=true,name=jmh-benchmark,filename=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/jmh-benchmark.jfr"]
+// jvmArgs += ["-agentpath:${rootDir}/dd-java-agent/benchmark/src/jmh/resources/libasyncProfiler.so=start,collapsed,file=${rootDir}/dd-java-agent/benchmark/build/reports/jmh/profiler.txt"]
failOnError = true // Should JMH fail immediately if any benchmark had experienced the unrecoverable error?
-// warmup = '2s' // Time to spend at each warmup iteration.
-// warmupIterations = 2 // Number of warmup iterations to do.
-// warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
+ warmup = '5s' // Time to spend at each warmup iteration.
+// warmupBatchSize = 10 // Warmup batch size: number of benchmark method calls per operation.
+ warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
+ warmupIterations = 1 // Number of warmup iterations to do.
-// profilers = ['stack']
+// profilers = ['stack:lines=5;detailLine=true;period=5;excludePackages=true']
// Use profilers to collect additional data. Supported profilers: [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr]
// humanOutputFile = project.file("${project.buildDir}/reports/jmh/human.txt") // human-readable output file
// operationsPerInvocation = 10 // Operations per invocation.
// synchronizeIterations = false // Synchronize iterations?
- timeout = '1s' // Timeout for benchmark iteration.
- includeTests = false
+ timeout = '5s' // Timeout for benchmark iteration.
+// includeTests = false
// Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes.
- duplicateClassesStrategy = 'fail'
- jmhVersion = '1.20' // Specifies JMH version
+ duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE
+ jmhVersion = '1.23' // Specifies JMH version
}
-// configured as a separate task since the 'jmh' task did not like adding a javaagent argument.
-tasks.register("jmhAgent", JavaExec) {
- classpath = files(project.jmhCompileGeneratedClasses.destinationDir)
- classpath += sourceSets.jmh.runtimeClasspath
- main = "org.openjdk.jmh.Main"
- args += ["-tu", "us"]
- args += ["-bm", "avgt"]
-// args += ["-prof", "stack:lines=5;detailLine=true;period=5;excludePackages=true"]
- args += ["-f", "1"]
- args += ["-foe", "true"]
-
-// args += ["-wi", "2"]
-// args += ["-i", "5"]
- dependsOn project.tasks.jmhCompileGeneratedClasses
-}
+tasks.jmh.dependsOn project(':dd-java-agent').shadowJar
-tasks.jmhAgent.dependsOn project(':dd-java-agent').shadowJar
+/*
+If using libasyncProfiler, use the following to generate nice svg flamegraphs.
+sed '/unknown/d' dd-java-agent/benchmark/build/reports/jmh/profiler.txt | sed '/^thread_start/d' | sed '/not_walkable/d' > dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt
+(using https://github.com/brendangregg/FlameGraph)
+./flamegraph.pl --color=java dd-java-agent/benchmark/build/reports/jmh/profiler-cleaned.txt > dd-java-agent/benchmark/build/reports/jmh/jmh-master.svg
+ */

MODIFY
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/ClassRetransformingBenchmark.java
@@ -4,49 +4,17 @@ import datadog.benchmark.classes.TracedClass;
import datadog.benchmark.classes.UntracedClass;
import java.lang.instrument.Instrumentation;
import java.lang.instrument.UnmodifiableClassException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.nio.file.Paths;
import net.bytebuddy.agent.ByteBuddyAgent;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
public class ClassRetransformingBenchmark {
- public static final String BENCHMARK_HOME =
- Paths.get(".").toAbsolutePath().normalize().toString();
-
- static {
- if (!BENCHMARK_HOME.endsWith("benchmark")) {
- throw new IllegalArgumentException("Invalid Home directory: " + BENCHMARK_HOME);
- }
- }
@State(Scope.Benchmark)
public static class BenchmarkState {
private final Instrumentation inst = ByteBuddyAgent.install();
-
- @Setup
- public void initializeInstrumentation() {
- // loading TracedClass will initialize helper injection
- TracedClass.class.getName();
- }
-
- @TearDown
- public void stopAgent() {
- try {
- final Class<?> gt = Class.forName("io.opentracing.util.GlobalTracer");
- final Field tracerField = gt.getDeclaredField("tracer");
- tracerField.setAccessible(true);
- final Object tracer = tracerField.get(null);
- final Method close = tracer.getClass().getMethod("close");
- close.invoke(tracer);
- } catch (final Exception e) {
- }
- }
}
@Benchmark
@@ -60,54 +28,11 @@ public class ClassRetransformingBenchmark {
state.inst.retransformClasses(TracedClass.class);
}
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.2.jar")
- public static class WithAgent022 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.4.jar")
- public static class WithAgent024 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.6.jar")
- public static class WithAgent026 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.7.jar")
- public static class WithAgent027 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.8.jar")
- public static class WithAgent028 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.9.jar")
- public static class WithAgent029 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.10.jar")
- public static class WithAgent0210 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.11.jar")
- public static class WithAgent0211 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.2.12.jar")
- public static class WithAgent0212 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.0.jar")
- public static class WithAgent030 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.1.jar")
- public static class WithAgent031 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.2.jar")
- public static class WithAgent032 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.3.3.jar")
- public static class WithAgent033 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.0.jar")
- public static class WithAgent040 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.4.1.jar")
- public static class WithAgent041 extends ClassRetransformingBenchmark {}
-
- @Fork(jvmArgsAppend = "-javaagent:releases/dd-java-agent-0.5.0.jar")
- public static class WithAgent050 extends ClassRetransformingBenchmark {}
+ @Fork(jvmArgsAppend = "-javaagent:/path/to/dd-java-agent-master.jar")
+ public static class WithAgentMaster extends ClassRetransformingBenchmark {}
- @Fork(jvmArgsAppend = "-javaagent:../build/libs/dd-java-agent.jar")
+ @Fork(
+ jvmArgsAppend =
+ "-javaagent:/path/to/dd-trace-java/dd-java-agent/build/libs/dd-java-agent.jar")
public static class WithAgent extends ClassRetransformingBenchmark {}
}

MODIFY
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/C.java
package datadog.benchmark.classes;
-public interface C extends B {
+public interface C extends A, B {
void c();
}

MODIFY
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/D.java
package datadog.benchmark.classes;
-public interface D extends C {
+public interface D extends A, B, C {
void d();
}

MODIFY
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java
dd-java-agent/benchmark/src/jmh/java/datadog/benchmark/classes/E.java
package datadog.benchmark.classes;
-public interface E extends D {
+public interface E extends B, C, D {
void e();
}

DELETE
dd-java-agent/benchmark/src/jmh/resources/dd-trace.yaml
-enableCustomAnnotationTracingOver: ["datadog.benchmark"]

MODIFY
dd-trace-ot/dd-trace-ot.gradle
dd-trace-ot/dd-trace-ot.gradle
@@ -138,7 +138,7 @@ jmh {
// warmupBenchmarks = ['.*Warmup'] // Warmup benchmarks to include in the run in addition to already selected. JMH will not measure these benchmarks, but only use them for the warmup.
// zip64 = true // Use ZIP64 format for bigger archives
- jmhVersion = '1.21' // Specifies JMH version
+ jmhVersion = '1.23' // Specifies JMH version
// includeTests = true // Allows to include test sources into generate JMH jar, i.e. use it when benchmarks depend on the test classes.
duplicateClassesStrategy = 'warn'
// Strategy to apply when encountring duplicate classes during creation of the fat jar (i.e. while executing jmhJar task)

Update JMH benchmarks

Benchmark is run with `./gradlew :dd-java-agent:benchmark:jmh` -- after updating the absolute paths to the various agents in `ClassRetransformingBenchmark`.  Unfortunately I was unable to have it launch correctly with a relative path.