Remove benchmark-integration
DELETE
benchmark-integration/README.md
-# Java Agent Performance Tests
-Integration level performance tests for the Java Agent.
-
-## Perf Script Dependencies
-
-`run-perf-test.sh` requires the following (available on homebrew or a linux package manager):
-
-* bash (>=4.0)
-* wrk
-* nc
-
-## Running a Test
-1. Build the shadow jar or the distribution zip for the server you wish to test against.
-2. Run the performance test script passing in the agent jars you wish to test.
-3. (optional) Save test results csv and ponder the great mysteries of performance optimization.
-
-### Example
-#### Jetty
-```
-./gradlew java-agent:benchmark-integration:jetty-perftest:shadowJar
-# Compare a baseline (no agent) to the 0.18.0 and 0.19.0 releases.
-/usr/local/bin/bash ./run-perf-test.sh jar jetty-perftest/build/libs/jetty-perftest-*-all.jar NoAgent ~/Downloads/dd-java-agent-0.18.0.jar ~/Downloads/dd-java-agent-0.19.0.jar
-cp /tmp/perf_results.csv ~/somewhere_else/
-```
-#### Play
-```
-./gradlew :java-agent:benchmark-integration:play-perftest:dist
-# Compare a baseline (no agent) to the 0.18.0 and 0.19.0 releases.
-/usr/local/bin/bash ./run-perf-test.sh play-zip play-perftest/build/distributions/playBinary NoAgent ~/Downloads/dd-java-agent-0.18.0.jar ~/Downloads/dd-java-agent-0.19.0.jar
-cp /tmp/perf_results.csv ~/somewhere_else/
-```

DELETE
benchmark-integration/build.gradle.kts
-plugins {
- id("com.github.johnrengelman.shadow")
-
- id("otel.java-conventions")
-}
-
-description = "Integration Level Agent benchmarks."
-
-subprojects {
- tasks {
- plugins.withId("java") {
- named("javadoc") {
- enabled = false
- }
- }
- }
-}

DELETE
benchmark-integration/jetty-perftest/build.gradle.kts
-plugins {
- id("com.github.johnrengelman.shadow")
-
- id("otel.java-conventions")
-}
-
-dependencies {
- implementation(project(":javaagent-bootstrap"))
- implementation(project(":benchmark-integration"))
-
- implementation("org.eclipse.jetty:jetty-server:9.4.1.v20170120")
- implementation("org.eclipse.jetty:jetty-servlet:9.4.1.v20170120")
-}
-
-tasks {
- jar {
- manifest {
- attributes("Main-Class" to "io.opentelemetry.perftest.jetty.JettyPerftest")
- }
- }
-}

DELETE
benchmark-integration/jetty-perftest/src/main/java/io/opentelemetry/perftest/Worker.java
-/*
- * Copyright The OpenTelemetry Authors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package io.opentelemetry.perftest;
-
-import io.opentelemetry.api.GlobalOpenTelemetry;
-import io.opentelemetry.api.trace.Span;
-import io.opentelemetry.api.trace.Tracer;
-import io.opentelemetry.context.Scope;
-import java.util.concurrent.TimeUnit;
-
-public final class Worker {
-
- private static final Tracer tracer = GlobalOpenTelemetry.getTracer("test");
-
- /** Simulate work for the give number of milliseconds. */
- public static void doWork(long workTimeMillis) {
- Span span = tracer.spanBuilder("work").startSpan();
- try (Scope ignored = span.makeCurrent()) {
- if (span != null) {
- span.setAttribute("work-time", workTimeMillis);
- span.setAttribute("info", "interesting stuff");
- span.setAttribute("additionalInfo", "interesting stuff");
- }
-
- long doneTimestamp = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(workTimeMillis);
- while (System.nanoTime() < doneTimestamp) {
- // busy-wait to simulate work
- }
- span.end();
- }
- }
-
- private Worker() {}
-}

DELETE
benchmark-integration/jetty-perftest/src/main/java/io/opentelemetry/perftest/jetty/JettyPerftest.java
-/*
- * Copyright The OpenTelemetry Authors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package io.opentelemetry.perftest.jetty;
-
-import io.opentelemetry.api.GlobalOpenTelemetry;
-import io.opentelemetry.api.trace.Span;
-import io.opentelemetry.api.trace.Tracer;
-import io.opentelemetry.context.Scope;
-import io.opentelemetry.perftest.Worker;
-import java.io.IOException;
-import javax.servlet.annotation.WebServlet;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-
-public final class JettyPerftest {
-
- private static final int PORT = 8080;
- private static final String PATH = "/work";
- private static final Server jettyServer = new Server(PORT);
- private static final ServletContextHandler servletContext = new ServletContextHandler();
-
- private static final Tracer tracer = GlobalOpenTelemetry.getTracer("test");
-
- public static void main(String[] args) throws Exception {
- servletContext.addServlet(PerfServlet.class, PATH);
- jettyServer.setHandler(servletContext);
- jettyServer.start();
-
- Runtime.getRuntime()
- .addShutdownHook(
- new Thread() {
- @Override
- public void run() {
- try {
- jettyServer.stop();
- jettyServer.destroy();
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
- });
- }
-
- @WebServlet
- public static class PerfServlet extends HttpServlet {
- @Override
- protected void doGet(HttpServletRequest request, HttpServletResponse response)
- throws IOException {
- if (request.getParameter("error") != null) {
- throw new IllegalStateException("some sync error");
- }
- String workVal = request.getParameter("workTimeMillis");
- long workTimeMillis = 0L;
- if (null != workVal) {
- workTimeMillis = Long.parseLong(workVal);
- }
- scheduleWork(workTimeMillis);
- response.getWriter().print("Did " + workTimeMillis + "ms of work.");
- }
-
- private static void scheduleWork(long workTimeMillis) {
- Span span = tracer.spanBuilder("work").startSpan();
- try (Scope ignored = span.makeCurrent()) {
- span.setAttribute("work-time", workTimeMillis);
- span.setAttribute("info", "interesting stuff");
- span.setAttribute("additionalInfo", "interesting stuff");
- if (workTimeMillis > 0) {
- Worker.doWork(workTimeMillis);
- }
- span.end();
- }
- }
- }
-
- private JettyPerftest() {}
-}

DELETE
benchmark-integration/perf-test-default-settings.rc
-# Default settings for running performance tests.
-# To customize, copy this file to perf-test-settings.rc and change desired settings.
-
-# wrk settings
-test_warmup_seconds=30
-test_time_seconds=90
-test_num_connections=5
-test_num_threads=5
-
-# endpoints to test
-declare -A endpoints
-endpoints['<1MS']='http://localhost:8080/work'
-endpoints['1MS']='http://localhost:8080/work?workTimeMillis=1'
-endpoints['2MS']='http://localhost:8080/work?workTimeMillis=2'
-endpoints['5MS']='http://localhost:8080/work?workTimeMillis=5'
-endpoints['10MS']='http://localhost:8080/work?workTimeMillis=10'
-test_order=( '<1MS' '1MS' '2MS' '5MS' '10MS' )
\ No newline at end of file

DELETE
benchmark-integration/play-perftest/app/controllers/HomeController.scala
-/*
- * Copyright The OpenTelemetry Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package controllers
-
-/**
- * This controller creates an `Action` to handle HTTP requests to the
- * application's work page which does busy wait to simulate some work
- */
-class HomeController @Inject()(cc: ControllerComponents) extends AbstractController(cc) {
- val TRACER: Tracer = OpenTelemetry.getTracerProvider.get("test")
-
- /**
- * Create an Action to perform busy wait
- */
- def doGet(workTimeMillis: Option[Long], error: Option[String]) = Action {
- implicit request: Request[AnyContent] =>
- error match {
- case Some(x) => throw new IllegalStateException("some sync error")
- case None => {
- var workTime = workTimeMillis.getOrElse(0L)
- scheduleWork(workTime)
- Ok("Did " + workTime + "ms of work.")
- }
- }
-
- }
-
- private def scheduleWork(workTimeMillis: Long) {
- val span = tracer().spanBuilder("work").startSpan()
- val scope = tracer().withSpan(span)
- try {
- if (span != null) {
- span.setAttribute("work-time", workTimeMillis)
- span.setAttribute("info", "interesting stuff")
- span.setAttribute("additionalInfo", "interesting stuff")
- }
- if (workTimeMillis > 0) {
- Worker.doWork(workTimeMillis)
- }
- } finally {
- span.end()
- scope.close()
- }
- }
-}

DELETE
benchmark-integration/play-perftest/app/controllers/Worker.scala
-/*
- * Copyright The OpenTelemetry Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package controllers
-
-import java.util.concurrent.TimeUnit
-
-object Worker {
- val TRACER: Tracer =
- OpenTelemetry.getTracerProvider.get("test")
-
- def doWork(workTimeMillis: Long) = {
- val span = tracer().spanBuilder("work").startSpan()
- val scope = tracer().withSpan(span)
- try {
- if (span != null) {
- span.setAttribute("work-time", workTimeMillis)
- span.setAttribute("info", "interesting stuff")
- span.setAttribute("additionalInfo", "interesting stuff")
- }
- val doneTimestamp = System.nanoTime + TimeUnit.MILLISECONDS.toNanos(
- workTimeMillis
- )
- while ( {
- System.nanoTime < doneTimestamp
- }) {
- // busy-wait to simulate work
- }
- } finally {
- span.end()
- scope.close()
- }
- }
-}

DELETE
benchmark-integration/play-perftest/build.gradle
-plugins {
- id "org.gradle.playframework" version "0.9"
- id("com.github.johnrengelman.shadow")
-
- id("otel.java-conventions")
-}
-
-afterEvaluate {
- // assemble fails without this because gradle play plugin looks for the jar file without
- // considering the updated archivesBaseName that we have set in java.gradle
- archivesBaseName = 'play-perftest'
-}
-
-ext {
- playVersion = "2.6.20"
- scalaVersion = System.getProperty("scala.binary.version", /* default = */ "2.12")
-}
-
-play {
- platform {
- playVersion = project.ext.playVersion
- scalaVersion = project.ext.scalaVersion
- javaVersion = JavaVersion.VERSION_1_8
- }
- injectedRoutesGenerator = true
-}
-
-dependencies {
- implementation "com.typesafe.play:play-guice_$scalaVersion:$playVersion"
- implementation "com.typesafe.play:play-logback_$scalaVersion:$playVersion"
- implementation "com.typesafe.play:filters-helpers_$scalaVersion:$playVersion"
-
- implementation project(':javaagent-bootstrap')
- implementation project(':benchmark-integration')
-}
-
-repositories {
- mavenLocal()
- mavenCentral()
- maven {
- name "lightbend-maven-releases"
- url "https://repo.lightbend.com/lightbend/maven-release"
- }
- ivy {
- name "lightbend-ivy-release"
- url "https://repo.lightbend.com/lightbend/ivy-releases"
- layout "ivy"
- }
-}

DELETE
benchmark-integration/play-perftest/conf/application.conf
-# https://www.playframework.com/documentation/latest/Configuration
-play.http.secret.key=agentbenchmarktest0xDEADBEEF
-http.port=8080

DELETE
benchmark-integration/play-perftest/conf/logback.xml
-<!-- https://www.playframework.com/documentation/latest/SettingsLogger -->
-<configuration>
-
- <conversionRule conversionWord="coloredLevel" converterClass="play.api.libs.logback.ColoredLevel"/>
-
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%coloredLevel %logger{15} - %message%n%xException{10}</pattern>
- </encoder>
- </appender>
-
- <appender name="ASYNCSTDOUT" class="ch.qos.logback.classic.AsyncAppender">
- <appender-ref ref="STDOUT"/>
- </appender>
-
- <logger name="play" level="INFO"/>
- <logger name="application" level="DEBUG"/>
-
- <!-- Off these ones as they are annoying, and anyway we manage configuration ourselves -->
- <logger name="com.avaje.ebean.config.PropertyMapLoader" level="OFF"/>
- <logger name="com.avaje.ebeaninternal.server.core.XmlConfigLoader" level="OFF"/>
- <logger name="com.avaje.ebeaninternal.server.lib.BackgroundThread" level="OFF"/>
- <logger name="com.gargoylesoftware.htmlunit.javascript" level="OFF"/>
-
- <root level="WARN">
- <appender-ref ref="ASYNCSTDOUT"/>
- </root>
-
-</configuration>

DELETE
benchmark-integration/play-perftest/conf/routes
-# Routes
-# This file defines all application routes (Higher priority routes first)
-# https://www.playframework.com/documentation/latest/ScalaRouting
-# ~~~~
-
-# An example controller showing a sample home page
-GET /work controllers.HomeController.doGet(workTimeMillis: Option[Long], error: Option[String])

DELETE
benchmark-integration/run-perf-test.sh
-#!/usr/bin/env bash
-
-# A script for measuring a server's throughput with or without a java agent.
-test_csv_file=/tmp/perf_results.csv
-server_output=/tmp/server_output.txt
-server_type=$1
-server_package=$2
-agent_jars="${@:3}"
-server_pid=""
-if [[ "$server_package" = "" ]] || [[ "$server_type" != "play-zip" && "$server_type" != "jar" ]]; then
- echo "usage: ./run-perf-test.sh [play-zip|jar] path-to-server-package path-to-agent1 path-to-agent2..."
- echo ""
- echo "[play-zip|jar] : Specify whether the server will be in zip format (play server) or jar format"
- echo "path-to-server-package : Must be a jar or binary zip package which creates an http server on local port 8080 when started."
- echo " Note: if the server package is a zip, then the script will attempt to unzip to a temp directory and run the server from there."
- echo "path-to-agent* : Each must be a javaagent jar, or NoAgent."
- echo ""
- echo "Example: This will run the perf tests against myserver.jar. It will run against no agent as a baseline, then against myagent-1.0.jar."
- echo " ./run-perf-test.sh /tmp/myserve.jar NoAgent /tmp/myagent-1.0.jar"
- echo ""
- echo "Test results are saved to $test_csv_file"
- exit 1
-fi
-
-if [ -f perf-test-settings.rc ]; then
- echo "loading custom settings"
- cat ./perf-test-settings.rc
- . ./perf-test-settings.rc
-else
- echo "loading default settings"
- cat ./perf-test-default-settings.rc
- . ./perf-test-default-settings.rc
-fi
-echo ""
-echo ""
-
-unzipped_server_path=""
-
-# Start up server passed into the script
-# Blocks until server is bound to local port 8080
-function start_server {
- agent_jar="$1"
- javaagent_arg=""
- if [ "$agent_jar" != "" -a -f "$agent_jar" ]; then
- javaagent_arg="-javaagent:$agent_jar -Dio.opentelemetry.javaagent.slf4j.simpleLogger.defaultLogLevel=off"
- fi
-
- if [ "$server_type" = "jar" ]; then
- echo "starting server: java $javaagent_arg -jar $server_package"
- { /usr/bin/time -l java $javaagent_arg -Xms256m -Xmx256m -jar $server_package ; } 2> $server_output &
- else
- # make a temp directory to hold the unzipped server
- unzip_temp=`mktemp -d`
- # perform the unzipping of the play zip
- unzip ${server_package} -d ${unzip_temp} &> /dev/null
-
- if [ $? -eq 0 ]; then
- echo "Unzipped server package at ${unzip_temp}"
- else
- echo "Failed to unzip server package to ${unzip_temp}"
- exit 2
- fi
-
- # get the unzipped directory name
- unzipped_dirname=`basename ${unzip_temp}/*`
- # unzipped server location, will be removed when the server is stopped
- unzipped_server_path=${unzip_temp}
-
- java_opts_env='JAVA_OPTS="'${javaagent_arg}'"'
- # it appears the binary script will always be named playBinary at the time of writing
- # no matter what the zip file is named.
- play_script=${unzipped_server_path}/${unzipped_dirname}/bin/playBinary
-
- # have to use env to set JAVA_OPTS because of a gradle play plugin bug:
- # https://github.com/gradle/gradle/issues/4471
- if [ "$agent_jar" != "" -a -f "$agent_jar" ]; then
- utility_cmd="env JAVA_OPTS=${javaagent_arg} ${play_script}"
- else
- utility_cmd="${play_script}"
- fi
-
- echo "starting server: ${utility_cmd}"
- { /usr/bin/time -l ${utility_cmd} ; } 2> $server_output &
- fi
-
- # Block until server is up
- until nc -z localhost 8080; do
- sleep 0.5
- done
- server_pid=$(lsof -i tcp:8080 | awk '$8 == "TCP" { print $2 }' | uniq)
- echo "server $server_pid started"
-}
-
-# Send a kill signal to the running server
-# and block until the server is stopped
-function stop_server {
- echo "Stopping $server_pid"
- kill $server_pid
- wait
- server_pid=""
- # clean up the unzipped server
- if [[ ${unzipped_server_path} != "" ]] && [[ ${server_type} = "play-zip" ]] && [[ -d ${unzipped_server_path} ]]; then
- echo "Cleaning up unzipped server at "${unzipped_server_path}
- rm -rf ${unzipped_server_path}
- fi
-}
-
-# Warmup and run wrk tests on a single endpoint.
-# echos out a file containing raw wrk output
-# and a final line of the average requests/second
-function test_endpoint {
- url=$1
- # warmup
- wrk -c $test_num_connections -t$test_num_threads -d ${test_warmup_seconds}s $url >/dev/null
-
- # run test
- wrk_results=/tmp/wrk_results.`date +%s`
- wrk -c $test_num_connections -t$test_num_threads -d ${test_time_seconds}s $url > $wrk_results
- echo $wrk_results
-}
-
-
-trap 'stop_server; exit' SIGINT
-trap 'kill $server_pid; exit' SIGTERM
-header='Client Version'
-for label in "${test_order[@]}"; do
- header="$header,$label Latency,$label Throughput"
-done
-header="$header,Server CPU Burn,Server Max RSS,Server Start RSS,Server Load Increase RSS"
-echo $header > $test_csv_file
-
-for agent_jar in $agent_jars; do
- echo "----Testing agent $agent_jar----"
- if [ "$agent_jar" == "NoAgent" ]; then
- result_row="NoAgent"
- start_server ""
- else
- agent_version=$(java -jar $agent_jar 2>/dev/null)
- result_row="$agent_version"
- start_server $agent_jar
- fi
-
-
- server_start_cpu=$(ps -o 'pid,time' | awk "\$1 == $server_pid { print \$2 }" | awk -F'[:\.]' '{ print ($1 * 3600) + ($2 * 60) + $3 }')
- server_start_rss=$(ps -o 'pid,rss' | awk "\$1 == $server_pid { print \$2 }")
-
- server_total_rss=0
- server_total_rss_count=0
-
- for t in "${test_order[@]}"; do
- label="$t"
- url="${endpoints[$label]}"
- echo "--Testing $label -- $url--"
- test_output_file=$(test_endpoint $url)
- let server_total_rss=$server_total_rss+$(ps -o 'pid,rss' | awk "\$1 == $server_pid { print \$2 }")
- let server_total_rss_count=$server_total_rss_count+1
- cat $test_output_file
- avg_latency=$(awk '$1 == "Latency" { print $2 }' $test_output_file)
- avg_throughput=$(awk '$1 == "Requests/sec:" { print $2 }' $test_output_file)
- result_row="$result_row,$avg_latency,$avg_throughput"
- rm $test_output_file
- done
-
- server_stop_cpu=$(ps -o 'pid,time' | awk "\$1 == $server_pid { print \$2 }" | awk -F'[:\.]' '{ print ($1 * 3600) + ($2 * 60) + $3 }')
-
- let server_cpu=$server_stop_cpu-$server_start_cpu
-
- server_load_increase_rss=$(echo "scale=2; ( $server_total_rss / $server_total_rss_count ) - $server_start_rss" | bc)
-
- stop_server
-
- server_max_rss=$(awk '/.* maximum resident set size/ { print $1 }' $server_output)
- rm $server_output
-
- echo "$result_row,$server_cpu,$server_max_rss,$server_start_rss,$server_load_increase_rss" >> $test_csv_file
- echo "----/Testing agent $agent_jar----"
- echo ""
-done
-
-echo ""
-echo "DONE. Test results saved to $test_csv_file"

MODIFY
settings.gradle
settings.gradle
@@ -316,6 +316,4 @@ include ':instrumentation:wicket-8.0:javaagent'
// benchmark
include ':benchmark'
-include ':benchmark-integration'
-include ':benchmark-integration:jetty-perftest'
include ':benchmark-e2e'

Remove benchmark-integration (#3460)