Add version scan verification and test in CI
MODIFY
.circleci/config.yml
.circleci/config.yml
@@ -14,7 +14,7 @@ jobs:
- run:
name: Run Tests
- command: ./gradlew clean check --parallel
+ command: ./gradlew clean check verifyVersionScan --parallel
- save_cache:
paths:

MODIFY
buildSrc/src/main/groovy/VersionScanExtension.groovy
buildSrc/src/main/groovy/VersionScanExtension.groovy
@@ -6,4 +6,6 @@ class VersionScanExtension {
boolean scanDependencies = false
String legacyGroup
String legacyModule
+ Map<String, String> verifyPresent = Collections.emptyMap()
+ List<String> verifyMissing = Collections.emptyList()
}

MODIFY
buildSrc/src/main/groovy/VersionScanPlugin.groovy
buildSrc/src/main/groovy/VersionScanPlugin.groovy
@@ -34,19 +34,23 @@ class VersionScanPlugin implements Plugin<Project> {
RepositorySystemSession session = newRepositorySystemSession(system)
project.extensions.create("versionScan", VersionScanExtension)
- project.task('scanVersions') {
+ def scanVersions = project.task('scanVersions') {
description = "Queries for all versions of configured modules and finds key classes"
}
- if (!project.gradle.startParameter.taskNames.contains('scanVersions')) {
+ def hasRelevantTask = project.gradle.startParameter.taskNames.contains('scanVersions')
+ hasRelevantTask |= project.gradle.startParameter.taskNames.contains('scanVersionsReport')
+ hasRelevantTask |= project.gradle.startParameter.taskNames.contains('verifyVersionScan')
+ if (!hasRelevantTask) {
return
}
- println "Adding scan tasks for $project"
+// println "Adding scan tasks for $project"
Set<String> allInclude = Sets.newConcurrentHashSet()
Set<String> allExclude = Sets.newConcurrentHashSet()
AtomicReference<Set<String>> keyPresent = new AtomicReference(Collections.emptySet())
AtomicReference<Set<String>> keyMissing = new AtomicReference(Collections.emptySet())
+
def scanVersionsReport = project.task('scanVersionsReport') {
description = "Prints the result of the scanVersions task"
doLast {
@@ -67,7 +71,9 @@ class VersionScanPlugin implements Plugin<Project> {
}
}
}
- project.tasks.scanVersions.finalizedBy(scanVersionsReport)
+ if (project.gradle.startParameter.taskNames.contains('scanVersions')) {
+ scanVersions.finalizedBy(scanVersionsReport)
+ }
project.repositories {
mavenCentral()
@@ -118,6 +124,48 @@ class VersionScanPlugin implements Plugin<Project> {
return
}
+ Map<String, String> verifyPresent = project.versionScan.verifyPresent
+ List<String> verifyMissing = project.versionScan.verifyMissing
+
+ if (!verifyPresent.isEmpty() || !verifyMissing.isEmpty()) {
+ def verifyVersionScan = project.task('verifyVersionScan') {
+ description = "Validates that the configured classes and methods are only present where expected."
+ doLast {
+ // This may already be done by the report task, but repeating for good measure.
+ keyPresent.get().removeAll(allExclude)
+ keyMissing.get().removeAll(allInclude)
+
+ assert keyPresent.get() != [] || keyMissing.get() != []
+
+ def errors = []
+ for (String className : verifyPresent.keySet()) {
+ String identifier = project.versionScan.scanMethods ? "$className|${verifyPresent.get(className)}" : className
+ if (!keyPresent.get().contains(identifier)) {
+ errors << "not a 'keyPresent' identifier: $identifier"
+ }
+ }
+ for (String className : verifyMissing) {
+ if (!keyMissing.get().contains(className)) {
+ errors << "not a 'keyMissing' identifier: $className"
+ }
+ }
+ errors.each {
+ System.err.println "Error for $group:$module - $it"
+ }
+ if (!errors.isEmpty()) {
+ throw new AssertionError("Version scan verification failed.\n" +
+ "Errors listed above are likely the result of a new module " +
+ "being published to Maven, not a code change in this repo.\n" +
+ "This does mean a fix should be made though to 'dd-trace-supported-framework.yaml'.")
+ }
+ }
+ }
+
+ if (project.gradle.startParameter.taskNames.contains('scanVersions')) {
+ scanVersions.finalizedBy(verifyVersionScan)
+ }
+ }
+
// println "Scanning ${includeVersionSet.size()} included and ${excludeVersionSet.size()} excluded versions. Included: ${includeVersionSet.collect { it.version }}}"
includeVersionSet.each { version ->
@@ -144,12 +192,15 @@ class VersionScanPlugin implements Plugin<Project> {
def jar = new JarFile(jarFile)
for (jarEntry in jar.entries()) {
if (jarEntry.name.endsWith(".class")) {
+ def className = jarEntry.name
+ className = className.replaceAll("/", ".")
+ className = className.replace(".class", "")
if (project.versionScan.scanMethods) {
findMethodNames(jar, jarEntry).each {
- contentSet.add("$jarEntry.name|$it")
+ contentSet.add("$className|$it".toString())
}
} else {
- contentSet.add("$jarEntry.name")
+ contentSet.add(className)
}
}
}
@@ -163,7 +214,10 @@ class VersionScanPlugin implements Plugin<Project> {
}
}
project.tasks.scanVersions.finalizedBy(task)
- project.tasks.scanVersionsReport.mustRunAfter(task)
+ project.tasks.scanVersionsReport.dependsOn(task)
+ if (project.tasks.hasProperty("verifyVersionScan")) {
+ project.tasks.verifyVersionScan.dependsOn(task)
+ }
}
def filter(List<Version> list) {

MODIFY
dd-java-agent/integrations/apache-httpclient/apache-httpclient.gradle
dd-java-agent/integrations/apache-httpclient/apache-httpclient.gradle
@@ -6,4 +6,8 @@ versionScan {
versions = "[4.3,)"
legacyGroup = "commons-httpclient"
legacyModule = "commons-httpclient"
+ verifyPresent = [
+ "org.apache.http.conn.SchemePortResolver": null,
+ "org.apache.http.conn.ssl.SSLContexts" : null,
+ ]
}

MODIFY
dd-java-agent/integrations/aws-sdk/aws-sdk.gradle
dd-java-agent/integrations/aws-sdk/aws-sdk.gradle
@@ -6,4 +6,9 @@ versionScan {
group = "com.amazonaws"
module = "aws-java-sdk-core"
versions = "[1.11.0,)"
+ verifyPresent = [
+ "com.amazonaws.http.client.HttpClientFactory" : null,
+ "com.amazonaws.http.apache.utils.ApacheUtils" : null,
+ "com.amazonaws.http.request.HttpRequestFactory": null,
+ ]
}

MODIFY
dd-java-agent/integrations/cassandra/cassandra.gradle
dd-java-agent/integrations/cassandra/cassandra.gradle
@@ -4,4 +4,9 @@ versionScan {
group = "com.datastax.cassandra"
module = "cassandra-driver-core"
versions = "[3.2,)"
+ verifyPresent = [
+ "com.datastax.driver.core.utils.MoreObjects" : null,
+ "com.datastax.driver.core.RemoteEndpointAwareNettySSLOptions": null,
+ "com.datastax.driver.core.GuavaCompatibility" : null,
+ ]
}

MODIFY
dd-java-agent/integrations/jms/jms.gradle
dd-java-agent/integrations/jms/jms.gradle
@@ -5,4 +5,8 @@ versionScan {
module = "javax.jms-api"
versions = "[2,)"
legacyModule = "jms-api"
+ verifyPresent = [
+ "javax.jms.JMSContext" : null,
+ "javax.jms.CompletionListener": null,
+ ]
}

MODIFY
dd-java-agent/integrations/mongo-async/mongo-async.gradle
dd-java-agent/integrations/mongo-async/mongo-async.gradle
@@ -4,4 +4,8 @@ versionScan {
group = "org.mongodb"
module = "mongodb-driver-async"
versions = "[3.0,)"
+ verifyPresent = [
+ "com.mongodb.operation.AsyncReadOperation": null,
+ "com.mongodb.client.model.MapReduceAction": null,
+ ]
}

MODIFY
dd-java-agent/integrations/mongo/mongo.gradle
dd-java-agent/integrations/mongo/mongo.gradle
@@ -4,4 +4,8 @@ versionScan {
group = "org.mongodb"
module = "mongo-java-driver"
versions = "[3.0,)"
+ verifyPresent = [
+ "com.mongodb.operation.AsyncReadOperation": null,
+ "com.mongodb.client.model.MapReduceAction": null,
+ ]
}

MODIFY
dd-java-agent/integrations/okhttp/okhttp.gradle
dd-java-agent/integrations/okhttp/okhttp.gradle
@@ -5,4 +5,9 @@ versionScan {
module = "okhttp"
versions = "[3.0,)"
legacyGroup = "com.squareup.okhttp"
+ verifyPresent = [
+ "okhttp3.Cookie" : null,
+ "okhttp3.ConnectionPool": null,
+ "okhttp3.Headers" : null,
+ ]
}

MODIFY
dd-java-agent/integrations/tomcat-embedded/tomcat-embedded.gradle
dd-java-agent/integrations/tomcat-embedded/tomcat-embedded.gradle
@@ -4,4 +4,9 @@ versionScan {
group = "org.apache.tomcat.embed"
module = "tomcat-embed-core"
versions = "[8.0,)"
+ verifyPresent = [
+ "org.apache.catalina.WebResource" : null,
+ "org.apache.catalina.webresources.TrackedInputStream" : null,
+ "org.apache.catalina.webresources.AbstractArchiveResource": null,
+ ]
}

MODIFY
dd-java-agent/integrations/tomcat/tomcat.gradle
dd-java-agent/integrations/tomcat/tomcat.gradle
@@ -4,4 +4,9 @@ versionScan {
group = "org.apache.tomcat"
module = "tomcat-catalina"
versions = "[8.0,)"
+ verifyPresent = [
+ "org.apache.catalina.WebResource" : null,
+ "org.apache.catalina.webresources.TrackedInputStream" : null,
+ "org.apache.catalina.webresources.AbstractArchiveResource": null,
+ ]
}

MODIFY
dd-java-agent/src/main/resources/dd-trace-supported-framework.yaml
dd-java-agent/src/main/resources/dd-trace-supported-framework.yaml
@@ -12,7 +12,6 @@ opentracing-apache-httpclient:
supported_version: 4\.[3|4|5]\..*
identifying_present_classes:
- org.apache.http.conn.SchemePortResolver
- - org.apache.http.config.Lookup
- org.apache.http.conn.ssl.SSLContexts
- artifact: commons-httpclient

Add version scan verification and test in CI