                 }
 
             case .run(let specifier):
-                let yamlPath = opts.path.build.appending(RelativePath("\(configuration).yaml"))
-                if opts.buildTests {
-                    try build(yamlPath: yamlPath, target: "test")
-                }
-                let success = try test(path: determineTestPath(opts: opts), xctestArg: specifier)
+                try buildTestsIfNeeded(opts)
+                let testPath = try determineTestPath(opts: opts)
+
+                let success = test(path: testPath, xctestArg: specifier)
                 exit(success ? 0 : 1)
             }
         } catch Error.buildYAMLNotFound {

        return (mode ?? .run(nil), opts)
    }

    private func test(path: AbsolutePath, xctestArg: String? = nil) throws -> Bool {
        guard isValidTestPath(path) else {
            throw TestError.testsExecutableNotFound
        }

        var args: [String] = []
      #if os(OSX)

        let packageName = opts.path.root.basename  //FIXME probably not true
        let maybePath = opts.path.build.appending(RelativePath(configuration)).appending(RelativePath("\(packageName)Tests.xctest"))

        let possibleTestPath: AbsolutePath

        if maybePath.asString.exists {
            possibleTestPath = maybePath
        } else {
            let possiblePaths = walk(opts.path.build.asString).filter {
                $0.basename != "Package.xctest" &&   // this was our hardcoded name, may still exist if no clean
