Remove perf monitoring for STL implementations.
Bug
ADD
internal/scripts/eris_perf.py
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+###############################################################################
+# Copyright (c) 2018 NVIDIA Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+from sys import exit
+
+from os.path import join, dirname, basename, realpath
+
+from csv import DictReader as csv_dict_reader
+
+from subprocess import Popen
+
+from argparse import ArgumentParser as argument_parser
+
+###############################################################################
+
+def printable_cmd(c):
+ """Converts a `list` of `str`s representing a shell command to a printable
+ `str`."""
+ return " ".join(map(lambda e: '"' + str(e) + '"', c))
+
+###############################################################################
+
+def print_file(p):
+ """Open the path `p` and print its contents to `stdout`."""
+ print "********************************************************************************"
+ with open(p) as f:
+ for line in f:
+ print line,
+ print "********************************************************************************"
+
+###############################################################################
+
+ap = argument_parser(
+ description = (
+ "CUDA Eris driver script: runs a benchmark suite multiple times, combines "
+ "the results, and outputs them in the CUDA Eris performance result format."
+ )
+)
+
+ap.add_argument(
+ "-b", "--benchmark",
+ help = ("The location of the benchmark suite executable to run."),
+ type = str,
+ default = join(dirname(realpath(__file__)), "bench"),
+ metavar = "R"
+)
+
+ap.add_argument(
+ "-p", "--postprocess",
+ help = ("The location of the postprocessing script to run to combine the "
+ "results."),
+ type = str,
+ default = join(dirname(realpath(__file__)), "combine_benchmark_results.py"),
+ metavar = "R"
+)
+
+ap.add_argument(
+ "-r", "--runs",
+ help = ("Run the benchmark suite `R` times.a),"),
+ type = int, default = 5,
+ metavar = "R"
+)
+
+args = ap.parse_args()
+
+if args.runs <= 0:
+ print "ERROR: `--runs` must be greater than `0`."
+ ap.print_help()
+ exit(1)
+
+BENCHMARK_EXE = args.benchmark
+BENCHMARK_NAME = basename(BENCHMARK_EXE)
+POSTPROCESS_EXE = args.postprocess
+OUTPUT_FILE_NAME = lambda i: BENCHMARK_NAME + "_" + str(i) + ".csv"
+COMBINED_OUTPUT_FILE_NAME = BENCHMARK_NAME + "_combined.csv"
+
+###############################################################################
+
+print '&&&& RUNNING {0}'.format(BENCHMARK_NAME)
+
+print '#### RUNS {0}'.format(args.runs)
+
+###############################################################################
+
+print '#### CMD {0}'.format(BENCHMARK_EXE)
+
+for i in xrange(args.runs):
+ with open(OUTPUT_FILE_NAME(i), "w") as output_file:
+ print '#### RUN {0} OUTPUT -> {1}'.format(i, OUTPUT_FILE_NAME(i))
+
+ p = None
+
+ try:
+ p = Popen(BENCHMARK_EXE, stdout = output_file, stderr = output_file)
+ p.communicate()
+ except OSError as ex:
+ print_file(OUTPUT_FILE_NAME(i))
+ print '#### ERROR Caught OSError `{0}`.'.format(ex)
+ print '&&&& FAILED {0}'.format(BENCHMARK_NAME)
+ exit(-1)
+
+ print_file(OUTPUT_FILE_NAME(i))
+
+ if p.returncode != 0:
+ print '#### ERROR Process exited with code {0}.'.format(p.returncode)
+ print '&&&& FAILED {0}'.format(BENCHMARK_NAME)
+ exit(p.returncode)
+
+###############################################################################
+
+post_cmd = [POSTPROCESS_EXE]
+
+# Add dependent variable options.
+post_cmd += ["-dSTL Average Walltime,STL Walltime Uncertainty,STL Trials"]
+post_cmd += ["-dSTL Average Throughput,STL Throughput Uncertainty,STL Trials"]
+post_cmd += ["-dThrust Average Walltime,Thrust Walltime Uncertainty,Thrust Trials"]
+post_cmd += ["-dThrust Average Throughput,Thrust Throughput Uncertainty,Thrust Trials"]
+
+post_cmd += [OUTPUT_FILE_NAME(i) for i in range(args.runs)]
+
+print '#### CMD {0}'.format(printable_cmd(post_cmd))
+
+with open(COMBINED_OUTPUT_FILE_NAME, "w") as output_file:
+ p = None
+
+ try:
+ p = Popen(post_cmd, stdout = output_file, stderr = output_file)
+ p.communicate()
+ except OSError as ex:
+ print_file(COMBINED_OUTPUT_FILE_NAME)
+ print '#### ERROR Caught OSError `{0}`.'.format(ex)
+ print '&&&& FAILED {0}'.format(BENCHMARK_NAME)
+ exit(-1)
+
+ print_file(COMBINED_OUTPUT_FILE_NAME)
+
+ if p.returncode != 0:
+ print '#### ERROR Process exited with code {0}.'.format(p.returncode)
+ print '&&&& FAILED {0}'.format(BENCHMARK_NAME)
+ exit(p.returncode)
+
+ with open(COMBINED_OUTPUT_FILE_NAME) as input_file:
+ reader = csv_dict_reader(input_file)
+
+ variable_units = reader.next() # Get units header row.
+
+ distinguishing_variables = reader.fieldnames
+
+ measured_variables = [
+ ("STL Average Throughput", "+"),
+ ("Thrust Average Throughput", "+")
+ ]
+
+ for record in reader:
+ for variable, directionality in measured_variables:
+ # Don't monitor regressions for STL implementations, nvbug 28980890:
+ if "STL" in variable:
+ continue
+ print "&&&& PERF {0}_{1}_{2}bit_{3}mib_{4} {5} {6}{7}".format(
+ record["Algorithm"],
+ record["Element Type"],
+ record["Element Size"],
+ record["Total Input Size"],
+ variable.replace(" ", "_").lower(),
+ record[variable],
+ directionality,
+ variable_units[variable]
+ )
+
+###############################################################################
+
+print '&&&& PASSED {0}'.format(BENCHMARK_NAME)
+

Remove perf monitoring for STL implementations.

Bug 200650178