Parallelize syntax regression tests
The syntax highlighting regression tests can be trivially parallelized.
On my notebook (8 core), this results in a 3.9x speedup.
@@ -6,7 +6,7 @@ import sys
import os.path as path
import os
import argparse
-
+from multiprocessing import Pool
BAT_OPTIONS = [
"--no-config",
@@ -38,14 +38,8 @@ def get_options(source):
return options
-def create_highlighted_versions(output_basepath):
- root = os.path.dirname(os.path.abspath(__file__))
- sources = path.join(root, "source", "*")
-
- for source in glob.glob(path.join(sources, "*")) + glob.glob(
- path.join(sources, ".*")
- ):
- try:
+def create_highlighted_version(args):
+ output_basepath, source = args
env = os.environ.copy()
env.pop("BAT_CACHE_PATH", None)
env.pop("BAT_CONFIG_DIR", None)
@@ -63,7 +57,7 @@ def create_highlighted_versions(output_basepath):
source_filename = path.basename(source)
if source_filename in SKIP_FILENAMES:
- continue
+ return
bat_output = subprocess.check_output(
["bat"] + get_options(source) + [source],
@@ -80,6 +74,21 @@ def create_highlighted_versions(output_basepath):
output_file.write(bat_output)
print("Created '{}'".format(output_path))
+
+
+def create_highlighted_versions(output_basepath):
+ root = os.path.dirname(os.path.abspath(__file__))
+ source_paths = path.join(root, "source", "*")
+
+ sources = []
+ for source in glob.glob(path.join(source_paths, "*")) + glob.glob(
+ path.join(source_paths, ".*")
+ ):
+ sources.append((output_basepath, source))
+
+ try:
+ with Pool() as p:
+ p.map(create_highlighted_version, sources)
except subprocess.CalledProcessError as err:
print(
"=== Error: Could not highlight source file '{}".format(source),
@@ -93,13 +102,15 @@ def create_highlighted_versions(output_basepath):
"=== bat stderr:\n{}".format(err.stderr.decode("utf-8")),
file=sys.stderr,
)
- sys.exit(1)
+ return False
except FileNotFoundError:
print(
"Error: Could not execute 'bat'. Please make sure that the executable "
"is available on the PATH."
)
- sys.exit(1)
+ return False
+
+ return True
if __name__ == "__main__":
@@ -118,4 +129,5 @@ if __name__ == "__main__":
args = parser.parse_args()
- create_highlighted_versions(output_basepath=args.output)
+ if not create_highlighted_versions(output_basepath=args.output):
+ sys.exit(1)
