Replace the custom test runners with pytest.
MODIFY
.github/workflows/zulip-tests.yml
.github/workflows/zulip-tests.yml
@@ -52,13 +52,14 @@ jobs:
name: Running Test-Suite on Linux
run: |
source zulip-api-py3-venv/bin/activate
- tools/test-main
+ pytest --cov --cov-config=tools/.coveragerc --cov-report=xml
- if: runner.os == 'Windows'
name: Running Test-Suite on Windows
run: |
zulip-api-py3-venv\Scripts\Activate.ps1
- tools\test-main.ps1
+ pytest --cov --cov-config=tools\.coveragerc --cov-report=xml
- - name: Codecov
- uses: codecov/codecov-action@v1.2.1
+ - uses: codecov/codecov-action@v2
+ with:
+ files: coverage.xml

MODIFY
README.md
README.md
@@ -59,13 +59,14 @@ and [commit guidelines](https://zulip.readthedocs.io/en/latest/contributing/vers
### Running tests
-To run the tests for
+You can run all the tests with:
-* *zulip*: run `./tools/test-zulip`
+`pytest`
-* *zulip_bots*: run `./tools/test-lib && ./tools/test-bots`
-
-* *zulip_botserver*: run `./tools/test-botserver`
+or test individual packages with `pytest zulip`, `pytest zulip_bots`,
+or `pytest zulip_botserver` (see the [pytest
+documentation](https://docs.pytest.org/en/latest/how-to/usage.html)
+for more options).
To run the linter, type:

MODIFY
requirements.txt
requirements.txt
crayons
twine
-coverage>=4.4.1
black
isort
flake8
mock
pytest
+pytest-cov
-e ./zulip
-e ./zulip_bots
-e ./zulip_botserver

MODIFY
tools/.coveragerc
tools/.coveragerc
@@ -14,9 +14,3 @@ exclude_lines =
[run]
source = zulip, zulip_bots, zulip_botserver
-omit =
- # Parts of the test runner infrastructure
- tools/test-main
- tools/test-bots
- tools/test-botserver
- tools/test-zulip

DELETE
tools/server_lib/__init__.py
tools/server_lib/__init__.py
DELETE
tools/server_lib/test_handler.py
-import argparse
-import os
-import shutil
-import sys
-import unittest
-
-import pytest
-
-TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-os.chdir(os.path.dirname(TOOLS_DIR))
-
-
-def handle_input_and_run_tests_for_package(package_name, path_list):
- parser = argparse.ArgumentParser(description=f"Run tests for {package_name}.")
- parser.add_argument(
- "--coverage",
- nargs="?",
- const=True,
- default=False,
- help="compute test coverage (--coverage combine to combine with previous reports)",
- )
- parser.add_argument(
- "--pytest", "-p", default=False, action="store_true", help="run tests with pytest"
- )
- parser.add_argument(
- "--verbose",
- "-v",
- default=False,
- action="store_true",
- help="show verbose output (with pytest)",
- )
- options = parser.parse_args()
-
- test_session_title = f" Running tests for {package_name} "
- header = test_session_title.center(shutil.get_terminal_size().columns, "#")
- print(header)
-
- if options.coverage:
- import coverage
-
- cov = coverage.Coverage(config_file="tools/.coveragerc")
- if options.coverage == "combine":
- cov.load()
- cov.start()
-
- if options.pytest:
- location_to_run_in = os.path.join(TOOLS_DIR, "..", *path_list)
- paths_to_test = ["."]
- pytest_options = [
- "-s", # show output from tests; this hides the progress bar though
- "-x", # stop on first test failure
- "--ff", # runs last failure first
- ]
- pytest_options += ["-v"] if options.verbose else []
- os.chdir(location_to_run_in)
- result = pytest.main(paths_to_test + pytest_options)
- if result != 0:
- sys.exit(1)
- failures = False
- else:
- # Codecov seems to work only when using loader.discover. It failed to capture line executions
- # for functions like loader.loadTestFromModule or loader.loadTestFromNames.
- test_suites = unittest.defaultTestLoader.discover(os.path.join(*path_list))
- suite = unittest.TestSuite(test_suites)
- runner = unittest.TextTestRunner(verbosity=2)
- result = runner.run(suite)
- failures = result.failures
- if result.failures or result.errors:
- sys.exit(1)
-
- if not failures and options.coverage:
- cov.stop()
- cov.data_suffix = False # Disable suffix so that filename is .coverage
- cov.save()
- cov.html_report()
- print("HTML report saved in directory 'htmlcov'.")

DELETE
tools/test-bots
-#!/usr/bin/env python3
-
-import argparse
-import glob
-import os
-import sys
-import unittest
-from os.path import basename, dirname
-
-import pytest
-
-
-def parse_args():
- description = """
-Script to run test_<bot_name>.py files in the
-zulip_bot/zulip_bots/bots/<bot_name> directories.
-
-Running all tests:
-
-./test-bots
-
-Running tests for specific bots:
-
-./test-bots define xkcd
-
-Running all tests excluding certain bots (the
-following command would run tests for all bots except
-the tests for xkcd and wikipedia bots):
-
-./test-bots --exclude xkcd wikipedia
-"""
- parser = argparse.ArgumentParser(description=description)
-
- parser.add_argument(
- "bots_to_test",
- metavar="bot",
- nargs="*",
- default=[],
- help="specific bots to test (default is all)",
- )
- parser.add_argument(
- "--coverage",
- nargs="?",
- const=True,
- default=False,
- help="compute test coverage (--coverage combine to combine with previous reports)",
- )
- parser.add_argument("--exclude", metavar="bot", nargs="*", default=[], help="bot(s) to exclude")
- parser.add_argument(
- "--error-on-no-init",
- default=False,
- action="store_true",
- help="whether to exit if a bot has tests which won't run due to no __init__.py",
- )
- parser.add_argument(
- "--pytest", "-p", default=False, action="store_true", help="run tests with pytest"
- )
- parser.add_argument(
- "--verbose",
- "-v",
- default=False,
- action="store_true",
- help="show verbose output (with pytest)",
- )
- return parser.parse_args()
-
-
-def main():
- TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
- os.chdir(os.path.dirname(TOOLS_DIR))
- sys.path.insert(0, TOOLS_DIR)
- bots_dir = os.path.join(TOOLS_DIR, "..", "zulip_bots/zulip_bots/bots")
- glob_pattern = bots_dir + "/*/test_*.py"
- test_modules = glob.glob(glob_pattern)
-
- # get only the names of bots that have tests
- available_bots = map(lambda path: basename(dirname(path)), test_modules)
-
- options = parse_args()
-
- if options.coverage:
- import coverage
-
- cov = coverage.Coverage(config_file="tools/.coveragerc")
- if options.coverage == "combine":
- cov.load()
- cov.start()
-
- if options.bots_to_test:
- specified_bots = options.bots_to_test
- else:
- specified_bots = available_bots
-
- # Use of a set ensures we don't end up with duplicate tests with unittest
- # (from globbing multiple test_*.py files, or multiple on the command line)
- bots_to_test = {bot for bot in specified_bots if bot not in options.exclude}
-
- if options.pytest:
- pytest_options = [
- "-s", # show output from tests; this hides the progress bar though
- "-x", # stop on first test failure
- "--ff", # runs last failure first
- ]
- pytest_options += ["-v"] if options.verbose else []
- os.chdir(bots_dir)
- result = pytest.main(sorted(bots_to_test) + pytest_options)
- if result != 0:
- sys.exit(1)
- failures = False
- else:
- # Codecov seems to work only when using loader.discover. It failed to
- # capture line executions for functions like loader.loadTestFromModule
- # or loader.loadTestFromNames.
- top_level = "zulip_bots/zulip_bots/bots/"
- loader = unittest.defaultTestLoader
- test_suites = []
- for name in bots_to_test:
- try:
- test_suites.append(loader.discover(top_level + name, top_level_dir=top_level))
- except ImportError as exception:
- print(exception)
- print(
- "This likely indicates that you need a '__init__.py' file in your bot directory."
- )
- if options.error_on_no_init:
- sys.exit(1)
-
- suite = unittest.TestSuite(test_suites)
- runner = unittest.TextTestRunner(verbosity=2)
- result = runner.run(suite)
- failures = result.failures
- if failures or result.errors:
- sys.exit(1)
-
- if not failures and options.coverage:
- cov.stop()
- cov.data_suffix = False # Disable suffix so that filename is .coverage
- cov.save()
- cov.html_report()
- print("HTML report saved under directory 'htmlcov'.")
-
-
-if __name__ == "__main__":
- main()

DELETE
tools/test-botserver
-#!/usr/bin/env python3
-
-from server_lib.test_handler import handle_input_and_run_tests_for_package
-
-if __name__ == "__main__":
- handle_input_and_run_tests_for_package("Botserver", ["zulip_botserver"])

DELETE
tools/test-lib
-#!/usr/bin/env python3
-
-from server_lib.test_handler import handle_input_and_run_tests_for_package
-
-if __name__ == "__main__":
- handle_input_and_run_tests_for_package("Bot library", ["zulip_bots", "zulip_bots", "tests"])

DELETE
tools/test-main
-#!/usr/bin/env bash
-
-set -ev
-
-tools/test-bots "$@" --coverage
-tools/test-botserver "$@" --coverage combine
-tools/test-zulip "$@" --coverage combine
-tools/test-lib "$@" --coverage combine

DELETE
tools/test-main.ps1
-Set-StrictMode -Version Latest
-$ErrorActionPreference = "Stop"
-
-python tools\test-bots "$args" --coverage
-python tools\test-botserver "$args" --coverage combine
-python tools\test-zulip "$args" --coverage combine
-python tools\test-lib "$args" --coverage combine

DELETE
tools/test-zulip
-#!/usr/bin/env python3
-
-from server_lib.test_handler import handle_input_and_run_tests_for_package
-
-if __name__ == "__main__":
- handle_input_and_run_tests_for_package("API", ["zulip"])

Replace the custom test runners with pytest.

Signed-off-by: Anders Kaseorg <anders@zulip.com>