testing: Add code test coverage reporting.
Fixes
ADD
.codecov.yml
+comment: off
+
+coverage:
+ status:
+ project:
+ default:
+ target: auto
+ threshold: 0.03
+ base: auto
+ patch: off

MODIFY
.travis.yml
.travis.yml
@@ -10,9 +10,17 @@ matrix:
- python: "3.6"
env: TEST_SUITE=lint
install:
+ # Install codecov, the library for the code coverage reporting tool we use
+ - pip install codecov
+
+ # Install pycodestyle, the library for checking pep8 code conformity.
- pip install pycodestyle==2.3.1
+
+ # Install all API packages
- pip install ./zulip
- pip install ./zulip_bots
- pip install ./zulip_botserver
script:
- tools/$TEST_SUITE
+after_success:
+ - codecov

ADD
tools/.coveragerc
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+ # Re-enable the standard coverage pragma
+ nocoverage
+ # Don't complain if non-runnable code isn't run:
+ if False:
+ # Don't require coverage for base class NotImplementedErrors
+ raise NotImplementedError()
+ # Don't require coverage for test suite AssertionError -- they're usually for clarity
+ raise AssertionError
+ # Don't require coverage for Python 2/3 variations; it's impossible to cover on both
+ if six.PY3:
+ if six.PY2:
+ # Don't require coverage for __unicode__ statements just used for printing
+ def __unicode__[(]self[)]:
+
+[run]
+source = zulip_bots, zulip_botserver
+omit =
+ # Parts of the test runner infrastructure
+ tools/test-main
+ tools/test-bots
+ tools/test-botserver

MODIFY
tools/test-bots
tools/test-bots
from os.path import dirname, basename
from importlib import import_module
-import argparse
+
import os
import sys
+import argparse
import glob
import unittest
-import logging
-
def load_tests_from_modules(names, template):
loader = unittest.defaultTestLoader
test_suites = []
-
for name in names:
module = import_module(template.format(name=name))
test_suites.append(loader.loadTestsFromModule(module))
return test_suites
+def load_all_tests():
+ loader = unittest.defaultTestLoader
+ return loader.discover('zulip_bots')
def parse_args(available_bots):
description = """
Script to run test_<bot_name>.py files in the
zulip_bot/zulip_bots/bots/<bot_name> directories.
-Running tests for all bots:
+Running all tests:
./test-bots
@@ -34,7 +35,7 @@ Running tests for specific bots:
./test-bots define xkcd
-Running tests for all bots excluding certain bots (the
+Running all tests excluding certain bots (the
following command would run tests for all bots except
the tests for xkcd and wikipedia bots):
@@ -45,9 +46,13 @@ the tests for xkcd and wikipedia bots):
parser.add_argument('bots_to_test',
metavar='bot',
nargs='*',
- default=available_bots,
+ default=[],
help='specific bots to test (default is all)')
-
+ parser.add_argument('--coverage',
+ nargs='?',
+ const=True,
+ default=False,
+ help='compute test coverage (--coverage combine to combine with previous reports)')
parser.add_argument('--exclude',
metavar='bot',
nargs='*',
@@ -57,22 +62,32 @@ the tests for xkcd and wikipedia bots):
def main():
- current_dir = os.path.dirname(os.path.abspath(__file__))
- bots_dir = os.path.join(current_dir, '..', 'zulip_bots/zulip_bots/bots')
+ TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+ os.chdir(os.path.dirname(TOOLS_DIR))
+ sys.path.insert(0, TOOLS_DIR)
+ bots_dir = os.path.join(TOOLS_DIR, '..', 'zulip_bots/zulip_bots/bots')
glob_pattern = bots_dir + '/*/test_*.py'
test_modules = glob.glob(glob_pattern)
# get only the names of bots that have tests
available_bots = map(lambda path: basename(dirname(path)), test_modules)
-
options = parse_args(available_bots)
+ if options.coverage:
+ import coverage
+ cov = coverage.Coverage(config_file="tools/.coveragerc")
+ if options.coverage == 'combine':
+ cov.load()
+ cov.start()
+
+ if options.bots_to_test:
bots_to_test = filter(lambda bot: bot not in options.exclude,
options.bots_to_test)
test_suites = load_tests_from_modules(
bots_to_test,
- template='zulip_bots.bots.{name}.test_{name}'
- )
+ template='zulip_bots.bots.{name}.test_{name}')
+ else:
+ test_suites = load_all_tests()
suite = unittest.TestSuite(test_suites)
runner = unittest.TextTestRunner(verbosity=2)
@@ -80,6 +95,12 @@ def main():
if result.failures or result.errors:
sys.exit(1)
+ if not result.failures and options.coverage:
+ cov.stop()
+ cov.data_suffix = False # Disable suffix so that filename is .coverage
+ cov.save()
+ cov.html_report()
+ print("HTML report saved under directory 'htmlcov'.")
if __name__ == '__main__':
main()

MODIFY
tools/test-botserver
tools/test-botserver
@@ -13,13 +13,17 @@ def main():
parser = argparse.ArgumentParser(description="Run tests for the zulip_botserver package.")
parser.add_argument('--coverage',
- action='store_true',
- help='compute test coverage')
+ nargs='?',
+ const=True,
+ default=False,
+ help='compute test coverage (--coverage combine to combine with previous reports)')
options = parser.parse_args()
if options.coverage:
import coverage
cov = coverage.Coverage(config_file="tools/.coveragerc")
+ if options.coverage == 'combine':
+ cov.load()
cov.start()
test_suites = unittest.defaultTestLoader.discover('zulip_botserver')
@@ -31,8 +35,6 @@ def main():
if not result.failures and options.coverage:
cov.stop()
- cov.save()
- cov.combine()
cov.data_suffix = False # Disable suffix so that filename is .coverage
cov.save()
cov.html_report()

MODIFY
tools/test-main
tools/test-main
set -ev
-CURRENT_DIR=$(dirname $(dirname "$0"))
-cd "$CURRENT_DIR"
-
-python -m unittest discover -v zulip_botserver
-tools/test-bots
+tools/test-bots --coverage combine
+tools/test-botserver --coverage combine

testing: Add code test coverage reporting.

Fixes #38.