diff --git a/.travis.yml b/.travis.yml index 126fced23..d9d26d9c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,7 @@ env: test_ipaplatform test_ipapython test_ipaserver + test_ipatests_plugins test_xmlrpc/test_[l-z]*.py" - TASK_TO_RUN="tox" TEST_RUNNER_CONFIG=".test_runner_config.yaml" diff --git a/Makefile.am b/Makefile.am index 7d84aa532..31a7abaf5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -207,7 +207,8 @@ fastcheck: fasttest: $(GENERATED_PYTHON_FILES) ipasetup.py @ # --ignore doubles speed of total test run compared to pytest.skip() @ # on module. - PYTHONPATH=$(abspath $(top_srcdir)) $(PYTHON) ipatests/ipa-run-tests \ + PATH=$(abspath ipatests):$$PATH PYTHONPATH=$(abspath $(top_srcdir)) \ + $(PYTHON) ipatests/ipa-run-tests \ --skip-ipaapi \ --ignore $(abspath $(top_srcdir))/ipatests/test_integration \ --ignore $(abspath $(top_srcdir))/ipatests/test_xmlrpc diff --git a/ipatests/azure/azure-pipelines.yml b/ipatests/azure/azure-pipelines.yml index 1c92f37b3..89f81b87f 100644 --- a/ipatests/azure/azure-pipelines.yml +++ b/ipatests/azure/azure-pipelines.yml @@ -137,6 +137,7 @@ jobs: - test_ipalib - test_ipaplatform - test_ipapython + - test_ipatests_plugins testsToIgnore: - test_integration - test_webui diff --git a/ipatests/conftest.py b/ipatests/conftest.py index c09df4aed..ae523c88a 100644 --- a/ipatests/conftest.py +++ b/ipatests/conftest.py @@ -28,7 +28,8 @@ pytest_plugins = [ 'ipatests.pytest_ipa.beakerlib', 'ipatests.pytest_ipa.declarative', 'ipatests.pytest_ipa.nose_compat', - 'ipatests.pytest_ipa.integration' + 'ipatests.pytest_ipa.integration', + 'pytester', ] diff --git a/ipatests/ipa-run-tests b/ipatests/ipa-run-tests index 4585e8e04..3b3e462b1 100755 --- a/ipatests/ipa-run-tests +++ b/ipatests/ipa-run-tests @@ -22,15 +22,13 @@ """Pytest wrapper for running an installed (not in-tree) IPA test suite -Any command-line arguments are passed directly to py.test. -The current directory is changed to the locaition of the ipatests package, -so any relative paths given will be based on the ipatests module's path +Any command-line arguments are passed directly to pytest. +The current directory is changed to the location of the ipatests package, +so any relative paths given will be based on the ipatests module's path. """ import os -import copy import sys -import glob import pytest @@ -41,82 +39,29 @@ os.environ['IPATEST_XUNIT_PATH'] = os.path.join(os.getcwd(), 'nosetests.xml') HERE = os.path.dirname(os.path.abspath(ipatests.__file__)) +def has_option(option): + if option in sys.argv: + return True + for arg in sys.argv: + for argi in arg.split("="): + if option in argi: + return True + return False -class ArgsManglePlugin(object): - """Modify pytest arguments +# don't override specified command line options +if not has_option("confcutdir"): + sys.argv.insert(1, "--confcutdir={}".format(HERE)) +# for backward compatibility +if not has_option("cache_dir"): + sys.argv[1:1] = ["-o", 'cache_dir={}'.format(os.path.join(os.getcwd(), + ".pytest_cache"))] - * Add confcutdir if hasn't been set already - * Mangle paths to support tests both relative to basedir and ipatests/ - * Default to ipatests/ if no tests are specified - """ - def pytest_load_initial_conftests(self, early_config, parser, args): - # During initial loading, parser supports only basic options - ns = early_config.known_args_namespace - if not ns.confcutdir: - # add config cut directory to only load fixtures from ipatests/ - args.insert(0, '--confcutdir={}'.format(HERE)) - - if not ns.file_or_dir: - # No file or directory found, run all tests - args.append(HERE) - else: - vargs = [] - for name in ns.file_or_dir: - idx = args.index(name) - # split on pytest separator - # ipatests/test_ipaplatform/test_importhook.py::test_override - filename, sep, suffix = name.partition('::') - # a file or directory relative to cwd or already absolute - if os.path.exists(filename): - continue - if '*' in filename: - # Expand a glob, we'll flatten the list later - paths = glob.glob(os.path.join(HERE, filename)) - vargs.append([idx, paths]) - else: - # a file or directory relative to ipatests package - args[idx] = sep.join((os.path.join(HERE, filename), suffix)) - # flatten and insert all expanded file names - base = 0 - for idx, items in vargs: - args.pop(base + idx) - for item in items: - args.insert(base + idx, item) - base += len(items) - - # replace ignores, e.g. "--ignore test_integration" is changed to - # "--ignore path/to/ipatests/test_integration" - if ns.ignore: - vargs = [] - for ignore in ns.ignore: - idx = args.index(ignore) - if os.path.exists(ignore): - continue - if '*' in ignore: - # expand a glob, we'll flatten the list later - paths = glob.glob(os.path.join(HERE, ignore)) - vargs.append([idx, paths]) - else: - args[idx] = os.path.join(HERE, ignore) - # flatten and insert all expanded file names - base = 0 - for idx, items in vargs: - # since we are expanding, remove old pair - # --ignore and old file name - args.pop(base + idx) - args.pop(base + idx) - for item in items: - # careful: we need to add a pair - # --ignore and new filename - args.insert(base + idx, '--ignore') - args.insert(base + idx, item) - base += len(items) * 2 - 1 - - # rebuild early_config's known args with new args. The known args - # are used for initial conftest.py from ipatests, which adds - # additional arguments. - early_config.known_args_namespace = parser.parse_known_args( - args, namespace=copy.copy(early_config.option)) - - -sys.exit(pytest.main(plugins=[ArgsManglePlugin()])) +pyt_args = [sys.executable, "-c", + "import sys,pytest;sys.exit(pytest.main())"] + sys.argv[1:] +# shell is needed to perform globbing +sh_args = ["/bin/sh", "--norc", "--noprofile", "-c", "--"] +pyt_args_esc = ["'{}'".format(x) if " " in x else x for x in pyt_args] +args = sh_args + [" ".join(pyt_args_esc)] +os.chdir(HERE) +sys.stdout.flush() +os.execv(args[0], args) diff --git a/ipatests/man/ipa-run-tests.1 b/ipatests/man/ipa-run-tests.1 index 66c2a0b00..08f2f0058 100644 --- a/ipatests/man/ipa-run-tests.1 +++ b/ipatests/man/ipa-run-tests.1 @@ -22,42 +22,32 @@ ipa\-run\-tests \- Run the FreeIPA test suite .SH "SYNOPSIS" ipa\-run\-tests [options] .SH "DESCRIPTION" -ipa\-run\-tests is a wrapper around nosetests that run the FreeIPA test suite. +ipa\-run\-tests is a wrapper around Pytest that run the FreeIPA test suite. It is intended to be used for developer testing and in continuous integration systems. -It loads IPA-internal Nose plugins ordered-tests and beakerlib. -The ordered-tests plugin is enabled automatically. - -The FreeIPA test suite installed system\-wide is selected via Nose's \-\-where -option. It is possible to select a subset of the entire test suite by specifying a test file relative to the ipatests package, for example: ipa-run-tests test_integration/test_simple_replication.py .SH "OPTIONS" -All command-line options are passed to the underlying Nose runner. -See nosetests(1) for a complete list. - -The internal IPA plugins add an extra option: - -.TP -\fB\-\-with-beakerlib\fR -Enable BeakerLib integration. -Test phases, failures and passes, and log messages are reported using -beakerlib(1) commands. -This option requires the beakerlib.sh script to be sourced. +All command-line options are passed to the underlying Pytest runner. +See "pytest --help" for a complete list. .SH "EXIT STATUS" -0 if the command was successful - -nonzero if any error or failure occurred +Running pytest can result in six different exit codes: +Exit code 0: All tests were collected and passed successfully +Exit code 1: Tests were collected and run but some of the tests failed +Exit code 2: Test execution was interrupted by the user +Exit code 3: Internal error happened while executing tests +Exit code 4: pytest command line usage error +Exit code 5: No tests were collected .SH "CONFIGURATION" Please see ipa-test-config(1) for a description of configuration environment variables. .SH "REFERENCES" -A full description of the FreeIPA integration testing framework is available at -http://www.freeipa.org/page/V3/Integration_testing +A full description of the FreeIPA testing is available at +https://www.freeipa.org/page/Testing diff --git a/ipatests/setup.py b/ipatests/setup.py index cd498b787..8b89f90ca 100644 --- a/ipatests/setup.py +++ b/ipatests/setup.py @@ -44,6 +44,7 @@ if __name__ == '__main__': "ipatests.test_ipapython", "ipatests.test_ipaserver", "ipatests.test_ipaserver.test_install", + "ipatests.test_ipatests_plugins", "ipatests.test_webui", "ipatests.test_xmlrpc", "ipatests.test_xmlrpc.tracker" diff --git a/ipatests/test_ipatests_plugins/__init__.py b/ipatests/test_ipatests_plugins/__init__.py new file mode 100644 index 000000000..4f35bc042 --- /dev/null +++ b/ipatests/test_ipatests_plugins/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (C) 2019 FreeIPA Contributors see COPYING for license +# + +""" +Sub-package containing unit tests for IPA internal test plugins +""" diff --git a/ipatests/test_ipatests_plugins/test_ipa_run_tests.py b/ipatests/test_ipatests_plugins/test_ipa_run_tests.py new file mode 100644 index 000000000..b1bc720a9 --- /dev/null +++ b/ipatests/test_ipatests_plugins/test_ipa_run_tests.py @@ -0,0 +1,162 @@ +# +# Copyright (C) 2019 FreeIPA Contributors see COPYING for license +# + +import os + +import pytest + +MOD_NAME = "test_module_{}" +FUNC_NAME = "test_func_{}" +MODS_NUM = 5 + + +@pytest.fixture +def ipatestdir(testdir, monkeypatch): + """ + Create MODS_NUM test modules within testdir/ipatests. + Each module contains 1 test function. + Patch PYTHONPATH with created package path to override the system's + ipatests + """ + ipatests_dir = testdir.mkpydir("ipatests") + for i in range(MODS_NUM): + ipatests_dir.join("{}.py".format(MOD_NAME.format(i))).write( + "def {}(): pass".format(FUNC_NAME.format(i))) + + python_path = os.pathsep.join( + filter(None, [str(testdir.tmpdir), os.environ.get("PYTHONPATH", "")])) + monkeypatch.setenv("PYTHONPATH", python_path) + + def run_ipa_tests(*args): + cmdargs = ["ipa-run-tests", "-v"] + list(args) + return testdir.run(*cmdargs, timeout=60) + + testdir.run_ipa_tests = run_ipa_tests + return testdir + + +def test_ipa_run_tests_basic(ipatestdir): + """ + Run ipa-run-tests with default arguments + """ + result = ipatestdir.run_ipa_tests() + assert result.ret == 0 + result.assert_outcomes(passed=MODS_NUM) + for mod_num in range(MODS_NUM): + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))]) + + +def test_ipa_run_tests_glob1(ipatestdir): + """ + Run ipa-run-tests using glob patterns to collect tests + """ + result = ipatestdir.run_ipa_tests("{mod}".format( + mod="test_modul[!E]?[0-5]*")) + assert result.ret == 0 + result.assert_outcomes(passed=MODS_NUM) + for mod_num in range(MODS_NUM): + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))]) + + +def test_ipa_run_tests_glob2(ipatestdir): + """ + Run ipa-run-tests using glob patterns to collect tests + """ + result = ipatestdir.run_ipa_tests("{mod}".format( + mod="test_module_{0,1}*")) + assert result.ret == 0 + result.assert_outcomes(passed=2) + for mod_num in range(2): + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))]) + + +def test_ipa_run_tests_specific_nodeid(ipatestdir): + """ + Run ipa-run-tests using nodeid to collect test + """ + mod_num = 0 + result = ipatestdir.run_ipa_tests("{mod}.py::{func}".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))) + assert result.ret == 0 + result.assert_outcomes(passed=1) + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))]) + + +@pytest.mark.parametrize( + "expr", + [["-k", "not {func}".format(func=FUNC_NAME.format(0))], + ["-k not {func}".format(func=FUNC_NAME.format(0))]]) +def test_ipa_run_tests_expression(ipatestdir, expr): + """ + Run ipa-run-tests using expression + """ + result = ipatestdir.run_ipa_tests(*expr) + assert result.ret == 0 + result.assert_outcomes(passed=4) + for mod_num in range(1, MODS_NUM): + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))]) + + +def test_ipa_run_tests_ignore_basic(ipatestdir): + """ + Run ipa-run-tests ignoring one test module + """ + result = ipatestdir.run_ipa_tests( + "--ignore", "{mod}.py".format(mod=MOD_NAME.format(0)), + "--ignore", "{mod}.py".format(mod=MOD_NAME.format(1)), + ) + assert result.ret == 0 + result.assert_outcomes(passed=MODS_NUM - 2) + for mod_num in range(2, MODS_NUM): + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))]) + + +def test_ipa_run_tests_defaultargs(ipatestdir): + """ + Checking the ipa-run-tests defaults: + * cachedir + * rootdir + """ + mod_num = 0 + result = ipatestdir.run_ipa_tests("{mod}.py::{func}".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))) + assert result.ret == 0 + result.assert_outcomes(passed=1) + result.stdout.re_match_lines([ + "^cachedir: {cachedir}$".format( + cachedir=os.path.join(os.getcwd(), ".pytest_cache")), + "^rootdir: {rootdir}([,].*)?$".format( + rootdir=os.path.join(str(ipatestdir.tmpdir), "ipatests")) + ]) + + +def test_ipa_run_tests_confcutdir(ipatestdir): + """ + Checking the ipa-run-tests defaults: + * confcutdir + """ + mod_num = 0 + ipatestdir.makeconftest("import somenotexistedpackage") + result = ipatestdir.run_ipa_tests("{mod}.py::{func}".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))) + assert result.ret == 0 + result.assert_outcomes(passed=1) + result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format( + mod=MOD_NAME.format(mod_num), + func=FUNC_NAME.format(mod_num))])