mirror of
https://salsa.debian.org/freeipa-team/freeipa.git
synced 2024-12-22 15:13:50 -06:00
Simplify ipa-run-tests script
This is a sort of rollback to the pre #93c158b05 state with several improvements. For now, the nodeids calculation by ipa-run-tests is not stable, since it depends on current working directory. Nodeids (tests addresses) are utilized by the other plugins, for example. Unfortunately, the `pytest_load_initial_conftests` hook doesn't correctly work with pytest internal paths, since it is called after the calculation of rootdir was performed, for example. Eventually, it's simpler to follow the default convention for Python test discovery. There is at least one drawback of new "old" implementation. The ignore rules don't support globs, because pytest 4.3.0+ has the same facility via `--ignore-glob`: > Add the `--ignore-glob` parameter to exclude test-modules with > Unix shell-style wildcards. Add the collect_ignore_glob for > conftest.py to exclude test-modules with Unix shell-style > wildcards. Upon switching to pytest4 it will be possible to utilize this. Anyway, tests for checking current basic facilities of ipa-run-tests were added. Fixes: https://pagure.io/freeipa/issue/8007 Signed-off-by: Stanislav Levin <slev@altlinux.org> Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>
This commit is contained in:
parent
ef39e1b02a
commit
2312b38a67
@ -38,6 +38,7 @@ env:
|
||||
test_ipaplatform
|
||||
test_ipapython
|
||||
test_ipaserver
|
||||
test_ipatests_plugins
|
||||
test_xmlrpc/test_[l-z]*.py"
|
||||
- TASK_TO_RUN="tox"
|
||||
TEST_RUNNER_CONFIG=".test_runner_config.yaml"
|
||||
|
@ -207,7 +207,8 @@ fastcheck:
|
||||
fasttest: $(GENERATED_PYTHON_FILES) ipasetup.py
|
||||
@ # --ignore doubles speed of total test run compared to pytest.skip()
|
||||
@ # on module.
|
||||
PYTHONPATH=$(abspath $(top_srcdir)) $(PYTHON) ipatests/ipa-run-tests \
|
||||
PATH=$(abspath ipatests):$$PATH PYTHONPATH=$(abspath $(top_srcdir)) \
|
||||
$(PYTHON) ipatests/ipa-run-tests \
|
||||
--skip-ipaapi \
|
||||
--ignore $(abspath $(top_srcdir))/ipatests/test_integration \
|
||||
--ignore $(abspath $(top_srcdir))/ipatests/test_xmlrpc
|
||||
|
@ -137,6 +137,7 @@ jobs:
|
||||
- test_ipalib
|
||||
- test_ipaplatform
|
||||
- test_ipapython
|
||||
- test_ipatests_plugins
|
||||
testsToIgnore:
|
||||
- test_integration
|
||||
- test_webui
|
||||
|
@ -28,7 +28,8 @@ pytest_plugins = [
|
||||
'ipatests.pytest_ipa.beakerlib',
|
||||
'ipatests.pytest_ipa.declarative',
|
||||
'ipatests.pytest_ipa.nose_compat',
|
||||
'ipatests.pytest_ipa.integration'
|
||||
'ipatests.pytest_ipa.integration',
|
||||
'pytester',
|
||||
]
|
||||
|
||||
|
||||
|
@ -22,15 +22,13 @@
|
||||
|
||||
"""Pytest wrapper for running an installed (not in-tree) IPA test suite
|
||||
|
||||
Any command-line arguments are passed directly to py.test.
|
||||
The current directory is changed to the locaition of the ipatests package,
|
||||
so any relative paths given will be based on the ipatests module's path
|
||||
Any command-line arguments are passed directly to pytest.
|
||||
The current directory is changed to the location of the ipatests package,
|
||||
so any relative paths given will be based on the ipatests module's path.
|
||||
"""
|
||||
|
||||
import os
|
||||
import copy
|
||||
import sys
|
||||
import glob
|
||||
|
||||
import pytest
|
||||
|
||||
@ -41,82 +39,29 @@ os.environ['IPATEST_XUNIT_PATH'] = os.path.join(os.getcwd(), 'nosetests.xml')
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(ipatests.__file__))
|
||||
|
||||
def has_option(option):
|
||||
if option in sys.argv:
|
||||
return True
|
||||
for arg in sys.argv:
|
||||
for argi in arg.split("="):
|
||||
if option in argi:
|
||||
return True
|
||||
return False
|
||||
|
||||
class ArgsManglePlugin(object):
|
||||
"""Modify pytest arguments
|
||||
# don't override specified command line options
|
||||
if not has_option("confcutdir"):
|
||||
sys.argv.insert(1, "--confcutdir={}".format(HERE))
|
||||
# for backward compatibility
|
||||
if not has_option("cache_dir"):
|
||||
sys.argv[1:1] = ["-o", 'cache_dir={}'.format(os.path.join(os.getcwd(),
|
||||
".pytest_cache"))]
|
||||
|
||||
* Add confcutdir if hasn't been set already
|
||||
* Mangle paths to support tests both relative to basedir and ipatests/
|
||||
* Default to ipatests/ if no tests are specified
|
||||
"""
|
||||
def pytest_load_initial_conftests(self, early_config, parser, args):
|
||||
# During initial loading, parser supports only basic options
|
||||
ns = early_config.known_args_namespace
|
||||
if not ns.confcutdir:
|
||||
# add config cut directory to only load fixtures from ipatests/
|
||||
args.insert(0, '--confcutdir={}'.format(HERE))
|
||||
|
||||
if not ns.file_or_dir:
|
||||
# No file or directory found, run all tests
|
||||
args.append(HERE)
|
||||
else:
|
||||
vargs = []
|
||||
for name in ns.file_or_dir:
|
||||
idx = args.index(name)
|
||||
# split on pytest separator
|
||||
# ipatests/test_ipaplatform/test_importhook.py::test_override
|
||||
filename, sep, suffix = name.partition('::')
|
||||
# a file or directory relative to cwd or already absolute
|
||||
if os.path.exists(filename):
|
||||
continue
|
||||
if '*' in filename:
|
||||
# Expand a glob, we'll flatten the list later
|
||||
paths = glob.glob(os.path.join(HERE, filename))
|
||||
vargs.append([idx, paths])
|
||||
else:
|
||||
# a file or directory relative to ipatests package
|
||||
args[idx] = sep.join((os.path.join(HERE, filename), suffix))
|
||||
# flatten and insert all expanded file names
|
||||
base = 0
|
||||
for idx, items in vargs:
|
||||
args.pop(base + idx)
|
||||
for item in items:
|
||||
args.insert(base + idx, item)
|
||||
base += len(items)
|
||||
|
||||
# replace ignores, e.g. "--ignore test_integration" is changed to
|
||||
# "--ignore path/to/ipatests/test_integration"
|
||||
if ns.ignore:
|
||||
vargs = []
|
||||
for ignore in ns.ignore:
|
||||
idx = args.index(ignore)
|
||||
if os.path.exists(ignore):
|
||||
continue
|
||||
if '*' in ignore:
|
||||
# expand a glob, we'll flatten the list later
|
||||
paths = glob.glob(os.path.join(HERE, ignore))
|
||||
vargs.append([idx, paths])
|
||||
else:
|
||||
args[idx] = os.path.join(HERE, ignore)
|
||||
# flatten and insert all expanded file names
|
||||
base = 0
|
||||
for idx, items in vargs:
|
||||
# since we are expanding, remove old pair
|
||||
# --ignore and old file name
|
||||
args.pop(base + idx)
|
||||
args.pop(base + idx)
|
||||
for item in items:
|
||||
# careful: we need to add a pair
|
||||
# --ignore and new filename
|
||||
args.insert(base + idx, '--ignore')
|
||||
args.insert(base + idx, item)
|
||||
base += len(items) * 2 - 1
|
||||
|
||||
# rebuild early_config's known args with new args. The known args
|
||||
# are used for initial conftest.py from ipatests, which adds
|
||||
# additional arguments.
|
||||
early_config.known_args_namespace = parser.parse_known_args(
|
||||
args, namespace=copy.copy(early_config.option))
|
||||
|
||||
|
||||
sys.exit(pytest.main(plugins=[ArgsManglePlugin()]))
|
||||
pyt_args = [sys.executable, "-c",
|
||||
"import sys,pytest;sys.exit(pytest.main())"] + sys.argv[1:]
|
||||
# shell is needed to perform globbing
|
||||
sh_args = ["/bin/sh", "--norc", "--noprofile", "-c", "--"]
|
||||
pyt_args_esc = ["'{}'".format(x) if " " in x else x for x in pyt_args]
|
||||
args = sh_args + [" ".join(pyt_args_esc)]
|
||||
os.chdir(HERE)
|
||||
sys.stdout.flush()
|
||||
os.execv(args[0], args)
|
||||
|
@ -22,42 +22,32 @@ ipa\-run\-tests \- Run the FreeIPA test suite
|
||||
.SH "SYNOPSIS"
|
||||
ipa\-run\-tests [options]
|
||||
.SH "DESCRIPTION"
|
||||
ipa\-run\-tests is a wrapper around nosetests that run the FreeIPA test suite.
|
||||
ipa\-run\-tests is a wrapper around Pytest that run the FreeIPA test suite.
|
||||
It is intended to be used for developer testing and in continuous
|
||||
integration systems.
|
||||
|
||||
It loads IPA-internal Nose plugins ordered-tests and beakerlib.
|
||||
The ordered-tests plugin is enabled automatically.
|
||||
|
||||
The FreeIPA test suite installed system\-wide is selected via Nose's \-\-where
|
||||
option.
|
||||
It is possible to select a subset of the entire test suite by specifying
|
||||
a test file relative to the ipatests package, for example:
|
||||
|
||||
ipa-run-tests test_integration/test_simple_replication.py
|
||||
|
||||
.SH "OPTIONS"
|
||||
All command-line options are passed to the underlying Nose runner.
|
||||
See nosetests(1) for a complete list.
|
||||
|
||||
The internal IPA plugins add an extra option:
|
||||
|
||||
.TP
|
||||
\fB\-\-with-beakerlib\fR
|
||||
Enable BeakerLib integration.
|
||||
Test phases, failures and passes, and log messages are reported using
|
||||
beakerlib(1) commands.
|
||||
This option requires the beakerlib.sh script to be sourced.
|
||||
All command-line options are passed to the underlying Pytest runner.
|
||||
See "pytest --help" for a complete list.
|
||||
|
||||
.SH "EXIT STATUS"
|
||||
0 if the command was successful
|
||||
|
||||
nonzero if any error or failure occurred
|
||||
Running pytest can result in six different exit codes:
|
||||
Exit code 0: All tests were collected and passed successfully
|
||||
Exit code 1: Tests were collected and run but some of the tests failed
|
||||
Exit code 2: Test execution was interrupted by the user
|
||||
Exit code 3: Internal error happened while executing tests
|
||||
Exit code 4: pytest command line usage error
|
||||
Exit code 5: No tests were collected
|
||||
|
||||
.SH "CONFIGURATION"
|
||||
Please see ipa-test-config(1) for a description of configuration environment
|
||||
variables.
|
||||
|
||||
.SH "REFERENCES"
|
||||
A full description of the FreeIPA integration testing framework is available at
|
||||
http://www.freeipa.org/page/V3/Integration_testing
|
||||
A full description of the FreeIPA testing is available at
|
||||
https://www.freeipa.org/page/Testing
|
||||
|
@ -44,6 +44,7 @@ if __name__ == '__main__':
|
||||
"ipatests.test_ipapython",
|
||||
"ipatests.test_ipaserver",
|
||||
"ipatests.test_ipaserver.test_install",
|
||||
"ipatests.test_ipatests_plugins",
|
||||
"ipatests.test_webui",
|
||||
"ipatests.test_xmlrpc",
|
||||
"ipatests.test_xmlrpc.tracker"
|
||||
|
7
ipatests/test_ipatests_plugins/__init__.py
Normal file
7
ipatests/test_ipatests_plugins/__init__.py
Normal file
@ -0,0 +1,7 @@
|
||||
#
|
||||
# Copyright (C) 2019 FreeIPA Contributors see COPYING for license
|
||||
#
|
||||
|
||||
"""
|
||||
Sub-package containing unit tests for IPA internal test plugins
|
||||
"""
|
162
ipatests/test_ipatests_plugins/test_ipa_run_tests.py
Normal file
162
ipatests/test_ipatests_plugins/test_ipa_run_tests.py
Normal file
@ -0,0 +1,162 @@
|
||||
#
|
||||
# Copyright (C) 2019 FreeIPA Contributors see COPYING for license
|
||||
#
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
MOD_NAME = "test_module_{}"
|
||||
FUNC_NAME = "test_func_{}"
|
||||
MODS_NUM = 5
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ipatestdir(testdir, monkeypatch):
|
||||
"""
|
||||
Create MODS_NUM test modules within testdir/ipatests.
|
||||
Each module contains 1 test function.
|
||||
Patch PYTHONPATH with created package path to override the system's
|
||||
ipatests
|
||||
"""
|
||||
ipatests_dir = testdir.mkpydir("ipatests")
|
||||
for i in range(MODS_NUM):
|
||||
ipatests_dir.join("{}.py".format(MOD_NAME.format(i))).write(
|
||||
"def {}(): pass".format(FUNC_NAME.format(i)))
|
||||
|
||||
python_path = os.pathsep.join(
|
||||
filter(None, [str(testdir.tmpdir), os.environ.get("PYTHONPATH", "")]))
|
||||
monkeypatch.setenv("PYTHONPATH", python_path)
|
||||
|
||||
def run_ipa_tests(*args):
|
||||
cmdargs = ["ipa-run-tests", "-v"] + list(args)
|
||||
return testdir.run(*cmdargs, timeout=60)
|
||||
|
||||
testdir.run_ipa_tests = run_ipa_tests
|
||||
return testdir
|
||||
|
||||
|
||||
def test_ipa_run_tests_basic(ipatestdir):
|
||||
"""
|
||||
Run ipa-run-tests with default arguments
|
||||
"""
|
||||
result = ipatestdir.run_ipa_tests()
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=MODS_NUM)
|
||||
for mod_num in range(MODS_NUM):
|
||||
result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num))])
|
||||
|
||||
|
||||
def test_ipa_run_tests_glob1(ipatestdir):
|
||||
"""
|
||||
Run ipa-run-tests using glob patterns to collect tests
|
||||
"""
|
||||
result = ipatestdir.run_ipa_tests("{mod}".format(
|
||||
mod="test_modul[!E]?[0-5]*"))
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=MODS_NUM)
|
||||
for mod_num in range(MODS_NUM):
|
||||
result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num))])
|
||||
|
||||
|
||||
def test_ipa_run_tests_glob2(ipatestdir):
|
||||
"""
|
||||
Run ipa-run-tests using glob patterns to collect tests
|
||||
"""
|
||||
result = ipatestdir.run_ipa_tests("{mod}".format(
|
||||
mod="test_module_{0,1}*"))
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=2)
|
||||
for mod_num in range(2):
|
||||
result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num))])
|
||||
|
||||
|
||||
def test_ipa_run_tests_specific_nodeid(ipatestdir):
|
||||
"""
|
||||
Run ipa-run-tests using nodeid to collect test
|
||||
"""
|
||||
mod_num = 0
|
||||
result = ipatestdir.run_ipa_tests("{mod}.py::{func}".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num)))
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=1)
|
||||
result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num))])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expr",
|
||||
[["-k", "not {func}".format(func=FUNC_NAME.format(0))],
|
||||
["-k not {func}".format(func=FUNC_NAME.format(0))]])
|
||||
def test_ipa_run_tests_expression(ipatestdir, expr):
|
||||
"""
|
||||
Run ipa-run-tests using expression
|
||||
"""
|
||||
result = ipatestdir.run_ipa_tests(*expr)
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=4)
|
||||
for mod_num in range(1, MODS_NUM):
|
||||
result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num))])
|
||||
|
||||
|
||||
def test_ipa_run_tests_ignore_basic(ipatestdir):
|
||||
"""
|
||||
Run ipa-run-tests ignoring one test module
|
||||
"""
|
||||
result = ipatestdir.run_ipa_tests(
|
||||
"--ignore", "{mod}.py".format(mod=MOD_NAME.format(0)),
|
||||
"--ignore", "{mod}.py".format(mod=MOD_NAME.format(1)),
|
||||
)
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=MODS_NUM - 2)
|
||||
for mod_num in range(2, MODS_NUM):
|
||||
result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num))])
|
||||
|
||||
|
||||
def test_ipa_run_tests_defaultargs(ipatestdir):
|
||||
"""
|
||||
Checking the ipa-run-tests defaults:
|
||||
* cachedir
|
||||
* rootdir
|
||||
"""
|
||||
mod_num = 0
|
||||
result = ipatestdir.run_ipa_tests("{mod}.py::{func}".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num)))
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=1)
|
||||
result.stdout.re_match_lines([
|
||||
"^cachedir: {cachedir}$".format(
|
||||
cachedir=os.path.join(os.getcwd(), ".pytest_cache")),
|
||||
"^rootdir: {rootdir}([,].*)?$".format(
|
||||
rootdir=os.path.join(str(ipatestdir.tmpdir), "ipatests"))
|
||||
])
|
||||
|
||||
|
||||
def test_ipa_run_tests_confcutdir(ipatestdir):
|
||||
"""
|
||||
Checking the ipa-run-tests defaults:
|
||||
* confcutdir
|
||||
"""
|
||||
mod_num = 0
|
||||
ipatestdir.makeconftest("import somenotexistedpackage")
|
||||
result = ipatestdir.run_ipa_tests("{mod}.py::{func}".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num)))
|
||||
assert result.ret == 0
|
||||
result.assert_outcomes(passed=1)
|
||||
result.stdout.fnmatch_lines(["*{mod}.py::{func} PASSED*".format(
|
||||
mod=MOD_NAME.format(mod_num),
|
||||
func=FUNC_NAME.format(mod_num))])
|
Loading…
Reference in New Issue
Block a user