Add marker needs_ipaapi and option to skip tests

The new marker needs_ipaapi is used to mark tests that needs an
initialized API (ipalib.api) or some sort of other API services (running
LDAP server) to work. Some packages use api.Command or api.Backend on
module level. They are not marked but rather skipped entirely.

A new option ``skip-ipaapi`` is added to skip all API based tests. With
the option, only simple unit tests are executed. As of now, freeIPA
contains more than 500 unit tests that can be executed in about 5
seconds.

Signed-off-by: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>

ipatests/conftest.py

@@ -41,6 +41,7 @@ MARKERS = [
     'cs_acceptance: Acceptance test suite for Dogtag Certificate Server',
     'ds_acceptance: Acceptance test suite for 389 Directory Server',
     'skip_ipaclient_unittest: Skip in ipaclient unittest mode',
+    'needs_ipaapi: Test needs IPA API',
 ]
 
 
@@ -95,6 +96,11 @@ def pytest_addoption(parser):
         help='Run ipaclient unit tests only (no RPC and ipaserver)',
         action='store_true'
     )
+    group.addoption(
+        '--skip-ipaapi',
+        help='Do not run tests that depends on IPA API',
+        action='store_true',
+    )
 
 
 def pytest_cmdline_main(config):
@@ -124,3 +130,7 @@ def pytest_runtest_setup(item):
             # pylint: disable=no-member
             if pytest.config.option.ipaclient_unittests:
                 pytest.skip("Skip in ipaclient unittest mode")
+        if item.get_marker('needs_ipaapi'):
+            # pylint: disable=no-member
+            if pytest.config.option.skip_ipaapi:
+                pytest.skip("Skip tests that needs an IPA API")

ipatests/test_cmdline/test_cli.py

@@ -20,7 +20,9 @@ TEST_ZONE = u'zoneadd.%(domain)s' % api.env
 HERE = os.path.abspath(os.path.dirname(__file__))
 BASE_DIR = os.path.abspath(os.path.join(HERE, os.pardir, os.pardir))
 
+
 @pytest.mark.tier0
+@pytest.mark.needs_ipaapi
 class TestCLIParsing(object):
     """Tests that commandlines are correctly parsed to Command keyword args
     """

ipatests/test_cmdline/test_help.py

@@ -30,6 +30,9 @@ if six.PY3:
     unicode = str
 
 
+pytestmark = pytest.mark.needs_ipaapi
+
+
 @pytest.mark.tier0
 class CLITestContext(object):
     """Context manager that replaces stdout & stderr, and catches SystemExit

ipatests/test_cmdline/test_ipagetkeytab.py

@@ -67,6 +67,7 @@ def test_service(request, test_host):
     return service_tracker.make_fixture(request)
 
 
+@pytest.mark.needs_ipaapi
 class KeytabRetrievalTest(cmdline_test):
     """
     Base class for keytab retrieval tests

ipatests/test_install/test_updates.py

@@ -48,6 +48,7 @@ The DM password needs to be set in ~/.ipa/.dmpw
 
 
 @pytest.mark.tier0
+@pytest.mark.needs_ipaapi
 class test_update(unittest.TestCase):
     """
     Test the LDAP updater.

ipatests/test_integration/__init__.py

@@ -20,3 +20,4 @@ import ipatests.util
 
 
 ipatests.util.check_ipaclient_unittests()
+ipatests.util.check_no_ipaapi()

ipatests/test_ipalib/test_rpc.py

@@ -260,6 +260,7 @@ class test_xmlclient(PluginTester):
 
 
 @pytest.mark.skip_ipaclient_unittest
+@pytest.mark.needs_ipaapi
 class test_xml_introspection(object):
     @classmethod
     def setup_class(cls):
@@ -346,6 +347,7 @@ class test_xml_introspection(object):
 
 
 @pytest.mark.skip_ipaclient_unittest
+@pytest.mark.needs_ipaapi
 class test_rpcclient_context(PluginTester):
     """
     Test the context in `ipalib.rpc.rpcclient` plugin.

ipatests/test_ipapython/test_session_storage.py

@@ -11,6 +11,7 @@ from ipapython import session_storage
 
 
 @pytest.mark.skip_ipaclient_unittest
+@pytest.mark.needs_ipaapi
 class test_session_storage(object):
     """
     Test the session storage interface

ipatests/test_ipaserver/test_ldap.py

@@ -43,6 +43,7 @@ if six.PY3:
 
 
 @pytest.mark.tier0
+@pytest.mark.needs_ipaapi
 class test_ldap(object):
     """
     Test various LDAP client bind methods.
@@ -135,6 +136,7 @@ class test_ldap(object):
 
 
 @pytest.mark.tier0
+@pytest.mark.needs_ipaapi
 class test_LDAPEntry(object):
     """
     Test the LDAPEntry class

ipatests/test_ipaserver/test_serverroles.py

@@ -15,6 +15,8 @@ from ipaplatform.paths import paths
 from ipalib import api, create_api, errors
 from ipapython.dn import DN
 
+pytestmark = pytest.mark.needs_ipaapi
+
 
 def _make_service_entry(ldap_backend, dn, enabled=True, other_config=None):
     mods = {

ipatests/test_webui/__init__.py

@@ -24,3 +24,4 @@ import ipatests.util
 
 
 ipatests.util.check_ipaclient_unittests()
+ipatests.util.check_no_ipaapi()  # also ignore in make fasttest

ipatests/test_webui/ui_driver.py

@@ -59,6 +59,7 @@ except ImportError:
     NO_YAML = True
 from ipaplatform.paths import paths
 
+
 ENV_MAP = {
     'MASTER': 'ipa_server',
     'ADMINID': 'ipa_admin',

ipatests/test_xmlrpc/__init__.py

@@ -24,3 +24,4 @@ import ipatests.util
 
 
 ipatests.util.check_ipaclient_unittests()
+ipatests.util.check_no_ipaapi()  # also ignore in make fasttest

ipatests/util.py

@@ -78,6 +78,18 @@ def check_ipaclient_unittests(reason="Skip in ipaclient unittest mode"):
             raise pytest.skip(reason)
 
 
+def check_no_ipaapi(reason="Skip tests that needs an IPA API"):
+    """Call this in a package to skip the package in no-ipaapi mode
+    """
+    if pytest.config.getoption('skip_ipaapi', False):
+        if PYTEST_VERSION[0] >= 3:
+            # pylint: disable=unexpected-keyword-arg
+            raise pytest.skip.Exception(reason, allow_module_level=True)
+            # pylint: enable=unexpected-keyword-arg
+        else:
+            raise pytest.skip(reason)
+
+
 class TempDir(object):
     def __init__(self):
         self.__path = tempfile.mkdtemp(prefix='ipa.tests.')