pytest: Warn about unittest/nose/xunit tests

This Pytest plugin is intended to issue warnings on collecting
tests, which employ unittest/nose frameworks or xunit style.
For example, this may look like:
"""
test_a/test_xunit.py:25
  test_a/test_xunit.py:25: PytestDeprecationWarning: xunit style is deprecated

    def test_foo_bar(self):

test_b/test_unittest.py:7
  test_b/test_unittest.py:7: PytestDeprecationWarning: unittest is deprecated
    def test_foo_bar(self):
"""

To treat these warnings as errors it's enough to run Pytest with:
-W error:'xunit style is deprecated':pytest.PytestDeprecationWarning

Related: https://pagure.io/freeipa/issue/7989
Signed-off-by: Stanislav Levin <slev@altlinux.org>
Reviewed-By: Christian Heimes <cheimes@redhat.com>

ipatests/conftest.py

@@ -25,6 +25,7 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 
 pytest_plugins = [
     'ipatests.pytest_ipa.additional_config',
+    'ipatests.pytest_ipa.deprecated_frameworks',
     'ipatests.pytest_ipa.slicing',
     'ipatests.pytest_ipa.beakerlib',
     'ipatests.pytest_ipa.declarative',

ipatests/pytest_ipa/deprecated_frameworks.py

@@ -0,0 +1,66 @@
+#
+# Copyright (C) 2019  FreeIPA Contributors see COPYING for license
+#
+
+"""Warns about xunit/unittest/nose tests.
+
+FreeIPA is a rather old project and hereby includes all the most
+famous in the past and present Python test idioms. Of course,
+this is difficult to play around all of them. For now, the runner
+of the IPA's test suite is Pytest.
+
+Pytest supports xunit style setups, unittest, nose tests. But this
+support is limited and may be dropped in the future releases.
+Worst of all is that the mixing of various test frameworks results
+in weird conflicts and of course, is not widely tested. In other
+words, there is a big risk. To eliminate this risk and to not pin
+Pytest to 3.x branch IPA's tests were refactored.
+
+This plugin is intended to issue warnings on collecting tests,
+which employ unittest/nose frameworks or xunit style.
+
+To treat these warnings as errors it's enough to run Pytest with:
+
+-W error:'xunit style is deprecated':pytest.PytestDeprecationWarning
+
+"""
+from unittest import TestCase
+
+import pytest
+
+forbidden_module_scopes = [
+    'setup_module',
+    'setup_function',
+    'teardown_module',
+    'teardown_function',
+]
+
+forbidden_class_scopes = [
+    'setup_class',
+    'setup_method',
+    'teardown_class',
+    'teardown_method',
+]
+
+
+def pytest_collection_finish(session):
+    for item in session.items:
+        cls = getattr(item, 'cls', None)
+        if cls is not None and issubclass(cls, TestCase):
+            item.warn(pytest.PytestDeprecationWarning(
+                "unittest is deprecated in favour of fixtures style"))
+            continue
+
+        def xunit_depr_warn(item, attr, names):
+            for n in names:
+                obj = getattr(item, attr, None)
+                method = getattr(obj, n, None)
+                fixtured = hasattr(method, '__pytest_wrapped__')
+                if method is not None and not fixtured:
+                    item.warn(
+                        pytest.PytestDeprecationWarning(
+                            "xunit style is deprecated in favour of "
+                            "fixtures style"))
+
+        xunit_depr_warn(item, 'module', forbidden_module_scopes)
+        xunit_depr_warn(item, 'cls', forbidden_class_scopes)

ipatests/test_ipatests_plugins/test_depr_frameworks.py

@@ -0,0 +1,99 @@
+#
+# Copyright (C) 2019  FreeIPA Contributors see COPYING for license
+#
+
+import pytest
+
+
+@pytest.fixture
+def ipa_testdir(testdir):
+    """
+    Create conftest within testdir.
+    """
+    testdir.makeconftest(
+        """
+        pytest_plugins = ["ipatests.pytest_ipa.deprecated_frameworks"]
+        """
+    )
+    return testdir
+
+
+@pytest.fixture
+def xunit_testdir(ipa_testdir):
+    """
+    Create xnit style test module within testdir.
+    """
+    ipa_testdir.makepyfile("""
+        def setup_module():
+            pass
+        def teardown_module():
+            pass
+        def setup_function():
+            pass
+        def teardown_function():
+            pass
+
+        class TestClass:
+            @classmethod
+            def setup_class(cls):
+                pass
+            @classmethod
+            def teardown_class(cls):
+                pass
+            def setup_method(self):
+                pass
+            def teardown_method(self):
+                pass
+            def test_m(self):
+                pass
+        """)
+    return ipa_testdir
+
+
+@pytest.fixture
+def unittest_testdir(ipa_testdir):
+    """
+    Create unittest style test module within testdir.
+    """
+    ipa_testdir.makepyfile("""
+        import unittest
+        def setUpModule():
+            pass
+        def tearDownModule():
+            pass
+        class TestClass(unittest.TestCase):
+            @classmethod
+            def setUp(self):
+                pass
+            def tearDown(self):
+                pass
+            @classmethod
+            def setUpClass(cls):
+                pass
+            @classmethod
+            def tearDownClass(cls):
+                pass
+            def test_m(self):
+                pass
+        """)
+    return ipa_testdir
+
+
+def test_xunit(xunit_testdir):
+    result = xunit_testdir.runpytest()
+    result.assert_outcomes(passed=1)
+    result.stdout.fnmatch_lines([
+        "* PytestDeprecationWarning: xunit style is deprecated in favour of "
+        "fixtures style",
+        "* 8 warning*",
+    ])
+
+
+def test_unittest(unittest_testdir):
+    result = unittest_testdir.runpytest()
+    result.assert_outcomes(passed=1)
+    result.stdout.fnmatch_lines([
+        "* PytestDeprecationWarning: unittest is deprecated in favour of "
+        "fixtures style",
+        "* 1 warning*",
+    ])