Merge pull request #5307 from akaihola/5273_doctest_conditional_skip

Feature: skip doctests conditionally

doc/usage/extensions/doctest.rst

@@ -198,6 +198,81 @@ The following is an example for the usage of the directives.  The test via
       This parrot wouldn't voom if you put 3000 volts through it!
 
 
+Skipping tests conditionally
+----------------------------
+
+``skipif``, a string option, can be used to skip directives conditionally. This
+may be useful e.g. when a different set of tests should be run depending on the
+environment (hardware, network/VPN, optional dependencies or different versions
+of dependencies). The ``skipif`` option is supported by all of the doctest
+directives. Below are typical use cases for ``skipif`` when used for different
+directives:
+
+- :rst:dir:`testsetup` and :rst:dir:`testcleanup`
+
+  - conditionally skip test setup and/or cleanup
+  - customize setup/cleanup code per environment
+
+- :rst:dir:`doctest`
+
+  - conditionally skip both a test and its output verification
+
+- :rst:dir:`testcode`
+
+  - conditionally skip a test
+  - customize test code per environment
+
+- :rst:dir:`testoutput`
+
+  - conditionally skip output assertion for a skipped test
+  - expect different output depending on the environment
+
+The value of the ``skipif`` option is evaluated as a Python expression. If the
+result is a true value, the directive is omitted from the test run just as if
+it wasn't present in the file at all.
+
+Instead of repeating an expression, the :confval:`doctest_global_setup`
+configuration option can be used to assign it to a variable which can then be
+used instead.
+
+Here's an example which skips some tests if Pandas is not installed:
+
+.. code-block:: py
+   :caption: conf.py
+
+   extensions = ['sphinx.ext.doctest']
+   doctest_global_setup = '''
+   try:
+       import pandas as pd
+   except ImportError:
+       pd = None
+   '''
+
+.. code-block:: rst
+   :caption: contents.rst
+
+   .. testsetup::
+      :skipif: pd is None
+
+      data = pd.Series([42])
+
+   .. doctest::
+      :skipif: pd is None
+
+      >>> data.iloc[0]
+      42
+
+   .. testcode::
+      :skipif: pd is None
+
+      print(data.iloc[-1])
+
+   .. testoutput::
+      :skipif: pd is None
+
+      42
+
+
 Configuration
 -------------
 

sphinx/ext/doctest.py

@@ -90,6 +90,16 @@ class TestDirective(SphinxDirective):
 
     def run(self):
         # type: () -> List[nodes.Node]
+        if 'skipif' in self.options:
+            condition = self.options['skipif']
+            context = {}  # type: Dict[str, Any]
+            if self.config.doctest_global_setup:
+                exec(self.config.doctest_global_setup, context)
+            should_skip = eval(condition, context)
+            if self.config.doctest_global_cleanup:
+                exec(self.config.doctest_global_cleanup, context)
+            if should_skip:
+                return []
         # use ordinary docutils nodes for test code: they get special attributes
         # so that our builder recognizes them, and the other builders are happy.
         code = '\n'.join(self.content)
@@ -155,11 +165,11 @@ class TestDirective(SphinxDirective):
 
 
 class TestsetupDirective(TestDirective):
-    option_spec = {}  # type: Dict
+    option_spec = {'skipif': directives.unchanged_required}  # type: Dict
 
 
 class TestcleanupDirective(TestDirective):
-    option_spec = {}  # type: Dict
+    option_spec = {'skipif': directives.unchanged_required}  # type: Dict
 
 
 class DoctestDirective(TestDirective):
@@ -167,6 +177,7 @@ class DoctestDirective(TestDirective):
         'hide': directives.flag,
         'options': directives.unchanged,
         'pyversion': directives.unchanged_required,
+        'skipif': directives.unchanged_required,
     }
 
 
@@ -174,6 +185,7 @@ class TestcodeDirective(TestDirective):
     option_spec = {
         'hide': directives.flag,
         'pyversion': directives.unchanged_required,
+        'skipif': directives.unchanged_required,
     }
 
 
@@ -182,6 +194,7 @@ class TestoutputDirective(TestDirective):
         'hide': directives.flag,
         'options': directives.unchanged,
         'pyversion': directives.unchanged_required,
+        'skipif': directives.unchanged_required,
     }
 
 

tests/roots/test-ext-doctest-skipif/conf.py

@@ -0,0 +1,16 @@
+extensions = ['sphinx.ext.doctest']
+
+project = 'test project for the doctest :skipif: directive'
+master_doc = 'skipif'
+source_suffix = '.txt'
+exclude_patterns = ['_build']
+
+doctest_global_setup = '''
+from test_ext_doctest import record
+
+record('doctest_global_setup', 'body', True)
+'''
+
+doctest_global_cleanup = '''
+record('doctest_global_cleanup', 'body', True)
+'''

tests/roots/test-ext-doctest-skipif/skipif.txt

@@ -0,0 +1,81 @@
+Testing the doctest extension's `:skipif:` option
+=================================================
+
+testsetup
+---------
+
+.. testsetup:: group-skipif
+   :skipif: record('testsetup', ':skipif:', True) != 'this will be True'
+
+   record('testsetup', 'body', True)
+
+.. testsetup:: group-skipif
+   :skipif: record('testsetup', ':skipif:', False) == 'this will be False'
+
+   record('testsetup', 'body', False)
+
+
+doctest
+-------
+.. doctest:: group-skipif
+   :skipif: record('doctest', ':skipif:', True) != 'this will be True'
+
+   >>> print(record('doctest', 'body', True))
+   The test is skipped, and this expected text is ignored
+
+
+.. doctest::
+   :skipif: record('doctest', ':skipif:', False) == 'this will be False'
+
+   >>> print(record('doctest', 'body', False))
+   Recorded doctest body False
+
+
+testcode and testoutput
+-----------------------
+
+testcode skipped
+~~~~~~~~~~~~~~~~
+
+.. testcode:: group-skipif
+   :skipif: record('testcode', ':skipif:', True) != 'this will be True'
+
+   print(record('testcode', 'body', True))
+
+.. testoutput:: group-skipif
+   :skipif: record('testoutput-1', ':skipif:', True) != 'this will be True'
+
+   The previous testcode is skipped, and the :skipif: condition is True,
+   so this testoutput is ignored
+
+testcode executed
+~~~~~~~~~~~~~~~~~
+
+.. testcode:: group-skipif
+   :skipif: record('testcode', ':skipif:', False) == 'this will be False'
+
+   print(record('testcode', 'body', False))
+
+.. testoutput:: group-skipif
+   :skipif: record('testoutput-2', ':skipif:', False) == 'this will be False'
+
+   Recorded testcode body False
+
+.. testoutput:: group-skipif
+   :skipif: record('testoutput-2', ':skipif:', True) != 'this will be True'
+
+   The :skipif: condition is False, so this testoutput is ignored
+
+
+testcleanup
+-----------
+
+.. testcleanup:: group-skipif
+   :skipif: record('testcleanup', ':skipif:', True) != 'this will be True'
+
+   record('testcleanup', 'body', True)
+
+.. testcleanup:: group-skipif
+   :skipif: record('testcleanup', ':skipif:', False) == 'this will be False'
+
+   record('testcleanup', 'body', False)

tests/test_ext_doctest.py

@@ -9,6 +9,7 @@
     :license: BSD, see LICENSE for details.
 """
 import os
+from collections import Counter
 
 import pytest
 from packaging.specifiers import InvalidSpecifier
@@ -61,6 +62,56 @@ def cleanup_call():
     cleanup_called += 1
 
 
+recorded_calls = Counter()
+
+
+@pytest.mark.sphinx('doctest', testroot='ext-doctest-skipif')
+def test_skipif(app, status, warning):
+    """Tests for the :skipif: option
+
+    The tests are separated into a different test root directory since the
+    ``app`` object only evaluates options once in its lifetime. If these tests
+    were combined with the other doctest tests, the ``:skipif:`` evaluations
+    would be recorded only on the first ``app.builder.build_all()`` run, i.e.
+    in ``test_build`` above, and the assertion below would fail.
+
+    """
+    global recorded_calls
+    recorded_calls = Counter()
+    app.builder.build_all()
+    if app.statuscode != 0:
+        assert False, 'failures in doctests:' + status.getvalue()
+    # The `:skipif:` expressions are always run.
+    # Actual tests and setup/cleanup code is only run if the `:skipif:`
+    # expression evaluates to a False value.
+    # Global setup/cleanup are run before/after evaluating the `:skipif:`
+    # option in each directive - thus 11 additional invocations for each on top
+    # of the ones made for the whole test file.
+    assert recorded_calls == {('doctest_global_setup', 'body', True): 13,
+                              ('testsetup', ':skipif:', True): 1,
+                              ('testsetup', ':skipif:', False): 1,
+                              ('testsetup', 'body', False): 1,
+                              ('doctest', ':skipif:', True): 1,
+                              ('doctest', ':skipif:', False): 1,
+                              ('doctest', 'body', False): 1,
+                              ('testcode', ':skipif:', True): 1,
+                              ('testcode', ':skipif:', False): 1,
+                              ('testcode', 'body', False): 1,
+                              ('testoutput-1', ':skipif:', True): 1,
+                              ('testoutput-2', ':skipif:', True): 1,
+                              ('testoutput-2', ':skipif:', False): 1,
+                              ('testcleanup', ':skipif:', True): 1,
+                              ('testcleanup', ':skipif:', False): 1,
+                              ('testcleanup', 'body', False): 1,
+                              ('doctest_global_cleanup', 'body', True): 13}
+
+
+def record(directive, part, should_skip):
+    global recorded_calls
+    recorded_calls[(directive, part, should_skip)] += 1
+    return 'Recorded {} {} {}'.format(directive, part, should_skip)
+
+
 @pytest.mark.xfail(
     PY2, reason='node.source points to document instead of filename',
 )