diff --git a/doc/usage/extensions/doctest.rst b/doc/usage/extensions/doctest.rst index 341b7a9a7..e321e3a54 100644 --- a/doc/usage/extensions/doctest.rst +++ b/doc/usage/extensions/doctest.rst @@ -198,6 +198,81 @@ The following is an example for the usage of the directives. The test via This parrot wouldn't voom if you put 3000 volts through it! +Skipping tests conditionally +---------------------------- + +``skipif``, a string option, can be used to skip directives conditionally. This +may be useful e.g. when a different set of tests should be run depending on the +environment (hardware, network/VPN, optional dependencies or different versions +of dependencies). The ``skipif`` option is supported by all of the doctest +directives. Below are typical use cases for ``skipif`` when used for different +directives: + +- :rst:dir:`testsetup` and :rst:dir:`testcleanup` + + - conditionally skip test setup and/or cleanup + - customize setup/cleanup code per environment + +- :rst:dir:`doctest` + + - conditionally skip both a test and its output verification + +- :rst:dir:`testcode` + + - conditionally skip a test + - customize test code per environment + +- :rst:dir:`testoutput` + + - conditionally skip output assertion for a skipped test + - expect different output depending on the environment + +The value of the ``skipif`` option is evaluated as a Python expression. If the +result is a true value, the directive is omitted from the test run just as if +it wasn't present in the file at all. + +Instead of repeating an expression, the :confval:`doctest_global_setup` +configuration option can be used to assign it to a variable which can then be +used instead. + +Here's an example which skips some tests if Pandas is not installed: + +.. code-block:: py + :caption: conf.py + + extensions = ['sphinx.ext.doctest'] + doctest_global_setup = ''' + try: + import pandas as pd + except ImportError: + pd = None + ''' + +.. code-block:: rst + :caption: contents.rst + + .. testsetup:: + :skipif: pd is None + + data = pd.Series([42]) + + .. doctest:: + :skipif: pd is None + + >>> data.iloc[0] + 42 + + .. testcode:: + :skipif: pd is None + + print(data.iloc[-1]) + + .. testoutput:: + :skipif: pd is None + + 42 + + Configuration ------------- diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py index 68877371c..fa6cbed6d 100644 --- a/sphinx/ext/doctest.py +++ b/sphinx/ext/doctest.py @@ -90,6 +90,16 @@ class TestDirective(SphinxDirective): def run(self): # type: () -> List[nodes.Node] + if 'skipif' in self.options: + condition = self.options['skipif'] + context = {} # type: Dict[str, Any] + if self.config.doctest_global_setup: + exec(self.config.doctest_global_setup, context) + should_skip = eval(condition, context) + if self.config.doctest_global_cleanup: + exec(self.config.doctest_global_cleanup, context) + if should_skip: + return [] # use ordinary docutils nodes for test code: they get special attributes # so that our builder recognizes them, and the other builders are happy. code = '\n'.join(self.content) @@ -155,11 +165,11 @@ class TestDirective(SphinxDirective): class TestsetupDirective(TestDirective): - option_spec = {} # type: Dict + option_spec = {'skipif': directives.unchanged_required} # type: Dict class TestcleanupDirective(TestDirective): - option_spec = {} # type: Dict + option_spec = {'skipif': directives.unchanged_required} # type: Dict class DoctestDirective(TestDirective): @@ -167,6 +177,7 @@ class DoctestDirective(TestDirective): 'hide': directives.flag, 'options': directives.unchanged, 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, } @@ -174,6 +185,7 @@ class TestcodeDirective(TestDirective): option_spec = { 'hide': directives.flag, 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, } @@ -182,6 +194,7 @@ class TestoutputDirective(TestDirective): 'hide': directives.flag, 'options': directives.unchanged, 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, } diff --git a/tests/roots/test-ext-doctest-skipif/conf.py b/tests/roots/test-ext-doctest-skipif/conf.py new file mode 100644 index 000000000..dd94b7049 --- /dev/null +++ b/tests/roots/test-ext-doctest-skipif/conf.py @@ -0,0 +1,16 @@ +extensions = ['sphinx.ext.doctest'] + +project = 'test project for the doctest :skipif: directive' +master_doc = 'skipif' +source_suffix = '.txt' +exclude_patterns = ['_build'] + +doctest_global_setup = ''' +from test_ext_doctest import record + +record('doctest_global_setup', 'body', True) +''' + +doctest_global_cleanup = ''' +record('doctest_global_cleanup', 'body', True) +''' \ No newline at end of file diff --git a/tests/roots/test-ext-doctest-skipif/skipif.txt b/tests/roots/test-ext-doctest-skipif/skipif.txt new file mode 100644 index 000000000..c5bd39821 --- /dev/null +++ b/tests/roots/test-ext-doctest-skipif/skipif.txt @@ -0,0 +1,81 @@ +Testing the doctest extension's `:skipif:` option +================================================= + +testsetup +--------- + +.. testsetup:: group-skipif + :skipif: record('testsetup', ':skipif:', True) != 'this will be True' + + record('testsetup', 'body', True) + +.. testsetup:: group-skipif + :skipif: record('testsetup', ':skipif:', False) == 'this will be False' + + record('testsetup', 'body', False) + + +doctest +------- +.. doctest:: group-skipif + :skipif: record('doctest', ':skipif:', True) != 'this will be True' + + >>> print(record('doctest', 'body', True)) + The test is skipped, and this expected text is ignored + + +.. doctest:: + :skipif: record('doctest', ':skipif:', False) == 'this will be False' + + >>> print(record('doctest', 'body', False)) + Recorded doctest body False + + +testcode and testoutput +----------------------- + +testcode skipped +~~~~~~~~~~~~~~~~ + +.. testcode:: group-skipif + :skipif: record('testcode', ':skipif:', True) != 'this will be True' + + print(record('testcode', 'body', True)) + +.. testoutput:: group-skipif + :skipif: record('testoutput-1', ':skipif:', True) != 'this will be True' + + The previous testcode is skipped, and the :skipif: condition is True, + so this testoutput is ignored + +testcode executed +~~~~~~~~~~~~~~~~~ + +.. testcode:: group-skipif + :skipif: record('testcode', ':skipif:', False) == 'this will be False' + + print(record('testcode', 'body', False)) + +.. testoutput:: group-skipif + :skipif: record('testoutput-2', ':skipif:', False) == 'this will be False' + + Recorded testcode body False + +.. testoutput:: group-skipif + :skipif: record('testoutput-2', ':skipif:', True) != 'this will be True' + + The :skipif: condition is False, so this testoutput is ignored + + +testcleanup +----------- + +.. testcleanup:: group-skipif + :skipif: record('testcleanup', ':skipif:', True) != 'this will be True' + + record('testcleanup', 'body', True) + +.. testcleanup:: group-skipif + :skipif: record('testcleanup', ':skipif:', False) == 'this will be False' + + record('testcleanup', 'body', False) diff --git a/tests/test_ext_doctest.py b/tests/test_ext_doctest.py index 7137af31b..f7b6ca5f8 100644 --- a/tests/test_ext_doctest.py +++ b/tests/test_ext_doctest.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for details. """ import os +from collections import Counter import pytest from packaging.specifiers import InvalidSpecifier @@ -61,6 +62,56 @@ def cleanup_call(): cleanup_called += 1 +recorded_calls = Counter() + + +@pytest.mark.sphinx('doctest', testroot='ext-doctest-skipif') +def test_skipif(app, status, warning): + """Tests for the :skipif: option + + The tests are separated into a different test root directory since the + ``app`` object only evaluates options once in its lifetime. If these tests + were combined with the other doctest tests, the ``:skipif:`` evaluations + would be recorded only on the first ``app.builder.build_all()`` run, i.e. + in ``test_build`` above, and the assertion below would fail. + + """ + global recorded_calls + recorded_calls = Counter() + app.builder.build_all() + if app.statuscode != 0: + assert False, 'failures in doctests:' + status.getvalue() + # The `:skipif:` expressions are always run. + # Actual tests and setup/cleanup code is only run if the `:skipif:` + # expression evaluates to a False value. + # Global setup/cleanup are run before/after evaluating the `:skipif:` + # option in each directive - thus 11 additional invocations for each on top + # of the ones made for the whole test file. + assert recorded_calls == {('doctest_global_setup', 'body', True): 13, + ('testsetup', ':skipif:', True): 1, + ('testsetup', ':skipif:', False): 1, + ('testsetup', 'body', False): 1, + ('doctest', ':skipif:', True): 1, + ('doctest', ':skipif:', False): 1, + ('doctest', 'body', False): 1, + ('testcode', ':skipif:', True): 1, + ('testcode', ':skipif:', False): 1, + ('testcode', 'body', False): 1, + ('testoutput-1', ':skipif:', True): 1, + ('testoutput-2', ':skipif:', True): 1, + ('testoutput-2', ':skipif:', False): 1, + ('testcleanup', ':skipif:', True): 1, + ('testcleanup', ':skipif:', False): 1, + ('testcleanup', 'body', False): 1, + ('doctest_global_cleanup', 'body', True): 13} + + +def record(directive, part, should_skip): + global recorded_calls + recorded_calls[(directive, part, should_skip)] += 1 + return 'Recorded {} {} {}'.format(directive, part, should_skip) + + @pytest.mark.xfail( PY2, reason='node.source points to document instead of filename', )