diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py
index b2b04aea0..f8a8a3900 100644
--- a/tests/test_ext_autosummary.py
+++ b/tests/test_ext_autosummary.py
@@ -54,7 +54,7 @@ def test_mangle_signature():
         assert res == outp, (u"'%s' -> '%s' != '%s'" % (inp, res, outp))
 
 
-@with_app(buildername='html', **default_kw)
+@with_app(buildername='dummy', **default_kw)
 def test_get_items_summary(app, status, warning):
     # monkey-patch Autosummary.get_items so we can easily get access to it's
     # results..