diff --git a/tests/test_build_epub.py b/tests/test_build_epub.py
index 52a6e3dfe..2f09f6d5a 100644
--- a/tests/test_build_epub.py
+++ b/tests/test_build_epub.py
@@ -19,13 +19,13 @@ import pytest
 # check given command is runnable
 def runnable(command):
     try:
-        p = Popen(command, stdout=PIPE)
+        p = Popen(command, stdout=PIPE, stderr=PIPE)
     except OSError:
         # command not found
         return False
     else:
         p.communicate()
-        return p.returncode
+        return p.returncode == 0
 
 
 class EPUBElementTree(object):
@@ -322,7 +322,7 @@ def test_run_epubcheck(app):
     app.build()
 
     epubcheck = os.environ.get('EPUBCHECK_PATH', '/usr/share/java/epubcheck.jar')
-    if runnable('java') and os.path.exists(epubcheck):
+    if runnable(['java', '-version']) and os.path.exists(epubcheck):
         p = Popen(['java', '-jar', epubcheck, app.outdir / 'SphinxTests.epub'],
                   stdout=PIPE, stderr=PIPE)
         stdout, stderr = p.communicate()
diff --git a/tox.ini b/tox.ini
index b8d6a0e32..16ea8f9ba 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@ envlist = docs,flake8,mypy,coverage,py{27,34,35,36,py},du{11,12,13,14}
 [testenv]
 usedevelop = True
 passenv =
-    https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS
+    https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS EPUBCHECK_PATH
 description =
     py{27,34,35,36,py}: Run unit tests against {envname}.
     du{11,12,13,14}: Run unit tests with the given version of docutils.