diff --git a/CHANGES b/CHANGES
index e9df11595..70f06ef4b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -183,6 +183,8 @@ Bugs fixed
 Testing
 --------
 
+* Stop to use ``SPHINX_TEST_TEMPDIR`` envvar
+
 Release 1.8.4 (in development)
 ==============================
 
diff --git a/sphinx/testing/fixtures.py b/sphinx/testing/fixtures.py
index ad5a9d1da..93195fc40 100644
--- a/sphinx/testing/fixtures.py
+++ b/sphinx/testing/fixtures.py
@@ -14,7 +14,6 @@ import sys
 from collections import namedtuple
 from io import StringIO
 from subprocess import PIPE
-from tempfile import gettempdir
 
 import pytest
 
@@ -221,11 +220,15 @@ def if_graphviz_found(app):
 
 
 @pytest.fixture(scope='session')
-def sphinx_test_tempdir():
+def sphinx_test_tempdir(tmpdir_factory):
     """
     temporary directory that wrapped with `path` class.
     """
-    return util.path(os.environ.get('SPHINX_TEST_TEMPDIR', gettempdir())).abspath()
+    tmpdir = os.environ.get('SPHINX_TEST_TEMPDIR')  # RemovedInSphinx40Warning
+    if tmpdir is None:
+        tmpdir = tmpdir_factory.getbasetemp()
+
+    return util.path(tmpdir).abspath()
 
 
 @pytest.fixture
diff --git a/tests/conftest.py b/tests/conftest.py
index 7d0c1570b..a8392a6c0 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,7 +7,6 @@
 """
 
 import os
-import shutil
 
 import docutils
 import pytest
@@ -32,23 +31,5 @@ def pytest_report_header(config):
             (sphinx.__display_version__, docutils.__version__))
 
 
-def _initialize_test_directory(session):
-    testroot = os.path.join(str(session.config.rootdir), 'tests')
-    tempdir = os.path.abspath(os.getenv('SPHINX_TEST_TEMPDIR',
-                              os.path.join(testroot, 'build')))
-    os.environ['SPHINX_TEST_TEMPDIR'] = tempdir
-
-    print('Temporary files will be placed in %s.' % tempdir)
-
-    if os.path.exists(tempdir):
-        shutil.rmtree(tempdir)
-
-    os.makedirs(tempdir)
-
-
-def pytest_sessionstart(session):
-    _initialize_test_directory(session)
-
-
 def pytest_assertrepr_compare(op, left, right):
     comparer.pytest_assertrepr_compare(op, left, right)
diff --git a/tox.ini b/tox.ini
index 20f8cfdc8..13e6f6be3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,7 +18,6 @@ extras =
     websupport
 setenv =
     PYTHONWARNINGS = all,ignore::ImportWarning:pkgutil,ignore::ImportWarning:importlib._bootstrap,ignore::ImportWarning:importlib._bootstrap_external,ignore::ImportWarning:pytest_cov.plugin,ignore::DeprecationWarning:site,ignore::DeprecationWarning:_pytest.assertion.rewrite,ignore::DeprecationWarning:_pytest.fixtures,ignore::DeprecationWarning:distutils
-    SPHINX_TEST_TEMPDIR = {envdir}/testbuild
 commands=
     pytest --durations 25 {posargs}