diff --git a/CHANGES b/CHANGES
index 5bea0d78b..5ac921cf9 100644
--- a/CHANGES
+++ b/CHANGES
@@ -23,6 +23,8 @@ Incompatible changes
   :py:meth:`.Sphinx.add_transform()`
 * #4827: All ``substitution_definition`` nodes are removed from doctree on
   reading phase
+* ``docutils.conf`` on ``$HOME`` and ``/etc`` directories are ignored.  Only
+  ``docutils.conf`` on confdir is refered.
 
 Deprecated
 ----------
diff --git a/sphinx/cmd/build.py b/sphinx/cmd/build.py
index bc8e5aa9c..53e93fb99 100644
--- a/sphinx/cmd/build.py
+++ b/sphinx/cmd/build.py
@@ -293,7 +293,8 @@ def build_main(argv=sys.argv[1:]):  # type: ignore
 
     app = None
     try:
-        with patch_docutils(), docutils_namespace():
+        confdir = args.confdir or args.sourcedir
+        with patch_docutils(confdir), docutils_namespace():
             app = Sphinx(args.sourcedir, args.confdir, args.outputdir,
                          args.doctreedir, args.builder, confoverrides, status,
                          warning, args.freshenv, args.warningiserror,
diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py
index ccdeb03bd..f69a44d76 100644
--- a/sphinx/environment/__init__.py
+++ b/sphinx/environment/__init__.py
@@ -19,7 +19,6 @@ from collections import defaultdict
 from copy import copy
 from os import path
 
-from docutils.frontend import OptionParser
 from docutils.utils import Reporter, get_source_line
 from six import BytesIO, class_types, next
 from six.moves import cPickle as pickle
@@ -561,9 +560,8 @@ class BuildEnvironment(object):
         """Parse a file and add/update inventory entries for the doctree."""
         self.prepare_settings(docname)
 
-        docutilsconf = path.join(self.srcdir, 'docutils.conf')
-        # read docutils.conf from source dir, not from current dir
-        OptionParser.standard_config_files[1] = docutilsconf
+        # Add confdir/docutils.conf to dependencies list if exists
+        docutilsconf = path.join(self.app.confdir, 'docutils.conf')
         if path.isfile(docutilsconf):
             self.note_dependency(docutilsconf)
 
diff --git a/sphinx/setup_command.py b/sphinx/setup_command.py
index 37fc2d042..eebc70def 100644
--- a/sphinx/setup_command.py
+++ b/sphinx/setup_command.py
@@ -179,7 +179,8 @@ class BuildDoc(Command):
             app = None
 
             try:
-                with patch_docutils(), docutils_namespace():
+                confdir = self.config_dir or self.source_dir
+                with patch_docutils(confdir), docutils_namespace():
                     app = Sphinx(self.source_dir, self.config_dir,
                                  builder_target_dir, self.doctree_dir,
                                  builder, confoverrides, status_stream,
diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py
index 747b24565..b09d4e841 100644
--- a/sphinx/util/docutils.py
+++ b/sphinx/util/docutils.py
@@ -10,16 +10,17 @@
 """
 from __future__ import absolute_import
 
+import os
 import re
 import types
 import warnings
 from contextlib import contextmanager
 from copy import copy
 from distutils.version import LooseVersion
+from os import path
 
 import docutils
 from docutils import nodes
-from docutils.languages import get_language
 from docutils.parsers.rst import Directive, directives, roles, convert_directive_function
 from docutils.statemachine import StateMachine
 from docutils.utils import Reporter
@@ -34,7 +35,7 @@ report_re = re.compile('^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(
 
 if False:
     # For type annotation
-    from typing import Any, Callable, Generator, Iterator, List, Set, Tuple  # NOQA
+    from typing import Any, Callable, Generator, List, Set, Tuple  # NOQA
     from docutils.statemachine import State, ViewList  # NOQA
     from sphinx.config import Config  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
@@ -47,7 +48,7 @@ additional_nodes = set()  # type: Set[nodes.Node]
 
 @contextmanager
 def docutils_namespace():
-    # type: () -> Iterator[None]
+    # type: () -> Generator[None, None, None]
     """Create namespace for reST parsers."""
     try:
         _directives = copy(directives._directives)
@@ -94,29 +95,53 @@ def unregister_node(node):
         delattr(nodes.SparseNodeVisitor, 'depart_' + node.__name__)
 
 
-def patched_get_language(language_code, reporter=None):
-    # type: (unicode, Reporter) -> Any
-    """A wrapper for docutils.languages.get_language().
+@contextmanager
+def patched_get_language():
+    # type: () -> Generator[None, None, None]
+    """Patch docutils.languages.get_language() temporarily.
 
     This ignores the second argument ``reporter`` to suppress warnings.
     refs: https://github.com/sphinx-doc/sphinx/issues/3788
     """
-    return get_language(language_code)
+    from docutils.languages import get_language
 
+    def patched_get_language(language_code, reporter=None):
+        # type: (unicode, Reporter) -> Any
+        return get_language(language_code)
 
-@contextmanager
-def patch_docutils():
-    # type: () -> Iterator[None]
-    """Patch to docutils temporarily."""
     try:
         docutils.languages.get_language = patched_get_language
-
         yield
     finally:
         # restore original implementations
         docutils.languages.get_language = get_language
 
 
+@contextmanager
+def using_user_docutils_conf(confdir):
+    # type: (unicode) -> Generator[None, None, None]
+    """Let docutils know the location of ``docutils.conf`` for Sphinx."""
+    try:
+        docutilsconfig = os.environ.get('DOCUTILSCONFIG', None)
+        if confdir:
+            os.environ['DOCUTILSCONFIG'] = path.join(path.abspath(confdir), 'docutils.conf')  # type: ignore  # NOQA
+
+        yield
+    finally:
+        if docutilsconfig is None:
+            os.environ.pop('DOCUTILSCONFIG')
+        else:
+            os.environ['DOCUTILSCONFIG'] = docutilsconfig
+
+
+@contextmanager
+def patch_docutils(confdir=None):
+    # type: (unicode) -> Generator[None, None, None]
+    """Patch to docutils temporarily."""
+    with patched_get_language(), using_user_docutils_conf(confdir):
+        yield
+
+
 class ElementLookupError(Exception):
     pass
 
@@ -257,7 +282,7 @@ def directive_helper(obj, has_content=None, argument_spec=None, **option_spec):
 
 @contextmanager
 def switch_source_input(state, content):
-    # type: (State, ViewList) -> Generator
+    # type: (State, ViewList) -> Generator[None, None, None]
     """Switch current source input of state temporarily."""
     try:
         # remember the original ``get_source_and_line()`` method
diff --git a/tests/test_docutilsconf.py b/tests/test_docutilsconf.py
index 2332dcf50..989edc6a8 100644
--- a/tests/test_docutilsconf.py
+++ b/tests/test_docutilsconf.py
@@ -15,6 +15,7 @@ import sys
 import pytest
 
 from sphinx.testing.path import path
+from sphinx.util.docutils import patch_docutils
 
 
 def regex_count(expr, result):
@@ -23,7 +24,9 @@ def regex_count(expr, result):
 
 @pytest.mark.sphinx('html', testroot='docutilsconf', freshenv=True, docutilsconf='')
 def test_html_with_default_docutilsconf(app, status, warning):
-    app.builder.build(['contents'])
+    with patch_docutils(app.confdir):
+        app.builder.build(['contents'])
+
     result = (app.outdir / 'contents.html').text(encoding='utf-8')
 
     assert regex_count(r'<th class="field-name">', result) == 1
@@ -39,7 +42,9 @@ def test_html_with_default_docutilsconf(app, status, warning):
     '\n')
 )
 def test_html_with_docutilsconf(app, status, warning):
-    app.builder.build(['contents'])
+    with patch_docutils(app.confdir):
+        app.builder.build(['contents'])
+
     result = (app.outdir / 'contents.html').text(encoding='utf-8')
 
     assert regex_count(r'<th class="field-name">', result) == 0
@@ -50,25 +55,29 @@ def test_html_with_docutilsconf(app, status, warning):
 
 @pytest.mark.sphinx('html', testroot='docutilsconf')
 def test_html(app, status, warning):
-    app.builder.build(['contents'])
+    with patch_docutils(app.confdir):
+        app.builder.build(['contents'])
     assert warning.getvalue() == ''
 
 
 @pytest.mark.sphinx('latex', testroot='docutilsconf')
 def test_latex(app, status, warning):
-    app.builder.build(['contents'])
+    with patch_docutils(app.confdir):
+        app.builder.build(['contents'])
     assert warning.getvalue() == ''
 
 
 @pytest.mark.sphinx('man', testroot='docutilsconf')
 def test_man(app, status, warning):
-    app.builder.build(['contents'])
+    with patch_docutils(app.confdir):
+        app.builder.build(['contents'])
     assert warning.getvalue() == ''
 
 
 @pytest.mark.sphinx('texinfo', testroot='docutilsconf')
 def test_texinfo(app, status, warning):
-    app.builder.build(['contents'])
+    with patch_docutils(app.confdir):
+        app.builder.build(['contents'])
 
 
 @pytest.mark.sphinx('html', testroot='docutilsconf',
@@ -87,4 +96,5 @@ def test_docutils_source_link_with_nonascii_file(app, status, warning):
             'nonascii filename not supported on this filesystem encoding: '
             '%s', FILESYSTEMENCODING)
 
-    app.builder.build_all()
+    with patch_docutils(app.confdir):
+        app.builder.build_all()