diff --git a/CHANGES b/CHANGES
index e850b3910..c7c273390 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,6 +27,7 @@ Bugs fixed
 * #4026: nature: Fix macOS Safari scrollbar color
 * #3877: Fix for C++ multiline signatures.
 * #4006: Fix crash on parallel build
+* #3969: private instance attributes causes AttributeError
 
 Testing
 --------
diff --git a/sphinx/ext/autodoc.py b/sphinx/ext/autodoc.py
index 91feb6a2a..bb5f78332 100644
--- a/sphinx/ext/autodoc.py
+++ b/sphinx/ext/autodoc.py
@@ -962,14 +962,20 @@ class Documenter(object):
                     self.options.special_members is not ALL and \
                         membername in self.options.special_members:
                     keep = has_doc or self.options.undoc_members
+            elif (namespace, membername) in attr_docs:
+                if want_all and membername.startswith('_'):
+                    # ignore members whose name starts with _ by default
+                    keep = self.options.private_members and \
+                        (has_doc or self.options.undoc_members)
+                else:
+                    # keep documented attributes
+                    keep = True
+                isattr = True
+                print(membername, keep)
             elif want_all and membername.startswith('_'):
                 # ignore members whose name starts with _ by default
                 keep = self.options.private_members and \
                     (has_doc or self.options.undoc_members)
-            elif (namespace, membername) in attr_docs:
-                # keep documented attributes
-                keep = True
-                isattr = True
             else:
                 # ignore undocumented members if :undoc-members: is not given
                 keep = has_doc or self.options.undoc_members
diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py
index e04e38bb3..6eb087567 100644
--- a/tests/test_autodoc.py
+++ b/tests/test_autodoc.py
@@ -107,7 +107,7 @@ def process_signature(app, what, name, obj, options, args, retann):
 def skip_member(app, what, name, obj, skip, options):
     if name in ('__special1__', '__special2__'):
         return skip
-    if name.startswith('_'):
+    if name.startswith('__'):
         return True
     if name == 'skipmeth':
         return True
@@ -756,6 +756,7 @@ def test_generate():
 
     # test autodoc_member_order == 'source'
     directive.env.ref_context['py:module'] = 'test_autodoc'
+    options.private_members = True
     if PY3:
         roger_line = '   .. py:classmethod:: Class.roger(a, *, b=2, c=3, d=4, e=5, f=6)'
     else:
@@ -773,6 +774,7 @@ def test_generate():
                   '   .. py:classmethod:: Class.moore(a, e, f) -> happiness',
                   '   .. py:attribute:: Class.inst_attr_comment',
                   '   .. py:attribute:: Class.inst_attr_string',
+                  '   .. py:attribute:: Class._private_inst_attr',
                   '   .. py:method:: Class.inheritedmeth()',
                   ],
                  'class', 'Class', member_order='bysource', all_members=True)
@@ -989,6 +991,7 @@ class Class(Base):
         self.inst_attr_comment = None
         self.inst_attr_string = None
         """a documented instance attribute"""
+        self._private_inst_attr = None  #: a private instance attribute
 
     def __special1__(self):
         """documented special method"""