diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index ff81f9cdd..04b10b0da 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -728,6 +728,14 @@ class LaTeXTranslator(nodes.NodeVisitor):
         return (anchor and '\\phantomsection' or '') + \
             '\\label{%s}' % self.idescape(id)
 
+    def hypertarget_to(self, node, anchor=False):
+        # type: (nodes.Node, bool) -> unicode
+        labels = ''.join(self.hypertarget(node_id, anchor=False) for node_id in node['ids'])
+        if anchor:
+            return r'\phantomsection' + labels
+        else:
+            return labels
+
     def hyperlink(self, id):
         # type: (unicode) -> unicode
         return '{\\hyperref[%s]{' % self.idescape(id)
@@ -1758,16 +1766,8 @@ class LaTeXTranslator(nodes.NodeVisitor):
 
     def visit_figure(self, node):
         # type: (nodes.Node) -> None
-        ids = ''  # type: unicode
-        for id in self.pop_hyperlink_ids('figure'):
-            ids += self.hypertarget(id, anchor=False)
-        if node['ids']:
-            ids += self.hypertarget(node['ids'][0], anchor=False)
+        labels = self.hypertarget_to(node)
         self.restrict_footnote(node)
-        if (len(node.children) and
-           isinstance(node.children[0], nodes.image) and
-           node.children[0]['ids']):
-            ids += self.hypertarget(node.children[0]['ids'][0], anchor=False)
         if self.table:
             # TODO: support align option
             if 'width' in node:
@@ -1779,7 +1779,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 self.body.append('\\begin{sphinxfigure-in-table}\n\\centering\n')
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart')
-            self.context.append(ids + '\\end{sphinxfigure-in-table}\\relax\n')
+            self.context.append(labels + '\\end{sphinxfigure-in-table}\\relax\n')
         elif node.get('align', '') in ('left', 'right'):
             length = None
             if 'width' in node:
@@ -1788,7 +1788,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 length = self.latex_image_length(node[0]['width'])
             self.body.append('\\begin{wrapfigure}{%s}{%s}\n\\centering' %
                              (node['align'] == 'right' and 'r' or 'l', length or '0pt'))
-            self.context.append(ids + '\\end{wrapfigure}\n')
+            self.context.append(labels + '\\end{wrapfigure}\n')
         elif self.in_minipage:
             self.body.append('\n\\begin{center}')
             self.context.append('\\end{center}\n')
@@ -1797,7 +1797,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                              self.elements['figure_align'])
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart\n')
-            self.context.append(ids + '\\end{figure}\n')
+            self.context.append(labels + '\\end{figure}\n')
 
     def depart_figure(self, node):
         # type: (nodes.Node) -> None
@@ -1899,6 +1899,11 @@ class LaTeXTranslator(nodes.NodeVisitor):
             anchor = not self.in_title
             self.body.append(self.hypertarget(id, anchor=anchor))
 
+        # skip if visitor for next node supports hyperlink
+        next_node = node.next_node(ascend=True)
+        if isinstance(next_node, nodes.figure):
+            return
+
         # postpone the labels until after the sectioning command
         parindex = node.parent.index(node)
         try:
diff --git a/tests/roots/test-latex-labels/conf.py b/tests/roots/test-latex-labels/conf.py
new file mode 100644
index 000000000..31e7a6ed4
--- /dev/null
+++ b/tests/roots/test-latex-labels/conf.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+master_doc = 'index'
+
+latex_documents = [
+    (master_doc, 'test.tex', 'The basic Sphinx documentation for testing', 'Sphinx', 'report')
+]
diff --git a/tests/roots/test-latex-labels/index.rst b/tests/roots/test-latex-labels/index.rst
new file mode 100644
index 000000000..8015ea761
--- /dev/null
+++ b/tests/roots/test-latex-labels/index.rst
@@ -0,0 +1,17 @@
+latex-labels
+============
+
+figures
+-------
+
+.. _figure1:
+.. _figure2:
+
+.. figure:: logo.jpg
+
+   labeled figure
+
+.. figure:: logo.jpg
+   :name: figure3
+
+   labeled figure
diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py
index 9c6edd2f1..530b3771e 100644
--- a/tests/test_build_latex.py
+++ b/tests/test_build_latex.py
@@ -1264,3 +1264,18 @@ def test_latex_glossary(app, status, warning):
             r'\label{\detokenize{index:term-electron}}}] \leavevmode' in result)
     assert (u'\\item[{über\\index{über|textbf}\\phantomsection'
             r'\label{\detokenize{index:term-uber}}}] \leavevmode' in result)
+
+
+@pytest.mark.sphinx('latex', testroot='latex-labels')
+def test_latex_labels(app, status, warning):
+    app.builder.build_all()
+
+    result = (app.outdir / 'test.tex').text(encoding='utf8')
+    assert (r'\caption{labeled figure}'
+            r'\label{\detokenize{index:id1}}'
+            r'\label{\detokenize{index:figure2}}'
+            r'\label{\detokenize{index:figure1}}'
+            r'\end{figure}' in result)
+    assert (r'\caption{labeled figure}'
+            r'\label{\detokenize{index:figure3}}'
+            r'\end{figure}' in result)