diff --git a/CHANGES b/CHANGES
index ea18c839f..00ebf1e76 100644
--- a/CHANGES
+++ b/CHANGES
@@ -18,6 +18,7 @@ Bugs fixed
 
 * #5490: latex: enumerated list causes a crash with recommonmark
 * #5492: sphinx-build fails to build docs w/ Python < 3.5.2
+* #3704: latex: wrong ``\label`` positioning for figures with a legend
 
 Testing
 --------
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index e9a07e688..4ef8ddbca 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -1720,7 +1720,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
 
     def visit_figure(self, node):
         # type: (nodes.Node) -> None
-        labels = self.hypertarget_to(node)
         if self.table:
             # TODO: support align option
             if 'width' in node:
@@ -1732,7 +1731,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 self.body.append('\\begin{sphinxfigure-in-table}\n\\centering\n')
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart')
-            self.context.append(labels + '\\end{sphinxfigure-in-table}\\relax\n')
+            self.context.append('\\end{sphinxfigure-in-table}\\relax\n')
         elif node.get('align', '') in ('left', 'right'):
             length = None
             if 'width' in node:
@@ -1741,7 +1740,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 length = self.latex_image_length(node[0]['width'])
             self.body.append('\\begin{wrapfigure}{%s}{%s}\n\\centering' %
                              (node['align'] == 'right' and 'r' or 'l', length or '0pt'))
-            self.context.append(labels + '\\end{wrapfigure}\n')
+            self.context.append('\\end{wrapfigure}\n')
         elif self.in_minipage:
             self.body.append('\n\\begin{center}')
             self.context.append('\\end{center}\n')
@@ -1750,7 +1749,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                              self.elements['figure_align'])
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart\n')
-            self.context.append(labels + '\\end{figure}\n')
+            self.context.append('\\end{figure}\n')
 
     def depart_figure(self, node):
         # type: (nodes.Node) -> None
@@ -1771,6 +1770,9 @@ class LaTeXTranslator(nodes.NodeVisitor):
     def depart_caption(self, node):
         # type: (nodes.Node) -> None
         self.body.append('}')
+        if isinstance(node.parent, nodes.figure):
+            labels = self.hypertarget_to(node.parent)
+            self.body.append(labels)
         self.in_caption -= 1
 
     def visit_legend(self, node):
diff --git a/tests/roots/test-latex-labels/index.rst b/tests/roots/test-latex-labels/index.rst
index 5859fb6d2..c55ee0b04 100644
--- a/tests/roots/test-latex-labels/index.rst
+++ b/tests/roots/test-latex-labels/index.rst
@@ -16,6 +16,8 @@ figures
 
    labeled figure
 
+   with a legend
+
 code-blocks
 -----------
 
diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py
index 8075bf8d0..b2ae24c52 100644
--- a/tests/test_build_latex.py
+++ b/tests/test_build_latex.py
@@ -1299,7 +1299,8 @@ def test_latex_labels(app, status, warning):
             r'\label{\detokenize{index:figure1}}'
             r'\end{figure}' in result)
     assert (r'\caption{labeled figure}'
-            r'\label{\detokenize{index:figure3}}'
+            '\\label{\detokenize{index:figure3}}\n'
+            '\\begin{sphinxlegend}\nwith a legend\n\\end{sphinxlegend}\n'
             r'\end{figure}' in result)
 
     # code-blocks