diff --git a/CHANGES b/CHANGES
index 78f8ca05f..760f30e6a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -37,6 +37,7 @@ Bugs fixed
 * #4956: autodoc: Failed to extract document from a subclass of the class on
   mocked module
 * #4973: latex: glossary directive adds whitespace to each item
+* #4980: latex: Explicit labels on code blocks are duplicated
 
 Testing
 --------
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index ff81f9cdd..bd88ca842 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -49,6 +49,12 @@ BEGIN_DOC = r'''
 
 LATEXSECTIONNAMES = ["part", "chapter", "section", "subsection",
                      "subsubsection", "paragraph", "subparagraph"]
+HYPERLINK_SUPPORT_NODES = (
+    nodes.figure,
+    nodes.literal_block,
+    nodes.table,
+    nodes.section,
+)
 
 DEFAULT_SETTINGS = {
     'latex_engine':    'pdflatex',
@@ -728,6 +734,14 @@ class LaTeXTranslator(nodes.NodeVisitor):
         return (anchor and '\\phantomsection' or '') + \
             '\\label{%s}' % self.idescape(id)
 
+    def hypertarget_to(self, node, anchor=False):
+        # type: (nodes.Node, bool) -> unicode
+        labels = ''.join(self.hypertarget(node_id, anchor=False) for node_id in node['ids'])
+        if anchor:
+            return r'\phantomsection' + labels
+        else:
+            return labels
+
     def hyperlink(self, id):
         # type: (unicode) -> unicode
         return '{\\hyperref[%s]{' % self.idescape(id)
@@ -937,8 +951,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
         if not self.this_is_the_title:
             self.sectionlevel += 1
         self.body.append('\n\n')
-        if node.get('ids'):
-            self.next_section_ids.update(node['ids'])
 
     def depart_section(self, node):
         # type: (nodes.Node) -> None
@@ -1033,9 +1045,9 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 except IndexError:
                     # just use "subparagraph", it's not numbered anyway
                     self.body.append(r'\%s%s{' % (self.sectionnames[-1], short))
-                self.context.append('}\n')
-
+                self.context.append('}\n' + self.hypertarget_to(node.parent))
                 self.restrict_footnote(node)
+
                 if self.next_section_ids:
                     for id in self.next_section_ids:
                         self.context[-1] += self.hypertarget(id, anchor=False)
@@ -1306,12 +1318,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
 
     def depart_table(self, node):
         # type: (nodes.Node) -> None
-        labels = ''  # type: unicode
-        for labelid in self.pop_hyperlink_ids('table'):
-            labels += self.hypertarget(labelid, anchor=False)
-        if node['ids']:
-            labels += self.hypertarget(node['ids'][0], anchor=False)
-
+        labels = self.hypertarget_to(node)
         table_type = self.table.get_table_type()
         table = self.render(table_type + '.tex_t',
                             dict(table=self.table, labels=labels))
@@ -1758,16 +1765,8 @@ class LaTeXTranslator(nodes.NodeVisitor):
 
     def visit_figure(self, node):
         # type: (nodes.Node) -> None
-        ids = ''  # type: unicode
-        for id in self.pop_hyperlink_ids('figure'):
-            ids += self.hypertarget(id, anchor=False)
-        if node['ids']:
-            ids += self.hypertarget(node['ids'][0], anchor=False)
+        labels = self.hypertarget_to(node)
         self.restrict_footnote(node)
-        if (len(node.children) and
-           isinstance(node.children[0], nodes.image) and
-           node.children[0]['ids']):
-            ids += self.hypertarget(node.children[0]['ids'][0], anchor=False)
         if self.table:
             # TODO: support align option
             if 'width' in node:
@@ -1779,7 +1778,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 self.body.append('\\begin{sphinxfigure-in-table}\n\\centering\n')
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart')
-            self.context.append(ids + '\\end{sphinxfigure-in-table}\\relax\n')
+            self.context.append(labels + '\\end{sphinxfigure-in-table}\\relax\n')
         elif node.get('align', '') in ('left', 'right'):
             length = None
             if 'width' in node:
@@ -1788,7 +1787,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 length = self.latex_image_length(node[0]['width'])
             self.body.append('\\begin{wrapfigure}{%s}{%s}\n\\centering' %
                              (node['align'] == 'right' and 'r' or 'l', length or '0pt'))
-            self.context.append(ids + '\\end{wrapfigure}\n')
+            self.context.append(labels + '\\end{wrapfigure}\n')
         elif self.in_minipage:
             self.body.append('\n\\begin{center}')
             self.context.append('\\end{center}\n')
@@ -1797,7 +1796,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                              self.elements['figure_align'])
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart\n')
-            self.context.append(ids + '\\end{figure}\n')
+            self.context.append(labels + '\\end{figure}\n')
 
     def depart_figure(self, node):
         # type: (nodes.Node) -> None
@@ -1899,6 +1898,11 @@ class LaTeXTranslator(nodes.NodeVisitor):
             anchor = not self.in_title
             self.body.append(self.hypertarget(id, anchor=anchor))
 
+        # skip if visitor for next node supports hyperlink
+        next_node = node.next_node(ascend=True)
+        if isinstance(next_node, HYPERLINK_SUPPORT_NODES):
+            return
+
         # postpone the labels until after the sectioning command
         parindex = node.parent.index(node)
         try:
@@ -1912,22 +1916,16 @@ class LaTeXTranslator(nodes.NodeVisitor):
                         node.parent.parent.index(node.parent)]
                 else:
                     raise
-            if isinstance(next, nodes.section):
+            domain = self.builder.env.get_domain('std')
+            figtype = domain.get_figtype(next)
+            if figtype and domain.get_numfig_title(next):
+                ids = set()
+                # labels for figures go in the figure body, not before
                 if node.get('refid'):
-                    self.next_section_ids.add(node['refid'])
-                self.next_section_ids.update(node['ids'])
+                    ids.add(node['refid'])
+                ids.update(node['ids'])
+                self.push_hyperlink_ids(figtype, ids)
                 return
-            else:
-                domain = self.builder.env.get_domain('std')
-                figtype = domain.get_figtype(next)
-                if figtype and domain.get_numfig_title(next):
-                    ids = set()
-                    # labels for figures go in the figure body, not before
-                    if node.get('refid'):
-                        ids.add(node['refid'])
-                    ids.update(node['ids'])
-                    self.push_hyperlink_ids(figtype, ids)
-                    return
         except IndexError:
             pass
         if 'refuri' in node:
@@ -2232,15 +2230,10 @@ class LaTeXTranslator(nodes.NodeVisitor):
             self.in_parsed_literal += 1
             self.body.append('\\begin{sphinxalltt}\n')
         else:
-            ids = ''  # type: unicode
-            for id in self.pop_hyperlink_ids('code-block'):
-                ids += self.hypertarget(id, anchor=False)
-            if node['ids']:
-                # suppress with anchor=False \phantomsection insertion
-                ids += self.hypertarget(node['ids'][0], anchor=False)
+            labels = self.hypertarget_to(node)
             # LaTeX code will insert \phantomsection prior to \label
-            if ids and not self.in_footnote:
-                self.body.append('\n\\def\\sphinxLiteralBlockLabel{' + ids + '}')
+            if labels and not self.in_footnote:
+                self.body.append('\n\\def\\sphinxLiteralBlockLabel{' + labels + '}')
             code = node.astext()
             lang = self.hlsettingstack[-1][0]
             linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
diff --git a/tests/roots/test-latex-labels/conf.py b/tests/roots/test-latex-labels/conf.py
new file mode 100644
index 000000000..31e7a6ed4
--- /dev/null
+++ b/tests/roots/test-latex-labels/conf.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+master_doc = 'index'
+
+latex_documents = [
+    (master_doc, 'test.tex', 'The basic Sphinx documentation for testing', 'Sphinx', 'report')
+]
diff --git a/tests/roots/test-latex-labels/index.rst b/tests/roots/test-latex-labels/index.rst
new file mode 100644
index 000000000..5859fb6d2
--- /dev/null
+++ b/tests/roots/test-latex-labels/index.rst
@@ -0,0 +1,68 @@
+latex-labels
+============
+
+figures
+-------
+
+.. _figure1:
+.. _figure2:
+
+.. figure:: logo.jpg
+
+   labeled figure
+
+.. figure:: logo.jpg
+   :name: figure3
+
+   labeled figure
+
+code-blocks
+-----------
+
+.. _codeblock1:
+.. _codeblock2:
+
+.. code-block:: none
+
+   blah blah blah
+
+.. code-block:: none
+   :name: codeblock3
+
+   blah blah blah
+
+tables
+------
+
+.. _table1:
+.. _table2:
+
+.. table:: table caption
+
+   ==== ====
+   head head
+   cell cell
+   ==== ====
+
+.. table:: table caption
+   :name: table3
+
+   ==== ====
+   head head
+   cell cell
+   ==== ====
+
+.. _section1:
+.. _section2:
+
+subsection
+----------
+
+.. _section3:
+
+subsubsection
+~~~~~~~~~~~~~
+
+.. toctree::
+
+   otherdoc
diff --git a/tests/roots/test-latex-labels/otherdoc.rst b/tests/roots/test-latex-labels/otherdoc.rst
new file mode 100644
index 000000000..55c5ca051
--- /dev/null
+++ b/tests/roots/test-latex-labels/otherdoc.rst
@@ -0,0 +1,2 @@
+otherdoc
+========
diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py
index 9c6edd2f1..96ae14089 100644
--- a/tests/test_build_latex.py
+++ b/tests/test_build_latex.py
@@ -1264,3 +1264,47 @@ def test_latex_glossary(app, status, warning):
             r'\label{\detokenize{index:term-electron}}}] \leavevmode' in result)
     assert (u'\\item[{über\\index{über|textbf}\\phantomsection'
             r'\label{\detokenize{index:term-uber}}}] \leavevmode' in result)
+
+
+@pytest.mark.sphinx('latex', testroot='latex-labels')
+def test_latex_labels(app, status, warning):
+    app.builder.build_all()
+
+    result = (app.outdir / 'test.tex').text(encoding='utf8')
+
+    # figures
+    assert (r'\caption{labeled figure}'
+            r'\label{\detokenize{index:id1}}'
+            r'\label{\detokenize{index:figure2}}'
+            r'\label{\detokenize{index:figure1}}'
+            r'\end{figure}' in result)
+    assert (r'\caption{labeled figure}'
+            r'\label{\detokenize{index:figure3}}'
+            r'\end{figure}' in result)
+
+    # code-blocks
+    assert (r'\def\sphinxLiteralBlockLabel{'
+            r'\label{\detokenize{index:codeblock2}}'
+            r'\label{\detokenize{index:codeblock1}}}' in result)
+    assert (r'\def\sphinxLiteralBlockLabel{'
+            r'\label{\detokenize{index:codeblock3}}}' in result)
+
+    # tables
+    assert (r'\sphinxcaption{table caption}'
+            r'\label{\detokenize{index:id2}}'
+            r'\label{\detokenize{index:table2}}'
+            r'\label{\detokenize{index:table1}}' in result)
+    assert (r'\sphinxcaption{table caption}'
+            r'\label{\detokenize{index:table3}}' in result)
+
+    # sections
+    assert ('\\chapter{subsection}\n'
+            r'\label{\detokenize{index:subsection}}'
+            r'\label{\detokenize{index:section2}}'
+            r'\label{\detokenize{index:section1}}' in result)
+    assert ('\\section{subsubsection}\n'
+            r'\label{\detokenize{index:subsubsection}}'
+            r'\label{\detokenize{index:section3}}' in result)
+    assert ('\\subsection{otherdoc}\n'
+            r'\label{\detokenize{otherdoc:otherdoc}}'
+            r'\label{\detokenize{otherdoc::doc}}' in result)