diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index 943f5a170..801726f4e 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -1445,8 +1445,8 @@ class LaTeXTranslator(nodes.NodeVisitor):
         if self.in_container_literal_block:
             self.body.append('\\needspace{\\literalblockneedspace}')
             self.body.append('\\vspace{\\literalblockcaptiontopvspace}\n')
-            self.body.append(self.context.pop())
-            self.body.append('\\SphinxSetupCaptionForVerbatim{literal-block}{')
+            self.body.append('\\SphinxSetupCaptionForVerbatim{literal-block}{'
+                             + self.context.pop())
         elif self.in_minipage and isinstance(node.parent, nodes.figure):
             self.body.append('\\captionof{figure}{')
         else:
@@ -1992,14 +1992,15 @@ class LaTeXTranslator(nodes.NodeVisitor):
             for id in self.pop_hyperlink_ids('code-block'):
                 ids += self.hypertarget(id, anchor=False)
             if node['ids']:
-                ids += self.hypertarget(node['ids'][0])
+                # suppress with anchor=False \phantomsection generation
+                ids += self.hypertarget(node['ids'][0], anchor=False)
             self.body.append('\n')
-            self.context.append(ids + '\n')
+            if ids:
+                self.context.append(ids) # will be used in visit_caption
 
     def depart_container(self, node):
         if node.get('literal_block'):
             self.in_container_literal_block -= 1
-            # self.body.append(self.context.pop()) moved to visit_caption
 
     def visit_decoration(self, node):
         pass
diff --git a/tests/test_directive_code.py b/tests/test_directive_code.py
index 6f7b749b1..59a440b53 100644
--- a/tests/test_directive_code.py
+++ b/tests/test_directive_code.py
@@ -64,7 +64,7 @@ def test_code_block_caption_html(app, status, warning):
 def test_code_block_caption_latex(app, status, warning):
     app.builder.build_all()
     latex = (app.outdir / 'Python.tex').text(encoding='utf-8')
-    caption = '\\SphinxSetupCaptionForVerbatim{literal-block}{caption \\emph{test} rb}'
+    caption = '\\SphinxSetupCaptionForVerbatim{literal-block}{\label{caption:caption-test-rb}caption \\emph{test} rb}'
     assert caption in latex
 
 
@@ -229,7 +229,7 @@ def test_literalinclude_caption_html(app, status, warning):
 def test_literalinclude_caption_latex(app, status, warning):
     app.builder.build('index')
     latex = (app.outdir / 'Python.tex').text(encoding='utf-8')
-    caption = '\\SphinxSetupCaptionForVerbatim{literal-block}{caption \\textbf{test} py}'
+    caption = '\\SphinxSetupCaptionForVerbatim{literal-block}{\label{caption:caption-test-py}caption \\textbf{test} py}'
     assert caption in latex