linkcheck: Fix race condition that could lead to checking the availability of the same URL twice

So far, linkcheck scans all of references and images from documents, and
checks them parallel.  As a result, some URL would be checked twice (or
more) by race condition.

This collects the URL via post-transforms, and removes duplicated URLs
before checking availability.

refs: #4303

tests/test_build_linkcheck.py

@@ -573,40 +573,3 @@ def test_limit_rate_bails_out_after_waiting_max_time(app):
     checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)}
     next_check = checker.limit_rate(FakeResponse())
     assert next_check is None
-
-
-@pytest.mark.sphinx(
-    'linkcheck', testroot='linkcheck-localserver-two-links', freshenv=True,
-)
-def test_priorityqueue_items_are_comparable(app):
-    with http_server(OKHandler):
-        app.builder.build_all()
-    content = (app.outdir / 'output.json').read_text()
-    rows = [json.loads(x) for x in sorted(content.splitlines())]
-    assert rows == [
-        {
-            'filename': 'index.rst',
-            # Should not be None.
-            'lineno': 0,
-            'status': 'working',
-            'code': 0,
-            'uri': 'http://localhost:7777/',
-            'info': '',
-        },
-        {
-            'filename': 'index.rst',
-            'lineno': 0,
-            'status': 'working',
-            'code': 0,
-            'uri': 'http://localhost:7777/',
-            'info': '',
-        },
-        {
-            'filename': 'index.rst',
-            'lineno': 4,
-            'status': 'working',
-            'code': 0,
-            'uri': 'http://localhost:7777/',
-            'info': '',
-        }
-    ]