From 54df51e86f8853f4d6215954e5ec0bcd5f9e4533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Freitag?= Date: Sun, 24 Jan 2021 11:52:04 +0100 Subject: [PATCH] =?UTF-8?q?Linkcheck:=20Don=E2=80=99t=20repeatedly=20open/?= =?UTF-8?q?close=20log=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opening and closing a file requires processing from the operating system. Repeatedly opening and closing wastes system resources and hinders buffering, causing a flush (disk I/O) after each write operation. Using a context manager ensures the logs are flushed to disk and files are properly closed whether the program exists successfully or an exception occurs. Compared to the previous implementation, a brutal shutdown of the machine (e.g. power cord disconnected) could cause some log lines not to be written. That should not be an issue in practice. Now, files are created and truncated when linkcheck submitted the links to check to the threads and is ready to process the results, instead of when the builder is constructed. It keeps the file operations closer to their use. --- sphinx/builders/linkcheck.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index f813922ec..9015b706e 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -118,10 +118,6 @@ class CheckExternalLinksBuilder(DummyBuilder): self._redirected = {} # type: Dict[str, Tuple[str, int]] # set a timeout for non-responding servers socket.setdefaulttimeout(5.0) - # create output file - open(path.join(self.outdir, 'output.txt'), 'w').close() - # create JSON output file - open(path.join(self.outdir, 'output.json'), 'w').close() # create queues and worker threads self.rate_limits = {} # type: Dict[str, RateLimit] @@ -435,13 +431,11 @@ class CheckExternalLinksBuilder(DummyBuilder): def write_entry(self, what: str, docname: str, filename: str, line: int, uri: str) -> None: - with open(path.join(self.outdir, 'output.txt'), 'a') as output: - output.write("%s:%s: [%s] %s\n" % (filename, line, what, uri)) + self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri)) def write_linkstat(self, data: dict) -> None: - with open(path.join(self.outdir, 'output.json'), 'a') as output: - output.write(json.dumps(data)) - output.write('\n') + self.json_outfile.write(json.dumps(data)) + self.json_outfile.write('\n') def finish(self) -> None: logger.info('') @@ -452,9 +446,11 @@ class CheckExternalLinksBuilder(DummyBuilder): n += 1 done = 0 - while done < n: - self.process_result(self.rqueue.get()) - done += 1 + with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\ + open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile: + while done < n: + self.process_result(self.rqueue.get()) + done += 1 if self._broken: self.app.statuscode = 1