IntenseWebs/sphinx
3
0
Fork 0
mirror of https://github.com/sphinx-doc/sphinx.git synced 2025-02-25 18:55:22 -06:00
Code Issues Packages Projects Releases Wiki Activity

Use more precise last-modified times for files (#12661)

Browse Source
This commit is contained in:
Adam Turner
2024-07-23 14:52:31 +01:00
committed by GitHub
parent de15d61a46
commit 57a63ea603
14 changed files with 94 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
sphinx/builders/html/__init__.py
View File

@@ -9,7 +9,6 @@ import os
import posixpath
import re
import sys
import time
import types
import warnings
from os import path
@@ -40,13 +39,21 @@ from sphinx.locale import _, __
from sphinx.search import js_index
from sphinx.theming import HTMLThemeFactory
from sphinx.util import isurl, logging
from sphinx.util._timestamps import _format_rfc3339_microseconds
from sphinx.util.display import progress_message, status_iterator
from sphinx.util.docutils import new_document
from sphinx.util.fileutil import copy_asset
from sphinx.util.i18n import format_date
from sphinx.util.inventory import InventoryFile
from sphinx.util.matching import DOTFILES, Matcher, patmatch
from sphinx.util.osutil import SEP, copyfile, ensuredir, os_path, relative_uri
from sphinx.util.osutil import (
SEP,
_last_modified_time,
copyfile,
ensuredir,
os_path,
relative_uri,
)
from sphinx.writers.html import HTMLWriter
from sphinx.writers.html5 import HTML5Translator
@@ -397,7 +404,7 @@ class StandaloneHTMLBuilder(Builder):
pass
if self.templates:
template_mtime = self.templates.newest_template_mtime()
template_mtime = int(self.templates.newest_template_mtime() * 10**6)
else:
template_mtime = 0
for docname in self.env.found_docs:
@@ -407,19 +414,19 @@ class StandaloneHTMLBuilder(Builder):
continue
targetname = self.get_outfilename(docname)
try:
targetmtime = path.getmtime(targetname)
targetmtime = _last_modified_time(targetname)
except Exception:
targetmtime = 0
try:
srcmtime = max(path.getmtime(self.env.doc2path(docname)), template_mtime)
srcmtime = max(_last_modified_time(self.env.doc2path(docname)), template_mtime)
if srcmtime > targetmtime:
logger.debug(
'[build target] targetname %r(%s), template(%s), docname %r(%s)',
targetname,
_format_modified_time(targetmtime),
_format_modified_time(template_mtime),
_format_rfc3339_microseconds(targetmtime),
_format_rfc3339_microseconds(template_mtime),
docname,
_format_modified_time(path.getmtime(self.env.doc2path(docname))),
_format_rfc3339_microseconds(_last_modified_time(self.env.doc2path(docname))),
)
yield docname
except OSError:
@@ -1224,12 +1231,6 @@ def convert_html_css_files(app: Sphinx, config: Config) -> None:
config.html_css_files = html_css_files
def _format_modified_time(timestamp: float) -> str:
"""Return an RFC 3339 formatted string representing the given timestamp."""
seconds, fraction = divmod(timestamp, 1)
return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(seconds)) + f'.{fraction:.3f}'
def convert_html_js_files(app: Sphinx, config: Config) -> None:
"""Convert string styled html_js_files to tuple styled one."""
html_js_files: list[tuple[str, dict[str, str]]] = []

10
sphinx/builders/text.py
View File

@@ -10,7 +10,11 @@ from docutils.io import StringOutput
from sphinx.builders import Builder
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.osutil import ensuredir, os_path
from sphinx.util.osutil import (
_last_modified_time,
ensuredir,
os_path,
)
from sphinx.writers.text import TextTranslator, TextWriter
if TYPE_CHECKING:
@@ -46,11 +50,11 @@ class TextBuilder(Builder):
continue
targetname = path.join(self.outdir, docname + self.out_suffix)
try:
targetmtime = path.getmtime(targetname)
targetmtime = _last_modified_time(targetname)
except Exception:
targetmtime = 0
try:
srcmtime = path.getmtime(self.env.doc2path(docname))
srcmtime = _last_modified_time(self.env.doc2path(docname))
if srcmtime > targetmtime:
yield docname
except OSError:

10
sphinx/builders/xml.py
View File

@@ -12,7 +12,11 @@ from docutils.writers.docutils_xml import XMLTranslator
from sphinx.builders import Builder
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.osutil import ensuredir, os_path
from sphinx.util.osutil import (
_last_modified_time,
ensuredir,
os_path,
)
from sphinx.writers.xml import PseudoXMLWriter, XMLWriter
if TYPE_CHECKING:
@@ -52,11 +56,11 @@ class XMLBuilder(Builder):
continue
targetname = path.join(self.outdir, docname + self.out_suffix)
try:
targetmtime = path.getmtime(targetname)
targetmtime = _last_modified_time(targetname)
except Exception:
targetmtime = 0
try:
srcmtime = path.getmtime(self.env.doc2path(docname))
srcmtime = _last_modified_time(self.env.doc2path(docname))
if srcmtime > targetmtime:
yield docname
except OSError:

30
sphinx/environment/__init__.py
View File

@@ -5,7 +5,6 @@ from __future__ import annotations
import functools
import os
import pickle
import time
from collections import defaultdict
from copy import copy
from os import path
@@ -17,10 +16,11 @@ from sphinx.errors import BuildEnvironmentError, DocumentError, ExtensionError,
from sphinx.locale import __
from sphinx.transforms import SphinxTransformer
from sphinx.util import DownloadFiles, FilenameUniqDict, logging
from sphinx.util._timestamps import _format_rfc3339_microseconds
from sphinx.util.docutils import LoggingReporter
from sphinx.util.i18n import CatalogRepository, docname_to_domain
from sphinx.util.nodes import is_translatable
from sphinx.util.osutil import canon_path, os_path
from sphinx.util.osutil import _last_modified_time, canon_path, os_path
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
@@ -508,7 +508,8 @@ class BuildEnvironment:
if newmtime > mtime:
logger.debug('[build target] outdated %r: %s -> %s',
docname,
_format_modified_time(mtime), _format_modified_time(newmtime))
_format_rfc3339_microseconds(mtime),
_format_rfc3339_microseconds(newmtime))
changed.add(docname)
continue
# finally, check the mtime of dependencies
@@ -528,7 +529,8 @@ class BuildEnvironment:
logger.debug(
'[build target] outdated %r from dependency %r: %s -> %s',
docname, deppath,
_format_modified_time(mtime), _format_modified_time(depmtime),
_format_rfc3339_microseconds(mtime),
_format_rfc3339_microseconds(depmtime),
)
changed.add(docname)
break
@@ -756,26 +758,6 @@ class BuildEnvironment:
self.events.emit('env-check-consistency', self)
def _last_modified_time(filename: str | os.PathLike[str]) -> int:
"""Return the last modified time of ``filename``.
The time is returned as integer microseconds.
The lowest common denominator of modern file-systems seems to be
microsecond-level precision.
We prefer to err on the side of re-rendering a file,
so we round up to the nearest microsecond.
"""
# upside-down floor division to get the ceiling
return -(os.stat(filename).st_mtime_ns // -1_000)
def _format_modified_time(timestamp: int) -> str:
"""Return an RFC 3339 formatted string representing the given timestamp."""
seconds, fraction = divmod(timestamp, 10**6)
return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(seconds)) + f'.{fraction // 1_000}'
def _traverse_toctree(
traversed: set[str],
parent: str | None,

3
sphinx/ext/viewcode.py
View File

@@ -21,6 +21,7 @@ from sphinx.transforms.post_transforms import SphinxPostTransform
from sphinx.util import logging
from sphinx.util.display import status_iterator
from sphinx.util.nodes import make_refnode
from sphinx.util.osutil import _last_modified_time
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
@@ -231,7 +232,7 @@ def should_generate_module_page(app: Sphinx, modname: str) -> bool:
page_filename = path.join(app.outdir, '_modules/', basename)
try:
if path.getmtime(module_filename) <= path.getmtime(page_filename):
if _last_modified_time(module_filename) <= _last_modified_time(page_filename):
# generation is not needed if the HTML page is newer than module file.
return False
except OSError:

15
sphinx/jinja2glue.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
import os
from os import path
from pprint import pformat
from typing import TYPE_CHECKING, Any
@@ -12,7 +13,7 @@ from jinja2.utils import open_if_exists, pass_context
from sphinx.application import TemplateBridge
from sphinx.util import logging
from sphinx.util.osutil import mtimes_of_files
from sphinx.util.osutil import _last_modified_time
if TYPE_CHECKING:
from collections.abc import Callable, Iterator
@@ -127,11 +128,11 @@ class SphinxFileSystemLoader(FileSystemLoader):
with f:
contents = f.read().decode(self.encoding)
mtime = path.getmtime(filename)
mtime = _last_modified_time(filename)
def uptodate() -> bool:
try:
return path.getmtime(filename) == mtime
return _last_modified_time(filename) == mtime
except OSError:
return False
@@ -203,7 +204,13 @@ class BuiltinTemplateLoader(TemplateBridge, BaseLoader):
return self.environment.from_string(source).render(context)
def newest_template_mtime(self) -> float:
return max(mtimes_of_files(self.pathchain, '.html'))
return max(
os.stat(os.path.join(root, sfile)).st_mtime_ns / 10**9
for dirname in self.pathchain
for root, _dirs, files in os.walk(dirname)
for sfile in files
if sfile.endswith('.html')
)
# Loader interface

2
sphinx/util/__init__.py
View File

@@ -29,10 +29,8 @@ from sphinx.util.nodes import ( # NoQA: F401
from sphinx.util.osutil import ( # NoQA: F401
SEP,
copyfile,
copytimes,
ensuredir,
make_filename,
mtimes_of_files,
os_path,
relative_uri,
)

12
sphinx/util/_timestamps.py Normal file
View File

@@ -0,0 +1,12 @@
from __future__ import annotations
import time
def _format_rfc3339_microseconds(timestamp: int, /) -> str:
"""Return an RFC 3339 formatted string representing the given timestamp.
:param timestamp: The timestamp to format, in microseconds.
"""
seconds, fraction = divmod(timestamp, 10**6)
return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(seconds)) + f'.{fraction // 1_000}'

9
sphinx/util/i18n.py
View File

@@ -15,7 +15,12 @@ from babel.messages.pofile import read_po
from sphinx.errors import SphinxError
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.osutil import SEP, canon_path, relpath
from sphinx.util.osutil import (
SEP,
_last_modified_time,
canon_path,
relpath,
)
if TYPE_CHECKING:
import datetime as dt
@@ -84,7 +89,7 @@ class CatalogInfo(LocaleFileInfoBase):
def is_outdated(self) -> bool:
return (
not path.exists(self.mo_path) or
path.getmtime(self.mo_path) < path.getmtime(self.po_path))
_last_modified_time(self.mo_path) < _last_modified_time(self.po_path))
def write_mo(self, locale: str, use_fuzzy: bool = False) -> None:
with open(self.po_path, encoding=self.charset) as file_po:

30
sphinx/util/osutil.py
View File

@@ -16,7 +16,6 @@ from typing import TYPE_CHECKING
from sphinx.locale import __
if TYPE_CHECKING:
from collections.abc import Iterator
from pathlib import Path
from types import TracebackType
from typing import Any
@@ -72,20 +71,25 @@ def ensuredir(file: str | os.PathLike[str]) -> None:
os.makedirs(file, exist_ok=True)
def mtimes_of_files(dirnames: list[str], suffix: str) -> Iterator[float]:
for dirname in dirnames:
for root, _dirs, files in os.walk(dirname):
for sfile in files:
if sfile.endswith(suffix):
with contextlib.suppress(OSError):
yield path.getmtime(path.join(root, sfile))
def _last_modified_time(source: str | os.PathLike[str], /) -> int:
"""Return the last modified time of ``filename``.
The time is returned as integer microseconds.
The lowest common denominator of modern file-systems seems to be
microsecond-level precision.
We prefer to err on the side of re-rendering a file,
so we round up to the nearest microsecond.
"""
st = source.stat() if isinstance(source, os.DirEntry) else os.stat(source)
# upside-down floor division to get the ceiling
return -(st.st_mtime_ns // -1_000)
def copytimes(source: str | os.PathLike[str], dest: str | os.PathLike[str]) -> None:
def _copy_times(source: str | os.PathLike[str], dest: str | os.PathLike[str]) -> None:
"""Copy a file's modification times."""
st = os.stat(source)
if hasattr(os, 'utime'):
os.utime(dest, (st.st_atime, st.st_mtime))
st = source.stat() if isinstance(source, os.DirEntry) else os.stat(source)
os.utime(dest, ns=(st.st_atime_ns, st.st_mtime_ns))
def copyfile(
@@ -128,7 +132,7 @@ def copyfile(
shutil.copyfile(source, dest)
with contextlib.suppress(OSError):
# don't do full copystat because the source may be read-only
copytimes(source, dest)
_copy_times(source, dest)
_no_fn_re = re.compile(r'[^a-zA-Z0-9_-]')

4
tests/test_extensions/test_ext_apidoc.py
View File

@@ -678,7 +678,7 @@ def test_remove_old_files(tmp_path: Path):
(gen_dir / 'other.rst').write_text('', encoding='utf8')
apidoc_main(['-o', str(gen_dir), str(module_dir)])
assert set(gen_dir.iterdir()) == {gen_dir / 'modules.rst', gen_dir / 'example.rst', gen_dir / 'other.rst'}
example_mtime = (gen_dir / 'example.rst').stat().st_mtime
example_mtime = (gen_dir / 'example.rst').stat().st_mtime_ns
apidoc_main(['--remove-old', '-o', str(gen_dir), str(module_dir)])
assert set(gen_dir.iterdir()) == {gen_dir / 'modules.rst', gen_dir / 'example.rst'}
assert (gen_dir / 'example.rst').stat().st_mtime == example_mtime
assert (gen_dir / 'example.rst').stat().st_mtime_ns == example_mtime

2
tests/test_intl/test_intl.py
View File

@@ -40,7 +40,7 @@ def write_mo(pathname, po):
return mofile.write_mo(f, po)
def _set_mtime_ns(target, value):
def _set_mtime_ns(target: str | os.PathLike[str], value: int) -> int:
os.utime(target, ns=(value, value))
return os.stat(target).st_mtime_ns

10
tests/test_util/test_util_docutils.py
View File

@@ -46,25 +46,25 @@ def test_SphinxFileOutput(tmpdir):
filename = str(tmpdir / 'test.txt')
output = SphinxFileOutput(destination_path=filename)
output.write(content)
os.utime(filename, (0, 0))
os.utime(filename, ns=(0, 0))
# overwrite it again
output.write(content)
assert os.stat(filename).st_mtime != 0 # updated
assert os.stat(filename).st_mtime_ns != 0 # updated
# write test2.txt at first
filename = str(tmpdir / 'test2.txt')
output = SphinxFileOutput(destination_path=filename, overwrite_if_changed=True)
output.write(content)
os.utime(filename, (0, 0))
os.utime(filename, ns=(0, 0))
# overwrite it again
output.write(content)
assert os.stat(filename).st_mtime == 0 # not updated
assert os.stat(filename).st_mtime_ns == 0 # not updated
# overwrite it again (content changed)
output.write(content + "; content change")
assert os.stat(filename).st_mtime != 0 # updated
assert os.stat(filename).st_mtime_ns != 0 # updated
def test_SphinxTranslator(app):

3
tests/test_util/test_util_i18n.py
View File

@@ -38,7 +38,8 @@ def test_catalog_outdated(tmp_path):
mo_file.write_text('#', encoding='utf8')
assert not cat.is_outdated() # if mo is exist and newer than po
os.utime(mo_file, (os.stat(mo_file).st_mtime - 10,) * 2) # to be outdate
new_mtime = os.stat(mo_file).st_mtime_ns - 10_000_000_000
os.utime(mo_file, ns=(new_mtime, new_mtime)) # to be outdated
assert cat.is_outdated() # if mo is exist and older than po