[apidoc] Add --remove-old option (#12448)

A common "gotcha" of re-running `sphinx-apidoc`, is that if the modules API changes it will not remove old files, leading to build errors for files not in a `toctree`

This commit introduces a `--remove-old` option to remove these files.

Note, a key detail here is that we don't want to simply clear the directory before running `sphinx-apidoc`,
since this would lead to all files having a new `mtime`,
and then `sphinx-build` would rebuild all of them even if they have not changed.
So we must first collect the list of all correct files, then remove any not in the list.

The commit also improves some typing of the code and replace `os.path` by `pathlib.Path` in most instances

tests/test_extensions/test_ext_apidoc.py

@@ -2,6 +2,7 @@
 
 import os.path
 from collections import namedtuple
+from pathlib import Path
 
 import pytest
 
@@ -661,3 +662,23 @@ def test_no_duplicates(rootdir, tmp_path):
 
     finally:
         sphinx.ext.apidoc.PY_SUFFIXES = original_suffixes
+
+
+def test_remove_old_files(tmp_path: Path):
+    """Test that old files are removed when using the -r option.
+
+    Also ensure that pre-existing files are not re-written, if unchanged.
+    This is required to avoid unnecessary rebuilds.
+    """
+    module_dir = tmp_path / 'module'
+    module_dir.mkdir()
+    (module_dir / 'example.py').write_text('', encoding='utf8')
+    gen_dir = tmp_path / 'gen'
+    gen_dir.mkdir()
+    (gen_dir / 'other.rst').write_text('', encoding='utf8')
+    apidoc_main(['-o', str(gen_dir), str(module_dir)])
+    assert set(gen_dir.iterdir()) == {gen_dir / 'modules.rst', gen_dir / 'example.rst', gen_dir / 'other.rst'}
+    example_mtime = (gen_dir / 'example.rst').stat().st_mtime
+    apidoc_main(['--remove-old', '-o', str(gen_dir), str(module_dir)])
+    assert set(gen_dir.iterdir()) == {gen_dir / 'modules.rst', gen_dir / 'example.rst'}
+    assert (gen_dir / 'example.rst').stat().st_mtime == example_mtime