Fix #8952: Exceptions raised in a Directive cause parallel builds to hang

Do shutdown explicitly when failure on the subprocess.

CHANGES

@@ -16,6 +16,8 @@ Features added
 Bugs fixed
 ----------
 
+* #8952: Exceptions raised in a Directive cause parallel builds to hang
+
 Testing
 --------
 

sphinx/util/parallel.py

@@ -103,8 +103,21 @@ class ParallelTasks:
         self._join_one()
 
     def join(self) -> None:
-        while self._pworking:
-            self._join_one()
+        try:
+            while self._pworking:
+                self._join_one()
+        except Exception:
+            # shutdown other child processes on failure
+            self.terminate()
+            raise
+
+    def terminate(self) -> None:
+        for tid in list(self._precvs):
+            self._procs[tid].terminate()
+            self._result_funcs.pop(tid)
+            self._procs.pop(tid)
+            self._precvs.pop(tid)
+            self._pworking -= 1
 
     def _join_one(self) -> None:
         for tid, pipe in self._precvs.items():