Infra: Prepare for moving PEPs to a subfolder (#3417)

.github/CODEOWNERS

@@ -12,7 +12,6 @@ requirements.txt        @AA-Turner
 infra/                  @ewdurbin
 
 pep_sphinx_extensions/  @AA-Turner
-AUTHOR_OVERRIDES.csv    @AA-Turner
 build.py                @AA-Turner
 conf.py                 @AA-Turner
 contents.rst            @AA-Turner

conf.py

@@ -3,10 +3,12 @@
 
 """Configuration for building PEPs using Sphinx."""
 
+import os
 from pathlib import Path
 import sys
 
-sys.path.append(str(Path(".").absolute()))
+_ROOT = Path(__file__).resolve().parent
+sys.path.append(os.fspath(_ROOT))
 
 # -- Project information -----------------------------------------------------
 
@@ -60,11 +62,13 @@ intersphinx_disabled_reftypes = []
 
 # -- Options for HTML output -------------------------------------------------
 
+_PSE_PATH = _ROOT / "pep_sphinx_extensions"
+
 # HTML output settings
 html_math_renderer = "maths_to_html"  # Maths rendering
 
 # Theme settings
-html_theme_path = ["pep_sphinx_extensions"]
+html_theme_path = [os.fspath(_PSE_PATH)]
 html_theme = "pep_theme"  # The actual theme directory (child of html_theme_path)
 html_use_index = False  # Disable index (we use PEP 0)
 html_style = ""  # must be defined here or in theme.conf, but is unused
@@ -72,4 +76,4 @@ html_permalinks = False  # handled in the PEPContents transform
 html_baseurl = "https://peps.python.org"  # to create the CNAME file
 gettext_auto_build = False  # speed-ups
 
-templates_path = ["pep_sphinx_extensions/pep_theme/templates"]  # Theme template relative paths from `confdir`
+templates_path = [os.fspath(_PSE_PATH / "pep_theme" / "templates")]  # Theme template relative paths from `confdir`

pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 from docutils import nodes
 from docutils.parsers import rst
 
-
 PYPA_SPEC_BASE_URL = "https://packaging.python.org/en/latest/specifications/"
 
 

pep_sphinx_extensions/pep_processor/transforms/pep_footer.py

@@ -1,4 +1,4 @@
-import datetime as dt
+import time
 from pathlib import Path
 import subprocess
 
@@ -23,7 +23,7 @@ class PEPFooter(transforms.Transform):
 
     def apply(self) -> None:
         pep_source_path = Path(self.document["source"])
-        if not pep_source_path.match("pep-*"):
+        if not pep_source_path.match("pep-????.???"):
             return  # not a PEP file, exit early
 
         # Iterate through sections from the end of the document
@@ -62,12 +62,10 @@ def _add_source_link(pep_source_path: Path) -> nodes.paragraph:
 def _add_commit_history_info(pep_source_path: Path) -> nodes.paragraph:
     """Use local git history to find last modified date."""
     try:
-        since_epoch = LAST_MODIFIED_TIMES[pep_source_path.name]
+        iso_time = _LAST_MODIFIED_TIMES[pep_source_path.stem]
     except KeyError:
         return nodes.paragraph()
 
-    epoch_dt = dt.datetime.fromtimestamp(since_epoch, dt.timezone.utc)
-    iso_time = epoch_dt.isoformat(sep=" ")
     commit_link = f"https://github.com/python/peps/commits/main/{pep_source_path.name}"
     link_node = nodes.reference("", f"{iso_time} GMT", refuri=commit_link)
     return nodes.paragraph("", "Last modified: ", link_node)
@@ -75,29 +73,33 @@ def _add_commit_history_info(pep_source_path: Path) -> nodes.paragraph:
 
 def _get_last_modified_timestamps():
     # get timestamps and changed files from all commits (without paging results)
-    args = ["git", "--no-pager", "log", "--format=#%at", "--name-only"]
+    args = ("git", "--no-pager", "log", "--format=#%at", "--name-only")
-    with subprocess.Popen(args, stdout=subprocess.PIPE) as process:
+    ret = subprocess.run(args, stdout=subprocess.PIPE, text=True, encoding="utf-8")
-        all_modified = process.stdout.read().decode("utf-8")
+    if ret.returncode:  # non-zero return code
-        process.stdout.close()
+        return {}
-        if process.wait():  # non-zero return code
+    all_modified = ret.stdout
-            return {}
 
     # set up the dictionary with the *current* files
-    last_modified = {path.name: 0 for path in Path().glob("pep-*") if path.suffix in {".txt", ".rst"}}
+    peps_dir = Path(__file__, "..", "..", "..", "..").resolve()
+    last_modified = {path.stem: "" for path in peps_dir.glob("pep-????.???") if path.suffix in {".txt", ".rst"}}
 
     # iterate through newest to oldest, updating per file timestamps
     change_sets = all_modified.removeprefix("#").split("#")
     for change_set in change_sets:
         timestamp, files = change_set.split("\n", 1)
         for file in files.strip().split("\n"):
-            if file.startswith("pep-") and file[-3:] in {"txt", "rst"}:
+            if not file.startswith("pep-") or not file.endswith((".rst", ".txt")):
-                if last_modified.get(file) == 0:
+                continue  # not a PEP
-                    try:
+            file = file[:-4]
-                        last_modified[file] = float(timestamp)
+            if last_modified.get(file) != "":
-                    except ValueError:
+                continue  # most recent modified date already found
-                        pass  # if float conversion fails
+            try:
+                y, m, d, hh, mm, ss, *_ = time.gmtime(float(timestamp))
+            except ValueError:
+                continue  # if float conversion fails
+            last_modified[file] = f"{y:04}-{m:02}-{d:02} {hh:02}:{mm:02}:{ss:02}"
 
     return last_modified
 
 
-LAST_MODIFIED_TIMES = _get_last_modified_timestamps()
+_LAST_MODIFIED_TIMES = _get_last_modified_timestamps()

pep_sphinx_extensions/pep_theme/templates/page.html

@@ -43,7 +43,7 @@
             <h2>Contents</h2>
             {{ toc }}
             <br>
-            {%- if not (sourcename.startswith("pep-0000") or sourcename.startswith("topic")) %}
+            {%- if not sourcename.startswith(("pep-0000", "topic")) %}
             <a id="source" href="https://github.com/python/peps/blob/main/{{sourcename}}">Page Source (GitHub)</a>
             {%- endif %}
         </nav>

pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py

@@ -18,22 +18,22 @@ to allow it to be processed as normal.
 from __future__ import annotations
 
 import json
+import os
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC
 from pep_sphinx_extensions.pep_zero_generator import parser
 from pep_sphinx_extensions.pep_zero_generator import subindices
 from pep_sphinx_extensions.pep_zero_generator import writer
+from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC
 
 if TYPE_CHECKING:
     from sphinx.application import Sphinx
     from sphinx.environment import BuildEnvironment
 
 
-def _parse_peps() -> list[parser.PEP]:
+def _parse_peps(path: Path) -> list[parser.PEP]:
     # Read from root directory
-    path = Path(".")
     peps: list[parser.PEP] = []
 
     for file_path in path.iterdir():
@@ -52,8 +52,16 @@ def create_pep_json(peps: list[parser.PEP]) -> str:
     return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1)
 
 
+def write_peps_json(peps: list[parser.PEP], path: Path) -> None:
+    # Create peps.json
+    json_peps = create_pep_json(peps)
+    Path(path, "peps.json").write_text(json_peps, encoding="utf-8")
+    os.makedirs(os.path.join(path, "api"), exist_ok=True)
+    Path(path, "api", "peps.json").write_text(json_peps, encoding="utf-8")
+
+
 def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None:
-    peps = _parse_peps()
+    peps = _parse_peps(Path(app.srcdir))
 
     pep0_text = writer.PEPZeroWriter().write_pep0(peps, builder=env.settings["builder"])
     pep0_path = subindices.update_sphinx("pep-0000", pep0_text, docnames, env)
@@ -61,7 +69,4 @@ def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) ->
 
     subindices.generate_subindices(SUBINDICES_BY_TOPIC, peps, docnames, env)
 
-    # Create peps.json
+    write_peps_json(peps, Path(app.outdir))
-    json_path = Path(app.outdir, "api", "peps.json").resolve()
-    json_path.parent.mkdir(exist_ok=True)
-    json_path.write_text(create_pep_json(peps), encoding="utf-8")

pep_sphinx_extensions/pep_zero_generator/subindices.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -14,8 +15,7 @@ if TYPE_CHECKING:
 
 
 def update_sphinx(filename: str, text: str, docnames: list[str], env: BuildEnvironment) -> Path:
-    file_path = Path(f"{filename}.rst").resolve()
+    file_path = Path(env.srcdir, f"{filename}.rst")
-    file_path.parent.mkdir(parents=True, exist_ok=True)
     file_path.write_text(text, encoding="utf-8")
 
     # Add to files for builder
@@ -32,6 +32,9 @@ def generate_subindices(
     docnames: list[str],
     env: BuildEnvironment,
 ) -> None:
+    # create topic directory
+    os.makedirs(os.path.join(env.srcdir, "topic"), exist_ok=True)
+
     # Create sub index page
     generate_topic_contents(docnames, env)
 

pep_sphinx_extensions/pep_zero_generator/writer.py

@@ -5,10 +5,8 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 import unicodedata
 
-from pep_sphinx_extensions.pep_processor.transforms.pep_headers import (
+from pep_sphinx_extensions.pep_processor.transforms.pep_headers import ABBREVIATED_STATUSES
-    ABBREVIATED_STATUSES,
+from pep_sphinx_extensions.pep_processor.transforms.pep_headers import ABBREVIATED_TYPES
-    ABBREVIATED_TYPES,
-)
 from pep_sphinx_extensions.pep_zero_generator.constants import DEAD_STATUSES
 from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACCEPTED
 from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACTIVE

pep_sphinx_extensions/tests/conftest.py

@@ -0,0 +1,3 @@
+from pathlib import Path
+
+PEP_ROOT = Path(__file__, "..", "..", "..").resolve()

pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py

@@ -1,16 +1,18 @@
-from pathlib import Path
+import datetime as dt
 
 from pep_sphinx_extensions.pep_processor.transforms import pep_footer
 
+from ...conftest import PEP_ROOT
+
 
 def test_add_source_link():
-    out = pep_footer._add_source_link(Path("pep-0008.txt"))
+    out = pep_footer._add_source_link(PEP_ROOT / "pep-0008.txt")
 
     assert "https://github.com/python/peps/blob/main/pep-0008.txt" in str(out)
 
 
 def test_add_commit_history_info():
-    out = pep_footer._add_commit_history_info(Path("pep-0008.txt"))
+    out = pep_footer._add_commit_history_info(PEP_ROOT / "pep-0008.txt")
 
     assert str(out).startswith(
         "<paragraph>Last modified: "
@@ -21,7 +23,7 @@ def test_add_commit_history_info():
 
 
 def test_add_commit_history_info_invalid():
-    out = pep_footer._add_commit_history_info(Path("pep-not-found.txt"))
+    out = pep_footer._add_commit_history_info(PEP_ROOT / "pep-not-found.rst")
 
     assert str(out) == "<paragraph/>"
 
@@ -31,4 +33,4 @@ def test_get_last_modified_timestamps():
 
     assert len(out) >= 585
     # Should be a Unix timestamp and at least this
-    assert out["pep-0008.txt"] >= 1643124055
+    assert dt.datetime.fromisoformat(out["pep-0008"]).timestamp() >= 1643124055

pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py

@@ -1,5 +1,3 @@
-from pathlib import Path
-
 import pytest
 
 from pep_sphinx_extensions.pep_zero_generator import parser
@@ -19,29 +17,31 @@ from pep_sphinx_extensions.pep_zero_generator.constants import (
 )
 from pep_sphinx_extensions.pep_zero_generator.parser import _Author
 
+from ..conftest import PEP_ROOT
+
 
 def test_pep_repr():
-    pep8 = parser.PEP(Path("pep-0008.txt"))
+    pep8 = parser.PEP(PEP_ROOT / "pep-0008.txt")
 
     assert repr(pep8) == "<PEP 0008 - Style Guide for Python Code>"
 
 
 def test_pep_less_than():
-    pep8 = parser.PEP(Path("pep-0008.txt"))
+    pep8 = parser.PEP(PEP_ROOT / "pep-0008.txt")
-    pep3333 = parser.PEP(Path("pep-3333.txt"))
+    pep3333 = parser.PEP(PEP_ROOT / "pep-3333.txt")
 
     assert pep8 < pep3333
 
 
 def test_pep_equal():
-    pep_a = parser.PEP(Path("pep-0008.txt"))
+    pep_a = parser.PEP(PEP_ROOT / "pep-0008.txt")
-    pep_b = parser.PEP(Path("pep-0008.txt"))
+    pep_b = parser.PEP(PEP_ROOT / "pep-0008.txt")
 
     assert pep_a == pep_b
 
 
 def test_pep_details(monkeypatch):
-    pep8 = parser.PEP(Path("pep-0008.txt"))
+    pep8 = parser.PEP(PEP_ROOT / "pep-0008.txt")
 
     assert pep8.details == {
         "authors": "Guido van Rossum, Barry Warsaw, Nick Coghlan",
@@ -106,7 +106,7 @@ def test_parse_authors_invalid():
 )
 def test_abbreviate_type_status(test_type, test_status, expected):
     # set up dummy PEP object and monkeypatch attributes
-    pep = parser.PEP(Path("pep-0008.txt"))
+    pep = parser.PEP(PEP_ROOT / "pep-0008.txt")
     pep.pep_type = test_type
     pep.status = test_status
 

pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py

@@ -1,10 +1,10 @@
-from pathlib import Path
-
 from pep_sphinx_extensions.pep_zero_generator import parser, pep_index_generator
 
+from ..conftest import PEP_ROOT
+
 
 def test_create_pep_json():
-    peps = [parser.PEP(Path("pep-0008.txt"))]
+    peps = [parser.PEP(PEP_ROOT / "pep-0008.txt")]
 
     out = pep_index_generator.create_pep_json(peps)