From 2d4c98dca777fceb88393c2635dca44d49cf7380 Mon Sep 17 00:00:00 2001
From: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Date: Sun, 3 Sep 2023 04:55:19 +0100
Subject: [PATCH] Infra: Prepare for moving PEPs to a subfolder (#3417)
---
.github/CODEOWNERS | 1 -
conf.py | 10 +++--
.../parsing/pep_banner_directive.py | 1 -
.../pep_processor/transforms/pep_footer.py | 40 ++++++++++---------
.../pep_theme/templates/page.html | 2 +-
.../pep_zero_generator/pep_index_generator.py | 21 ++++++----
.../pep_zero_generator/subindices.py | 7 +++-
.../pep_zero_generator/writer.py | 6 +--
pep_sphinx_extensions/tests/conftest.py | 3 ++
.../transform/test_pep_footer.py | 12 +++---
.../tests/pep_zero_generator/test_parser.py | 18 ++++-----
.../test_pep_index_generator.py | 6 +--
12 files changed, 71 insertions(+), 56 deletions(-)
create mode 100644 pep_sphinx_extensions/tests/conftest.py
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 4602dbe7b..5b02ed791 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -12,7 +12,6 @@ requirements.txt @AA-Turner
infra/ @ewdurbin
pep_sphinx_extensions/ @AA-Turner
-AUTHOR_OVERRIDES.csv @AA-Turner
build.py @AA-Turner
conf.py @AA-Turner
contents.rst @AA-Turner
diff --git a/conf.py b/conf.py
index 95a1debd4..b795aa874 100644
--- a/conf.py
+++ b/conf.py
@@ -3,10 +3,12 @@
"""Configuration for building PEPs using Sphinx."""
+import os
from pathlib import Path
import sys
-sys.path.append(str(Path(".").absolute()))
+_ROOT = Path(__file__).resolve().parent
+sys.path.append(os.fspath(_ROOT))
# -- Project information -----------------------------------------------------
@@ -60,11 +62,13 @@ intersphinx_disabled_reftypes = []
# -- Options for HTML output -------------------------------------------------
+_PSE_PATH = _ROOT / "pep_sphinx_extensions"
+
# HTML output settings
html_math_renderer = "maths_to_html" # Maths rendering
# Theme settings
-html_theme_path = ["pep_sphinx_extensions"]
+html_theme_path = [os.fspath(_PSE_PATH)]
html_theme = "pep_theme" # The actual theme directory (child of html_theme_path)
html_use_index = False # Disable index (we use PEP 0)
html_style = "" # must be defined here or in theme.conf, but is unused
@@ -72,4 +76,4 @@ html_permalinks = False # handled in the PEPContents transform
html_baseurl = "https://peps.python.org" # to create the CNAME file
gettext_auto_build = False # speed-ups
-templates_path = ["pep_sphinx_extensions/pep_theme/templates"] # Theme template relative paths from `confdir`
+templates_path = [os.fspath(_PSE_PATH / "pep_theme" / "templates")] # Theme template relative paths from `confdir`
diff --git a/pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py b/pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py
index f3a55270c..d0f1cbee2 100644
--- a/pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py
+++ b/pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py
@@ -5,7 +5,6 @@ from __future__ import annotations
from docutils import nodes
from docutils.parsers import rst
-
PYPA_SPEC_BASE_URL = "https://packaging.python.org/en/latest/specifications/"
diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py b/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py
index 7b9c29d51..efd94ca36 100644
--- a/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py
+++ b/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py
@@ -1,4 +1,4 @@
-import datetime as dt
+import time
from pathlib import Path
import subprocess
@@ -23,7 +23,7 @@ class PEPFooter(transforms.Transform):
def apply(self) -> None:
pep_source_path = Path(self.document["source"])
- if not pep_source_path.match("pep-*"):
+ if not pep_source_path.match("pep-????.???"):
return # not a PEP file, exit early
# Iterate through sections from the end of the document
@@ -62,12 +62,10 @@ def _add_source_link(pep_source_path: Path) -> nodes.paragraph:
def _add_commit_history_info(pep_source_path: Path) -> nodes.paragraph:
"""Use local git history to find last modified date."""
try:
- since_epoch = LAST_MODIFIED_TIMES[pep_source_path.name]
+ iso_time = _LAST_MODIFIED_TIMES[pep_source_path.stem]
except KeyError:
return nodes.paragraph()
- epoch_dt = dt.datetime.fromtimestamp(since_epoch, dt.timezone.utc)
- iso_time = epoch_dt.isoformat(sep=" ")
commit_link = f"https://github.com/python/peps/commits/main/{pep_source_path.name}"
link_node = nodes.reference("", f"{iso_time} GMT", refuri=commit_link)
return nodes.paragraph("", "Last modified: ", link_node)
@@ -75,29 +73,33 @@ def _add_commit_history_info(pep_source_path: Path) -> nodes.paragraph:
def _get_last_modified_timestamps():
# get timestamps and changed files from all commits (without paging results)
- args = ["git", "--no-pager", "log", "--format=#%at", "--name-only"]
- with subprocess.Popen(args, stdout=subprocess.PIPE) as process:
- all_modified = process.stdout.read().decode("utf-8")
- process.stdout.close()
- if process.wait(): # non-zero return code
- return {}
+ args = ("git", "--no-pager", "log", "--format=#%at", "--name-only")
+ ret = subprocess.run(args, stdout=subprocess.PIPE, text=True, encoding="utf-8")
+ if ret.returncode: # non-zero return code
+ return {}
+ all_modified = ret.stdout
# set up the dictionary with the *current* files
- last_modified = {path.name: 0 for path in Path().glob("pep-*") if path.suffix in {".txt", ".rst"}}
+ peps_dir = Path(__file__, "..", "..", "..", "..").resolve()
+ last_modified = {path.stem: "" for path in peps_dir.glob("pep-????.???") if path.suffix in {".txt", ".rst"}}
# iterate through newest to oldest, updating per file timestamps
change_sets = all_modified.removeprefix("#").split("#")
for change_set in change_sets:
timestamp, files = change_set.split("\n", 1)
for file in files.strip().split("\n"):
- if file.startswith("pep-") and file[-3:] in {"txt", "rst"}:
- if last_modified.get(file) == 0:
- try:
- last_modified[file] = float(timestamp)
- except ValueError:
- pass # if float conversion fails
+ if not file.startswith("pep-") or not file.endswith((".rst", ".txt")):
+ continue # not a PEP
+ file = file[:-4]
+ if last_modified.get(file) != "":
+ continue # most recent modified date already found
+ try:
+ y, m, d, hh, mm, ss, *_ = time.gmtime(float(timestamp))
+ except ValueError:
+ continue # if float conversion fails
+ last_modified[file] = f"{y:04}-{m:02}-{d:02} {hh:02}:{mm:02}:{ss:02}"
return last_modified
-LAST_MODIFIED_TIMES = _get_last_modified_timestamps()
+_LAST_MODIFIED_TIMES = _get_last_modified_timestamps()
diff --git a/pep_sphinx_extensions/pep_theme/templates/page.html b/pep_sphinx_extensions/pep_theme/templates/page.html
index 2831fce0c..8bd879c0f 100644
--- a/pep_sphinx_extensions/pep_theme/templates/page.html
+++ b/pep_sphinx_extensions/pep_theme/templates/page.html
@@ -43,7 +43,7 @@
Contents
{{ toc }}
- {%- if not (sourcename.startswith("pep-0000") or sourcename.startswith("topic")) %}
+ {%- if not sourcename.startswith(("pep-0000", "topic")) %}
Page Source (GitHub)
{%- endif %}
diff --git a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py
index e2c1a7963..0804b4aa6 100644
--- a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py
+++ b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py
@@ -18,22 +18,22 @@ to allow it to be processed as normal.
from __future__ import annotations
import json
+import os
from pathlib import Path
from typing import TYPE_CHECKING
-from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC
from pep_sphinx_extensions.pep_zero_generator import parser
from pep_sphinx_extensions.pep_zero_generator import subindices
from pep_sphinx_extensions.pep_zero_generator import writer
+from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC
if TYPE_CHECKING:
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
-def _parse_peps() -> list[parser.PEP]:
+def _parse_peps(path: Path) -> list[parser.PEP]:
# Read from root directory
- path = Path(".")
peps: list[parser.PEP] = []
for file_path in path.iterdir():
@@ -52,8 +52,16 @@ def create_pep_json(peps: list[parser.PEP]) -> str:
return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1)
+def write_peps_json(peps: list[parser.PEP], path: Path) -> None:
+ # Create peps.json
+ json_peps = create_pep_json(peps)
+ Path(path, "peps.json").write_text(json_peps, encoding="utf-8")
+ os.makedirs(os.path.join(path, "api"), exist_ok=True)
+ Path(path, "api", "peps.json").write_text(json_peps, encoding="utf-8")
+
+
def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None:
- peps = _parse_peps()
+ peps = _parse_peps(Path(app.srcdir))
pep0_text = writer.PEPZeroWriter().write_pep0(peps, builder=env.settings["builder"])
pep0_path = subindices.update_sphinx("pep-0000", pep0_text, docnames, env)
@@ -61,7 +69,4 @@ def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) ->
subindices.generate_subindices(SUBINDICES_BY_TOPIC, peps, docnames, env)
- # Create peps.json
- json_path = Path(app.outdir, "api", "peps.json").resolve()
- json_path.parent.mkdir(exist_ok=True)
- json_path.write_text(create_pep_json(peps), encoding="utf-8")
+ write_peps_json(peps, Path(app.outdir))
diff --git a/pep_sphinx_extensions/pep_zero_generator/subindices.py b/pep_sphinx_extensions/pep_zero_generator/subindices.py
index 455a49dd8..3f61b3dd4 100644
--- a/pep_sphinx_extensions/pep_zero_generator/subindices.py
+++ b/pep_sphinx_extensions/pep_zero_generator/subindices.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import os
from pathlib import Path
from typing import TYPE_CHECKING
@@ -14,8 +15,7 @@ if TYPE_CHECKING:
def update_sphinx(filename: str, text: str, docnames: list[str], env: BuildEnvironment) -> Path:
- file_path = Path(f"{filename}.rst").resolve()
- file_path.parent.mkdir(parents=True, exist_ok=True)
+ file_path = Path(env.srcdir, f"{filename}.rst")
file_path.write_text(text, encoding="utf-8")
# Add to files for builder
@@ -32,6 +32,9 @@ def generate_subindices(
docnames: list[str],
env: BuildEnvironment,
) -> None:
+ # create topic directory
+ os.makedirs(os.path.join(env.srcdir, "topic"), exist_ok=True)
+
# Create sub index page
generate_topic_contents(docnames, env)
diff --git a/pep_sphinx_extensions/pep_zero_generator/writer.py b/pep_sphinx_extensions/pep_zero_generator/writer.py
index 02af0c8bd..69a5fe4bc 100644
--- a/pep_sphinx_extensions/pep_zero_generator/writer.py
+++ b/pep_sphinx_extensions/pep_zero_generator/writer.py
@@ -5,10 +5,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import unicodedata
-from pep_sphinx_extensions.pep_processor.transforms.pep_headers import (
- ABBREVIATED_STATUSES,
- ABBREVIATED_TYPES,
-)
+from pep_sphinx_extensions.pep_processor.transforms.pep_headers import ABBREVIATED_STATUSES
+from pep_sphinx_extensions.pep_processor.transforms.pep_headers import ABBREVIATED_TYPES
from pep_sphinx_extensions.pep_zero_generator.constants import DEAD_STATUSES
from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACCEPTED
from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACTIVE
diff --git a/pep_sphinx_extensions/tests/conftest.py b/pep_sphinx_extensions/tests/conftest.py
new file mode 100644
index 000000000..e1417e08e
--- /dev/null
+++ b/pep_sphinx_extensions/tests/conftest.py
@@ -0,0 +1,3 @@
+from pathlib import Path
+
+PEP_ROOT = Path(__file__, "..", "..", "..").resolve()
diff --git a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py
index ad8cf2782..6bd0cdca9 100644
--- a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py
+++ b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py
@@ -1,16 +1,18 @@
-from pathlib import Path
+import datetime as dt
from pep_sphinx_extensions.pep_processor.transforms import pep_footer
+from ...conftest import PEP_ROOT
+
def test_add_source_link():
- out = pep_footer._add_source_link(Path("pep-0008.txt"))
+ out = pep_footer._add_source_link(PEP_ROOT / "pep-0008.txt")
assert "https://github.com/python/peps/blob/main/pep-0008.txt" in str(out)
def test_add_commit_history_info():
- out = pep_footer._add_commit_history_info(Path("pep-0008.txt"))
+ out = pep_footer._add_commit_history_info(PEP_ROOT / "pep-0008.txt")
assert str(out).startswith(
"Last modified: "
@@ -21,7 +23,7 @@ def test_add_commit_history_info():
def test_add_commit_history_info_invalid():
- out = pep_footer._add_commit_history_info(Path("pep-not-found.txt"))
+ out = pep_footer._add_commit_history_info(PEP_ROOT / "pep-not-found.rst")
assert str(out) == ""
@@ -31,4 +33,4 @@ def test_get_last_modified_timestamps():
assert len(out) >= 585
# Should be a Unix timestamp and at least this
- assert out["pep-0008.txt"] >= 1643124055
+ assert dt.datetime.fromisoformat(out["pep-0008"]).timestamp() >= 1643124055
diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py
index 2cba74df1..cea1d61ab 100644
--- a/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py
+++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py
@@ -1,5 +1,3 @@
-from pathlib import Path
-
import pytest
from pep_sphinx_extensions.pep_zero_generator import parser
@@ -19,29 +17,31 @@ from pep_sphinx_extensions.pep_zero_generator.constants import (
)
from pep_sphinx_extensions.pep_zero_generator.parser import _Author
+from ..conftest import PEP_ROOT
+
def test_pep_repr():
- pep8 = parser.PEP(Path("pep-0008.txt"))
+ pep8 = parser.PEP(PEP_ROOT / "pep-0008.txt")
assert repr(pep8) == ""
def test_pep_less_than():
- pep8 = parser.PEP(Path("pep-0008.txt"))
- pep3333 = parser.PEP(Path("pep-3333.txt"))
+ pep8 = parser.PEP(PEP_ROOT / "pep-0008.txt")
+ pep3333 = parser.PEP(PEP_ROOT / "pep-3333.txt")
assert pep8 < pep3333
def test_pep_equal():
- pep_a = parser.PEP(Path("pep-0008.txt"))
- pep_b = parser.PEP(Path("pep-0008.txt"))
+ pep_a = parser.PEP(PEP_ROOT / "pep-0008.txt")
+ pep_b = parser.PEP(PEP_ROOT / "pep-0008.txt")
assert pep_a == pep_b
def test_pep_details(monkeypatch):
- pep8 = parser.PEP(Path("pep-0008.txt"))
+ pep8 = parser.PEP(PEP_ROOT / "pep-0008.txt")
assert pep8.details == {
"authors": "Guido van Rossum, Barry Warsaw, Nick Coghlan",
@@ -106,7 +106,7 @@ def test_parse_authors_invalid():
)
def test_abbreviate_type_status(test_type, test_status, expected):
# set up dummy PEP object and monkeypatch attributes
- pep = parser.PEP(Path("pep-0008.txt"))
+ pep = parser.PEP(PEP_ROOT / "pep-0008.txt")
pep.pep_type = test_type
pep.status = test_status
diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py
index c2e15844f..e920d9773 100644
--- a/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py
+++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py
@@ -1,10 +1,10 @@
-from pathlib import Path
-
from pep_sphinx_extensions.pep_zero_generator import parser, pep_index_generator
+from ..conftest import PEP_ROOT
+
def test_create_pep_json():
- peps = [parser.PEP(Path("pep-0008.txt"))]
+ peps = [parser.PEP(PEP_ROOT / "pep-0008.txt")]
out = pep_index_generator.create_pep_json(peps)