Add support for topic indices (#2579)

Co-authored-by: Łukasz Langa <lukasz@langa.pl>

.pre-commit-config.yaml

@@ -176,6 +176,13 @@ repos:
         files: '^pep-\d+\.(rst|txt)$'
         types: [text]
 
+      - id: validate-topic
+        name: "'Topic' must be for a valid sub-index"
+        language: pygrep
+        entry: '^Topic:(?:(?! +(Packaging|Typing|Packaging, Typing)$))'
+        files: '^pep-\d+\.(rst|txt)$'
+        types: [text]
+
       - id: validate-content-type
         name: "'Content-Type' must be 'text/x-rst'"
         language: pygrep

contents.rst

@@ -16,3 +16,4 @@ This is an internal Sphinx page; please go to the :doc:`PEP Index <pep-0000>`.
 
    docs/*
    pep-*
+   topic/*

pep_sphinx_extensions/pep_processor/html/pep_html_builder.py

@@ -36,7 +36,7 @@ class FileBuilder(StandaloneHTMLBuilder):
 
         # local table of contents
         toc_tree = self.env.tocs[docname].deepcopy()
-        if len(toc_tree[0]) > 1:
+        if len(toc_tree) and len(toc_tree[0]) > 1:
             toc_tree = toc_tree[0][1]  # don't include document title
             del toc_tree[0]  # remove contents node
             for node in toc_tree.findall(nodes.reference):

pep_sphinx_extensions/pep_zero_generator/constants.py

@@ -32,3 +32,17 @@ TYPE_STANDARDS = "Standards Track"
 TYPE_VALUES = {TYPE_STANDARDS, TYPE_INFO, TYPE_PROCESS}
 # Active PEPs can only be for Informational or Process PEPs.
 ACTIVE_ALLOWED = {TYPE_PROCESS, TYPE_INFO}
+
+# map of topic -> additional description
+SUBINDICES_BY_TOPIC = {
+    "packaging": """\
+The canonical, up-to-date packaging specifications can be found on the
+`Python Packaging Authority`_ (PyPA) `specifications`_ page.
+Packaging PEPs follow the `PyPA specification update process`_.
+They are used to propose major additions or changes to the PyPA specifications.
+
+.. _Python Packaging Authority: https://www.pypa.io/
+.. _specifications: https://packaging.python.org/en/latest/specifications/
+.. _PyPA specification update process: https://www.pypa.io/en/latest/specifications/#specification-update-process
+""",
+}

pep_sphinx_extensions/pep_zero_generator/parser.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import csv
 from email.parser import HeaderParser
 from pathlib import Path
 import re
@@ -22,6 +23,14 @@ if TYPE_CHECKING:
     from pep_sphinx_extensions.pep_zero_generator.author import Author
 
 
+# AUTHOR_OVERRIDES.csv is an exception file for PEP 0 name parsing
+AUTHOR_OVERRIDES: dict[str, dict[str, str]] = {}
+with open("AUTHOR_OVERRIDES.csv", "r", encoding="utf-8") as f:
+    for line in csv.DictReader(f):
+        full_name = line.pop("Overridden Name")
+        AUTHOR_OVERRIDES[full_name] = line
+
+
 class PEP:
     """Representation of PEPs.
 
@@ -37,7 +46,7 @@ class PEP:
     # The required RFC 822 headers for all PEPs.
     required_headers = {"PEP", "Title", "Author", "Status", "Type", "Created"}
 
-    def __init__(self, filename: Path, authors_overrides: dict):
+    def __init__(self, filename: Path):
         """Init object from an open PEP file object.
 
         pep_file is full text of the PEP file, filename is path of the PEP file, author_lookup is author exceptions file
@@ -88,7 +97,11 @@ class PEP:
         self.status: str = status
 
         # Parse PEP authors
-        self.authors: list[Author] = _parse_authors(self, metadata["Author"], authors_overrides)
+        self.authors: list[Author] = _parse_authors(self, metadata["Author"], AUTHOR_OVERRIDES)
+
+        # Topic (for sub-indices)
+        _topic = metadata.get("Topic", "").lower().split(",")
+        self.topic: set[str] = {topic for topic_raw in _topic if (topic := topic_raw.strip())}
 
         # Other headers
         self.created = metadata["Created"]
@@ -136,6 +149,7 @@ class PEP:
             "discussions_to": self.discussions_to,
             "status": self.status,
             "type": self.pep_type,
+            "topic": ", ".join(sorted(self.topic)),
             "created": self.created,
             "python_version": self.python_version,
             "post_history": self.post_history,

pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py

@@ -17,13 +17,13 @@ to allow it to be processed as normal.
 """
 from __future__ import annotations
 
-import csv
 import json
 from pathlib import Path
-import re
 from typing import TYPE_CHECKING
 
+from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC
 from pep_sphinx_extensions.pep_zero_generator import parser
+from pep_sphinx_extensions.pep_zero_generator import subindices
 from pep_sphinx_extensions.pep_zero_generator import writer
 
 if TYPE_CHECKING:
@@ -31,46 +31,35 @@ if TYPE_CHECKING:
     from sphinx.environment import BuildEnvironment
 
 
-def create_pep_json(peps: list[parser.PEP]) -> str:
+def _parse_peps() -> list[parser.PEP]:
-    return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1)
-
-
-def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None:
     # Read from root directory
     path = Path(".")
-
-    pep_zero_filename = "pep-0000"
     peps: list[parser.PEP] = []
-    pep_pat = re.compile(r"pep-\d{4}")  # Path.match() doesn't support regular expressions
-
-    # AUTHOR_OVERRIDES.csv is an exception file for PEP0 name parsing
-    with open("AUTHOR_OVERRIDES.csv", "r", encoding="utf-8") as f:
-        authors_overrides = {}
-        for line in csv.DictReader(f):
-            full_name = line.pop("Overridden Name")
-            authors_overrides[full_name] = line
 
     for file_path in path.iterdir():
         if not file_path.is_file():
             continue  # Skip directories etc.
         if file_path.match("pep-0000*"):
             continue  # Skip pre-existing PEP 0 files
-        if pep_pat.match(str(file_path)) and file_path.suffix in {".txt", ".rst"}:
+        if file_path.match("pep-????.???") and file_path.suffix in {".txt", ".rst"}:
-            pep = parser.PEP(path.joinpath(file_path).absolute(), authors_overrides)
+            pep = parser.PEP(path.joinpath(file_path).absolute())
             peps.append(pep)
 
-    peps = sorted(peps)
+    return sorted(peps)
+
+
+def create_pep_json(peps: list[parser.PEP]) -> str:
+    return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1)
+
+
+def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None:
+    peps = _parse_peps()
 
     pep0_text = writer.PEPZeroWriter().write_pep0(peps)
-    pep0_path = Path(f"{pep_zero_filename}.rst")
+    pep0_path = subindices.update_sphinx("pep-0000", pep0_text, docnames, env)
-    pep0_path.write_text(pep0_text, encoding="utf-8")
+    peps.append(parser.PEP(pep0_path))
 
-    peps.append(parser.PEP(pep0_path, authors_overrides))
+    subindices.generate_subindices(SUBINDICES_BY_TOPIC, peps, docnames, env)
-
-    # Add to files for builder
-    docnames.insert(1, pep_zero_filename)
-    # Add to files for writer
-    env.found_docs.add(pep_zero_filename)
 
     # Create peps.json
     json_path = Path(app.outdir, "api", "peps.json").resolve()

pep_sphinx_extensions/pep_zero_generator/subindices.py

@@ -0,0 +1,71 @@
+"""Utilities to support sub-indices for PEPs."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pep_sphinx_extensions.pep_zero_generator import writer
+
+if TYPE_CHECKING:
+    from sphinx.environment import BuildEnvironment
+
+    from pep_sphinx_extensions.pep_zero_generator.parser import PEP
+
+
+def update_sphinx(filename: str, text: str, docnames: list[str], env: BuildEnvironment) -> Path:
+    file_path = Path(f"{filename}.rst").resolve()
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    file_path.write_text(text, encoding="utf-8")
+
+    # Add to files for builder
+    docnames.append(filename)
+    # Add to files for writer
+    env.found_docs.add(filename)
+
+    return file_path
+
+
+def generate_subindices(
+    subindices: dict[str, str],
+    peps: list[PEP],
+    docnames: list[str],
+    env: BuildEnvironment,
+) -> None:
+    # Create sub index page
+    generate_topic_contents(docnames, env)
+
+    for subindex, additional_description in subindices.items():
+        header_text = f"{subindex.title()} PEPs"
+        header_line = "#" * len(header_text)
+        header = header_text + "\n" + header_line + "\n"
+
+        topic = subindex.lower()
+        filtered_peps = [pep for pep in peps if topic in pep.topic]
+        subindex_intro = f"""\
+This is the index of all Python Enhancement Proposals (PEPs) labelled
+under the '{subindex.title()}' topic. This is a sub-index of :pep:`0`,
+the PEP index.
+
+{additional_description}
+"""
+        subindex_text = writer.PEPZeroWriter().write_pep0(
+            filtered_peps, header, subindex_intro, is_pep0=False,
+        )
+        update_sphinx(f"topic/{subindex}", subindex_text, docnames, env)
+
+
+def generate_topic_contents(docnames: list[str], env: BuildEnvironment):
+    update_sphinx(f"topic/index", """\
+Topic Index
+***********
+
+PEPs are indexed by topic on the pages below:
+
+.. toctree::
+   :maxdepth: 1
+   :titlesonly:
+   :glob:
+
+   *
+""", docnames, env)

pep_sphinx_extensions/pep_zero_generator/writer.py

@@ -25,7 +25,7 @@ from pep_sphinx_extensions.pep_zero_generator.errors import PEPError
 if TYPE_CHECKING:
     from pep_sphinx_extensions.pep_zero_generator.parser import PEP
 
-header = f"""\
+HEADER = f"""\
 PEP: 0
 Title: Index of Python Enhancement Proposals (PEPs)
 Last-Modified: {datetime.date.today()}
@@ -36,12 +36,13 @@ Content-Type: text/x-rst
 Created: 13-Jul-2000
 """
 
-intro = """\
+INTRO = """\
 This PEP contains the index of all Python Enhancement Proposals,
 known as PEPs.  PEP numbers are :pep:`assigned <1#pep-editors>`
 by the PEP editors, and once assigned are never changed.  The
 `version control history <https://github.com/python/peps>`_ of
-the PEP texts represent their historical record.
+the PEP texts represent their historical record.  The PEPs are
+:doc:`indexed by topic <topic/index>` for specialist subjects.
 """
 
 
@@ -112,7 +113,9 @@ class PEPZeroWriter:
             self.emit_text("     -")
         self.emit_newline()
 
-    def write_pep0(self, peps: list[PEP]):
+    def write_pep0(self, peps: list[PEP], header: str = HEADER, intro: str = INTRO, is_pep0: bool = True):
+        if len(peps) == 0:
+            return ""
 
         # PEP metadata
         self.emit_text(header)
@@ -138,7 +141,10 @@ class PEPZeroWriter:
             ("Abandoned, Withdrawn, and Rejected PEPs", dead),
         ]
         for (category, peps_in_category) in pep_categories:
-            self.emit_pep_category(category, peps_in_category)
+            # For sub-indices, only emit categories with entries.
+            # For PEP 0, emit every category
+            if is_pep0 or len(peps_in_category) > 0:
+                self.emit_pep_category(category, peps_in_category)
 
         self.emit_newline()
 
@@ -151,12 +157,14 @@ class PEPZeroWriter:
         self.emit_newline()
 
         # Reserved PEP numbers
-        self.emit_title("Reserved PEP Numbers")
+        if is_pep0:
-        self.emit_column_headers()
+            self.emit_title("Reserved PEP Numbers")
-        for number, claimants in sorted(self.RESERVED.items()):
+            self.emit_column_headers()
-            self.emit_pep_row(type="", status="", number=number, title="RESERVED", authors=claimants)
+            for number, claimants in sorted(self.RESERVED.items()):
+                self.emit_pep_row(type="", status="", number=number, title="RESERVED", authors=claimants)
 
-        self.emit_newline()
+
+            self.emit_newline()
 
         # PEP types key
         self.emit_title("PEP Types Key")

pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py

@@ -9,27 +9,27 @@ from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES
 
 
 def test_pep_repr():
-    pep8 = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)
+    pep8 = parser.PEP(Path("pep-0008.txt"))
 
     assert repr(pep8) == "<PEP 0008 - Style Guide for Python Code>"
 
 
 def test_pep_less_than():
-    pep8 = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)
+    pep8 = parser.PEP(Path("pep-0008.txt"))
-    pep3333 = parser.PEP(Path("pep-3333.txt"), AUTHORS_OVERRIDES)
+    pep3333 = parser.PEP(Path("pep-3333.txt"))
 
     assert pep8 < pep3333
 
 
 def test_pep_equal():
-    pep_a = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)
+    pep_a = parser.PEP(Path("pep-0008.txt"))
-    pep_b = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)
+    pep_b = parser.PEP(Path("pep-0008.txt"))
 
     assert pep_a == pep_b
 
 
-def test_pep_details():
+def test_pep_details(monkeypatch):
-    pep8 = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)
+    pep8 = parser.PEP(Path("pep-0008.txt"))
 
     assert pep8.details == {
         "authors": "GvR, Warsaw, Coghlan",
@@ -64,10 +64,10 @@ def test_pep_details():
 )
 def test_parse_authors(test_input, expected):
     # Arrange
-    pep = parser.PEP(Path("pep-0160.txt"), AUTHORS_OVERRIDES)
+    dummy_object = parser.PEP(Path("pep-0160.txt"))
 
     # Act
-    out = parser._parse_authors(pep, test_input, AUTHORS_OVERRIDES)
+    out = parser._parse_authors(dummy_object, test_input, AUTHORS_OVERRIDES)
 
     # Assert
     assert out == expected
@@ -75,7 +75,7 @@ def test_parse_authors(test_input, expected):
 
 def test_parse_authors_invalid():
 
-    pep = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)
+    pep = parser.PEP(Path("pep-0008.txt"))
 
     with pytest.raises(PEPError, match="no authors found"):
         parser._parse_authors(pep, "", AUTHORS_OVERRIDES)

pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py

@@ -1,11 +1,10 @@
 from pathlib import Path
 
 from pep_sphinx_extensions.pep_zero_generator import parser, pep_index_generator
-from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES
 
 
 def test_create_pep_json():
-    peps = [parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)]
+    peps = [parser.PEP(Path("pep-0008.txt"))]
 
     out = pep_index_generator.create_pep_json(peps)
 

pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py

@@ -3,7 +3,6 @@ from pathlib import Path
 import pytest
 
 from pep_sphinx_extensions.pep_zero_generator import parser, writer
-from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES
 
 
 def test_pep_zero_writer_emit_text_newline():
@@ -48,11 +47,7 @@ def test_pep_zero_writer_emit_title():
 )
 def test_verify_email_addresses(test_input, expected):
     # Arrange
-    peps = [
+    peps = [parser.PEP(Path(f"pep_sphinx_extensions/tests/peps/{test_input}"))]
-        parser.PEP(
-            Path(f"pep_sphinx_extensions/tests/peps/{test_input}"), AUTHORS_OVERRIDES
-        )
-    ]
 
     # Act
     out = writer._verify_email_addresses(peps)