From a2f2d6cc5a2a109c81d43b74e36afe5353488953 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 18 Jun 2022 19:04:46 +0100 Subject: [PATCH] Add support for topic indices (#2579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- .pre-commit-config.yaml | 7 ++ contents.rst | 1 + .../pep_processor/html/pep_html_builder.py | 2 +- .../pep_zero_generator/constants.py | 14 ++++ .../pep_zero_generator/parser.py | 18 ++++- .../pep_zero_generator/pep_index_generator.py | 45 +++++------- .../pep_zero_generator/subindices.py | 71 +++++++++++++++++++ .../pep_zero_generator/writer.py | 28 +++++--- .../tests/pep_zero_generator/test_parser.py | 20 +++--- .../test_pep_index_generator.py | 3 +- .../tests/pep_zero_generator/test_writer.py | 7 +- 11 files changed, 157 insertions(+), 59 deletions(-) create mode 100644 pep_sphinx_extensions/pep_zero_generator/subindices.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bb2ed78a0..a4c436d0e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -176,6 +176,13 @@ repos: files: '^pep-\d+\.(rst|txt)$' types: [text] + - id: validate-topic + name: "'Topic' must be for a valid sub-index" + language: pygrep + entry: '^Topic:(?:(?! +(Packaging|Typing|Packaging, Typing)$))' + files: '^pep-\d+\.(rst|txt)$' + types: [text] + - id: validate-content-type name: "'Content-Type' must be 'text/x-rst'" language: pygrep diff --git a/contents.rst b/contents.rst index aa1bf01d4..3aa0d788d 100644 --- a/contents.rst +++ b/contents.rst @@ -16,3 +16,4 @@ This is an internal Sphinx page; please go to the :doc:`PEP Index `. docs/* pep-* + topic/* diff --git a/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py b/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py index 6a8447cfe..f95d90411 100644 --- a/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py +++ b/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py @@ -36,7 +36,7 @@ class FileBuilder(StandaloneHTMLBuilder): # local table of contents toc_tree = self.env.tocs[docname].deepcopy() - if len(toc_tree[0]) > 1: + if len(toc_tree) and len(toc_tree[0]) > 1: toc_tree = toc_tree[0][1] # don't include document title del toc_tree[0] # remove contents node for node in toc_tree.findall(nodes.reference): diff --git a/pep_sphinx_extensions/pep_zero_generator/constants.py b/pep_sphinx_extensions/pep_zero_generator/constants.py index 5b3ea5f6f..8b346c585 100644 --- a/pep_sphinx_extensions/pep_zero_generator/constants.py +++ b/pep_sphinx_extensions/pep_zero_generator/constants.py @@ -32,3 +32,17 @@ TYPE_STANDARDS = "Standards Track" TYPE_VALUES = {TYPE_STANDARDS, TYPE_INFO, TYPE_PROCESS} # Active PEPs can only be for Informational or Process PEPs. ACTIVE_ALLOWED = {TYPE_PROCESS, TYPE_INFO} + +# map of topic -> additional description +SUBINDICES_BY_TOPIC = { + "packaging": """\ +The canonical, up-to-date packaging specifications can be found on the +`Python Packaging Authority`_ (PyPA) `specifications`_ page. +Packaging PEPs follow the `PyPA specification update process`_. +They are used to propose major additions or changes to the PyPA specifications. + +.. _Python Packaging Authority: https://www.pypa.io/ +.. _specifications: https://packaging.python.org/en/latest/specifications/ +.. _PyPA specification update process: https://www.pypa.io/en/latest/specifications/#specification-update-process +""", +} diff --git a/pep_sphinx_extensions/pep_zero_generator/parser.py b/pep_sphinx_extensions/pep_zero_generator/parser.py index 620319b3d..c1c14e9cc 100644 --- a/pep_sphinx_extensions/pep_zero_generator/parser.py +++ b/pep_sphinx_extensions/pep_zero_generator/parser.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv from email.parser import HeaderParser from pathlib import Path import re @@ -22,6 +23,14 @@ if TYPE_CHECKING: from pep_sphinx_extensions.pep_zero_generator.author import Author +# AUTHOR_OVERRIDES.csv is an exception file for PEP 0 name parsing +AUTHOR_OVERRIDES: dict[str, dict[str, str]] = {} +with open("AUTHOR_OVERRIDES.csv", "r", encoding="utf-8") as f: + for line in csv.DictReader(f): + full_name = line.pop("Overridden Name") + AUTHOR_OVERRIDES[full_name] = line + + class PEP: """Representation of PEPs. @@ -37,7 +46,7 @@ class PEP: # The required RFC 822 headers for all PEPs. required_headers = {"PEP", "Title", "Author", "Status", "Type", "Created"} - def __init__(self, filename: Path, authors_overrides: dict): + def __init__(self, filename: Path): """Init object from an open PEP file object. pep_file is full text of the PEP file, filename is path of the PEP file, author_lookup is author exceptions file @@ -88,7 +97,11 @@ class PEP: self.status: str = status # Parse PEP authors - self.authors: list[Author] = _parse_authors(self, metadata["Author"], authors_overrides) + self.authors: list[Author] = _parse_authors(self, metadata["Author"], AUTHOR_OVERRIDES) + + # Topic (for sub-indices) + _topic = metadata.get("Topic", "").lower().split(",") + self.topic: set[str] = {topic for topic_raw in _topic if (topic := topic_raw.strip())} # Other headers self.created = metadata["Created"] @@ -136,6 +149,7 @@ class PEP: "discussions_to": self.discussions_to, "status": self.status, "type": self.pep_type, + "topic": ", ".join(sorted(self.topic)), "created": self.created, "python_version": self.python_version, "post_history": self.post_history, diff --git a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py index 6292bd8a7..4ebaf0cbf 100644 --- a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py +++ b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py @@ -17,13 +17,13 @@ to allow it to be processed as normal. """ from __future__ import annotations -import csv import json from pathlib import Path -import re from typing import TYPE_CHECKING +from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC from pep_sphinx_extensions.pep_zero_generator import parser +from pep_sphinx_extensions.pep_zero_generator import subindices from pep_sphinx_extensions.pep_zero_generator import writer if TYPE_CHECKING: @@ -31,46 +31,35 @@ if TYPE_CHECKING: from sphinx.environment import BuildEnvironment -def create_pep_json(peps: list[parser.PEP]) -> str: - return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1) - - -def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None: +def _parse_peps() -> list[parser.PEP]: # Read from root directory path = Path(".") - - pep_zero_filename = "pep-0000" peps: list[parser.PEP] = [] - pep_pat = re.compile(r"pep-\d{4}") # Path.match() doesn't support regular expressions - - # AUTHOR_OVERRIDES.csv is an exception file for PEP0 name parsing - with open("AUTHOR_OVERRIDES.csv", "r", encoding="utf-8") as f: - authors_overrides = {} - for line in csv.DictReader(f): - full_name = line.pop("Overridden Name") - authors_overrides[full_name] = line for file_path in path.iterdir(): if not file_path.is_file(): continue # Skip directories etc. if file_path.match("pep-0000*"): continue # Skip pre-existing PEP 0 files - if pep_pat.match(str(file_path)) and file_path.suffix in {".txt", ".rst"}: - pep = parser.PEP(path.joinpath(file_path).absolute(), authors_overrides) + if file_path.match("pep-????.???") and file_path.suffix in {".txt", ".rst"}: + pep = parser.PEP(path.joinpath(file_path).absolute()) peps.append(pep) - peps = sorted(peps) + return sorted(peps) + + +def create_pep_json(peps: list[parser.PEP]) -> str: + return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1) + + +def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None: + peps = _parse_peps() pep0_text = writer.PEPZeroWriter().write_pep0(peps) - pep0_path = Path(f"{pep_zero_filename}.rst") - pep0_path.write_text(pep0_text, encoding="utf-8") + pep0_path = subindices.update_sphinx("pep-0000", pep0_text, docnames, env) + peps.append(parser.PEP(pep0_path)) - peps.append(parser.PEP(pep0_path, authors_overrides)) - - # Add to files for builder - docnames.insert(1, pep_zero_filename) - # Add to files for writer - env.found_docs.add(pep_zero_filename) + subindices.generate_subindices(SUBINDICES_BY_TOPIC, peps, docnames, env) # Create peps.json json_path = Path(app.outdir, "api", "peps.json").resolve() diff --git a/pep_sphinx_extensions/pep_zero_generator/subindices.py b/pep_sphinx_extensions/pep_zero_generator/subindices.py new file mode 100644 index 000000000..62e138d17 --- /dev/null +++ b/pep_sphinx_extensions/pep_zero_generator/subindices.py @@ -0,0 +1,71 @@ +"""Utilities to support sub-indices for PEPs.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from pep_sphinx_extensions.pep_zero_generator import writer + +if TYPE_CHECKING: + from sphinx.environment import BuildEnvironment + + from pep_sphinx_extensions.pep_zero_generator.parser import PEP + + +def update_sphinx(filename: str, text: str, docnames: list[str], env: BuildEnvironment) -> Path: + file_path = Path(f"{filename}.rst").resolve() + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(text, encoding="utf-8") + + # Add to files for builder + docnames.append(filename) + # Add to files for writer + env.found_docs.add(filename) + + return file_path + + +def generate_subindices( + subindices: dict[str, str], + peps: list[PEP], + docnames: list[str], + env: BuildEnvironment, +) -> None: + # Create sub index page + generate_topic_contents(docnames, env) + + for subindex, additional_description in subindices.items(): + header_text = f"{subindex.title()} PEPs" + header_line = "#" * len(header_text) + header = header_text + "\n" + header_line + "\n" + + topic = subindex.lower() + filtered_peps = [pep for pep in peps if topic in pep.topic] + subindex_intro = f"""\ +This is the index of all Python Enhancement Proposals (PEPs) labelled +under the '{subindex.title()}' topic. This is a sub-index of :pep:`0`, +the PEP index. + +{additional_description} +""" + subindex_text = writer.PEPZeroWriter().write_pep0( + filtered_peps, header, subindex_intro, is_pep0=False, + ) + update_sphinx(f"topic/{subindex}", subindex_text, docnames, env) + + +def generate_topic_contents(docnames: list[str], env: BuildEnvironment): + update_sphinx(f"topic/index", """\ +Topic Index +*********** + +PEPs are indexed by topic on the pages below: + +.. toctree:: + :maxdepth: 1 + :titlesonly: + :glob: + + * +""", docnames, env) diff --git a/pep_sphinx_extensions/pep_zero_generator/writer.py b/pep_sphinx_extensions/pep_zero_generator/writer.py index d94873260..75459e64c 100644 --- a/pep_sphinx_extensions/pep_zero_generator/writer.py +++ b/pep_sphinx_extensions/pep_zero_generator/writer.py @@ -25,7 +25,7 @@ from pep_sphinx_extensions.pep_zero_generator.errors import PEPError if TYPE_CHECKING: from pep_sphinx_extensions.pep_zero_generator.parser import PEP -header = f"""\ +HEADER = f"""\ PEP: 0 Title: Index of Python Enhancement Proposals (PEPs) Last-Modified: {datetime.date.today()} @@ -36,12 +36,13 @@ Content-Type: text/x-rst Created: 13-Jul-2000 """ -intro = """\ +INTRO = """\ This PEP contains the index of all Python Enhancement Proposals, known as PEPs. PEP numbers are :pep:`assigned <1#pep-editors>` by the PEP editors, and once assigned are never changed. The `version control history `_ of -the PEP texts represent their historical record. +the PEP texts represent their historical record. The PEPs are +:doc:`indexed by topic ` for specialist subjects. """ @@ -112,7 +113,9 @@ class PEPZeroWriter: self.emit_text(" -") self.emit_newline() - def write_pep0(self, peps: list[PEP]): + def write_pep0(self, peps: list[PEP], header: str = HEADER, intro: str = INTRO, is_pep0: bool = True): + if len(peps) == 0: + return "" # PEP metadata self.emit_text(header) @@ -138,7 +141,10 @@ class PEPZeroWriter: ("Abandoned, Withdrawn, and Rejected PEPs", dead), ] for (category, peps_in_category) in pep_categories: - self.emit_pep_category(category, peps_in_category) + # For sub-indices, only emit categories with entries. + # For PEP 0, emit every category + if is_pep0 or len(peps_in_category) > 0: + self.emit_pep_category(category, peps_in_category) self.emit_newline() @@ -151,12 +157,14 @@ class PEPZeroWriter: self.emit_newline() # Reserved PEP numbers - self.emit_title("Reserved PEP Numbers") - self.emit_column_headers() - for number, claimants in sorted(self.RESERVED.items()): - self.emit_pep_row(type="", status="", number=number, title="RESERVED", authors=claimants) + if is_pep0: + self.emit_title("Reserved PEP Numbers") + self.emit_column_headers() + for number, claimants in sorted(self.RESERVED.items()): + self.emit_pep_row(type="", status="", number=number, title="RESERVED", authors=claimants) - self.emit_newline() + + self.emit_newline() # PEP types key self.emit_title("PEP Types Key") diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py index 8165abcd0..e84dd98dd 100644 --- a/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py +++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py @@ -9,27 +9,27 @@ from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES def test_pep_repr(): - pep8 = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES) + pep8 = parser.PEP(Path("pep-0008.txt")) assert repr(pep8) == "" def test_pep_less_than(): - pep8 = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES) - pep3333 = parser.PEP(Path("pep-3333.txt"), AUTHORS_OVERRIDES) + pep8 = parser.PEP(Path("pep-0008.txt")) + pep3333 = parser.PEP(Path("pep-3333.txt")) assert pep8 < pep3333 def test_pep_equal(): - pep_a = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES) - pep_b = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES) + pep_a = parser.PEP(Path("pep-0008.txt")) + pep_b = parser.PEP(Path("pep-0008.txt")) assert pep_a == pep_b -def test_pep_details(): - pep8 = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES) +def test_pep_details(monkeypatch): + pep8 = parser.PEP(Path("pep-0008.txt")) assert pep8.details == { "authors": "GvR, Warsaw, Coghlan", @@ -64,10 +64,10 @@ def test_pep_details(): ) def test_parse_authors(test_input, expected): # Arrange - pep = parser.PEP(Path("pep-0160.txt"), AUTHORS_OVERRIDES) + dummy_object = parser.PEP(Path("pep-0160.txt")) # Act - out = parser._parse_authors(pep, test_input, AUTHORS_OVERRIDES) + out = parser._parse_authors(dummy_object, test_input, AUTHORS_OVERRIDES) # Assert assert out == expected @@ -75,7 +75,7 @@ def test_parse_authors(test_input, expected): def test_parse_authors_invalid(): - pep = parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES) + pep = parser.PEP(Path("pep-0008.txt")) with pytest.raises(PEPError, match="no authors found"): parser._parse_authors(pep, "", AUTHORS_OVERRIDES) diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py index 35a6a937c..c2e15844f 100644 --- a/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py +++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py @@ -1,11 +1,10 @@ from pathlib import Path from pep_sphinx_extensions.pep_zero_generator import parser, pep_index_generator -from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES def test_create_pep_json(): - peps = [parser.PEP(Path("pep-0008.txt"), AUTHORS_OVERRIDES)] + peps = [parser.PEP(Path("pep-0008.txt"))] out = pep_index_generator.create_pep_json(peps) diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py index 9cae97a0a..19eeca2d9 100644 --- a/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py +++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py @@ -3,7 +3,6 @@ from pathlib import Path import pytest from pep_sphinx_extensions.pep_zero_generator import parser, writer -from pep_sphinx_extensions.tests.utils import AUTHORS_OVERRIDES def test_pep_zero_writer_emit_text_newline(): @@ -48,11 +47,7 @@ def test_pep_zero_writer_emit_title(): ) def test_verify_email_addresses(test_input, expected): # Arrange - peps = [ - parser.PEP( - Path(f"pep_sphinx_extensions/tests/peps/{test_input}"), AUTHORS_OVERRIDES - ) - ] + peps = [parser.PEP(Path(f"pep_sphinx_extensions/tests/peps/{test_input}"))] # Act out = writer._verify_email_addresses(peps)