From 477139b3f7e8cde689a1998d3a02e2e3dc0e41a3 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 14 Jun 2017 12:38:14 +0100 Subject: [PATCH] PEP 517: Latest recommended changes - use sdist and wheel archives as required interchange formats - use `prepare_*` prefix for optional file export APIs - return relative paths for generated aritifacts/directories --- pep-0517.txt | 110 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 30 deletions(-) diff --git a/pep-0517.txt b/pep-0517.txt index d91ec049f..062425307 100644 --- a/pep-0517.txt +++ b/pep-0517.txt @@ -181,7 +181,7 @@ Optional. If not defined, the default implementation is equivalent to :: - def get_wheel_metadata(metadata_directory, config_settings): + def prepare_wheel_metadata(metadata_directory, config_settings): ... Must create a ``.dist-info`` directory containing wheel metadata @@ -195,7 +195,8 @@ here is that in cases where the metadata depends on build-time decisions, the build backend may need to record these decisions in some convenient format for re-use by the actual wheel-building step. -Return value is ignored. +This must return the basename (not the full path) of the ``.dist-info`` +directory it creates, as a unicode string. Optional. If a build frontend needs this information and the method is not defined, it should call ``build_wheel`` and look at the resulting @@ -206,34 +207,47 @@ metadata directly. def build_wheel(wheel_directory, config_settings, metadata_directory=None): ... -Must build a ``.whl`` file, and place it in the specified -``wheel_directory``. +Must build a .whl file, and place it in the specified ``wheel_directory``. -If the build frontend has previously called ``get_wheel_metadata`` and +If the build frontend has previously called ``prepare_wheel_metadata`` and depends on the wheel resulting from this call to have metadata matching this earlier call, then it should provide the path to the previous ``metadata_directory`` as an argument. If this argument is provided, then ``build_wheel`` MUST produce a wheel with identical metadata. The directory passed in by the build frontend MUST be -identical to the directory created by ``get_wheel_metadata``, +identical to the directory created by ``prepare_wheel_metadata``, including any unrecognized files it created. +This must return the basename (not the full path) of the ``.whl`` file it +creates, as a unicode string. + Mandatory. :: - def export_sdist(sdist_directory, config_settings): + def build_sdist(sdist_directory, config_settings): ... -Must export an unpacked source distribution into the specified +Must build a .tar.gz source distribution and place it in the specified ``sdist_directory``. -An unpacked source distribution (sdist) consists of a directory called +A .tar.gz source distribution (sdist) contains a single top-level directory called ``{name}-{version}`` (e.g. ``foo-1.0``), containing the source files of the package. This directory must also contain the ``pyproject.toml`` from the build directory, and a PKG-INFO file containing metadata in the format described in -`PEP 345 `_. +`PEP 345 `_. Although historically +zip files have also been used as sdists, this hook should produce a gzipped +tarball. This is already the more common format for sdists, and having a +consistent format makes for simpler tooling. + +The generated tarball should use the modern POSIX.1-2001 pax tar format, which +specifies UTF-8 based file names. This is not yet the default for the tarfile +module shipped with Python 3.6, so backends using the tarfile module need to +explicitly pass ``format=tarfile.PAX_FORMAT``. + +This must return the basename (not the full path) of the ``.tar.gz`` file it +creates, as a unicode string. Mandatory, but it may not succeed in all situations: for instance, some tools can only build an sdist from a VCS checkout. @@ -257,8 +271,10 @@ Because the wheel will be built from a temporary build directory, ``build_wheel` may create intermediate files in the working directory, and does not need to take care to clean them up. -Optional. If this hook is not defined, frontends may call ``export_sdist`` and -use the unpacked sdist as a build directory. Backends in which +The return value will be ignored. + +Optional. If this hook is not defined, frontends may call ``build_sdist`` +and unpack the archive to use as a build directory. Backends in which building an sdist has additional requirements should define ``prepare_build_files``. @@ -307,6 +323,10 @@ Of course, it's up to users to make sure that they pass options which make sense for the particular build backend and package that they are building. +The hooks may be called with positional or keyword arguments, so backends +implementing them should be careful to make sure that their signatures match +both the order and the names of the arguments above. + All hooks are run with working directory set to the root of the source tree, and MAY print arbitrary informational text on stdout and stderr. They MUST NOT read from stdin, and the build frontend MAY @@ -546,22 +566,39 @@ build backend:: # mypackage_custom_build_backend.py import os.path + import pathlib def get_build_requires(config_settings, config_directory): return ["wheel"] def build_wheel(wheel_directory, config_settings, config_directory=None): from wheel.archive import archive_wheelfile - path = os.path.join(wheel_directory, - "mypackage-0.1-py2.py3-none-any") + filename = "mypackage-0.1-py2.py3-none-any" + path = os.path.join(wheel_directory, filename) archive_wheelfile(path, "src/") + return filename + + def _exclude_hidden_and_special_files(archive_entry): + """Tarfile filter to exclude hidden and special files from the archive""" + if entry.isfile() or entry.isdir(): + if not os.path.basename(archive_entry.name).startswith("."): + return archive_entry + return None + + def build_sdist(sdist_dir, config_settings): + sdist_subdir = "mypackage-0.1" + sdist_path = pathlib.Path(sdist_dir) / (sdist_subdir + ".tar.gz") + sdist = tarfile.open(sdist_path, "w:gz", format=tarfile.PAX_FORMAT) + # Tar up the whole directory, minus hidden and special files + sdist.add(os.getcwd(), arcname=sdist_subdir, + filter=_exclude_hidden_and_special_files) + return sdist_subdir + ".tar.gz" Of course, this is a *terrible* build backend: it requires the user to have manually set up the wheel metadata in ``src/mypackage-0.1.dist-info/``; when the version number changes it -must be manually updated in multiple places; it doesn't implement the -metadata or develop hooks, ... but it works, and these features can be -added incrementally. Much experience suggests that large successful +must be manually updated in multiple places... but it works, and more features +could be added incrementally. Much experience suggests that large successful projects often originate as quick hacks (e.g., Linux -- "just a hobby, won't be big and professional"; `IPython/Jupyter `_ -- `a grad @@ -604,12 +641,12 @@ across the ecosystem. more powerful options for evolving this specification in the future. For concreteness, imagine that next year we add a new -``get_wheel_metadata2`` hook, which replaces the current -``get_wheel_metadata`` hook with something that produces more data, or a +``prepare_wheel_metadata2`` hook, which replaces the current +``prepare_wheel_metadata`` hook with something that produces more data, or a different metadata format. In order to manage the transition, we want it to be possible for build frontends -to transparently use ``get_wheel_metadata2`` when available and fall -back onto ``get_wheel_metadata`` otherwise; and we want it to be +to transparently use ``prepare_wheel_metadata2`` when available and fall +back onto ``prepare_wheel_metadata`` otherwise; and we want it to be possible for build backends to define both methods, for compatibility with both old and new build frontends. @@ -627,11 +664,11 @@ achieve. Because ``pip`` controls the code that runs inside the child process, it can easily write it to do something like:: command, backend, args = parse_command_line_args(...) - if command == "get_wheel_metadata": - if hasattr(backend, "get_wheel_metadata2"): - backend.get_wheel_metadata2(...) - elif hasattr(backend, "get_wheel_metadata"): - backend.get_wheel_metadata(...) + if command == "prepare_wheel_metadata": + if hasattr(backend, "prepare_wheel_metadata2"): + backend.prepare_wheel_metadata2(...) + elif hasattr(backend, "prepare_wheel_metadata"): + backend.prepare_wheel_metadata(...) else: # error handling @@ -646,13 +683,13 @@ any change can go live, and that any changes will necessarily be restricted to new releases. One specific consequence of this is that in this PEP, we're able to -make the ``get_wheel_metadata`` command optional. In our design, this +make the ``prepare_wheel_metadata`` command optional. In our design, this can easily be worked around by a tool like ``pip``, which can put code in its subprocess runner like:: - def get_wheel_metadata(output_dir, config_settings): - if hasattr(backend, "get_wheel_metadata"): - backend.get_wheel_metadata(output_dir, config_settings) + def prepare_wheel_metadata(output_dir, config_settings): + if hasattr(backend, "prepare_wheel_metadata"): + backend.prepare_wheel_metadata(output_dir, config_settings) else: backend.build_wheel(output_dir, config_settings) touch(output_dir / "PIP_ALREADY_BUILT_WHEELS") @@ -741,6 +778,19 @@ automatically upgrade packages to the new format: will need to stop doing that. +================== + Rejected options +================== + +* We discussed making the wheel and sdist hooks build unpacked directories + containing the same contents as their respective archives. In some cases this + could avoid the need to pack and unpack an archive, but this seems like + premature optimisation. It's advantageous for tools to work with archives + as the canonical interchange formats (especially for wheels, where the archive + format is already standardised). Close control of archive creation is + important for reproducible builds. And it's not clear that tasks requiring an + unpacked distribution will be more common than those requiring an archive. + =========== Copyright ===========