703 lines
60 KiB
HTML
703 lines
60 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 402 – Simplified Package Layout and Partitioning | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0402/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 402 – Simplified Package Layout and Partitioning | peps.python.org'>
|
||
<meta property="og:description" content="This PEP proposes an enhancement to Python’s package importing to:">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0402/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="This PEP proposes an enhancement to Python’s package importing to:">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 402</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 402 – Simplified Package Layout and Partitioning</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Phillip J. Eby</dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Formally declined and will not be accepted">Rejected</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Topic<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="../topic/packaging/">Packaging</a></dd>
|
||
<dt class="field-odd">Created<span class="colon">:</span></dt>
|
||
<dd class="field-odd">12-Jul-2011</dd>
|
||
<dt class="field-even">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-even">3.3</dd>
|
||
<dt class="field-odd">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-odd">20-Jul-2011</dd>
|
||
<dt class="field-even">Replaces<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="../pep-0382/">382</a></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#rejection-notice">Rejection Notice</a></li>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#the-problem">The Problem</a></li>
|
||
<li><a class="reference internal" href="#the-solution">The Solution</a><ul>
|
||
<li><a class="reference internal" href="#a-thought-experiment">A Thought Experiment</a></li>
|
||
<li><a class="reference internal" href="#self-contained-vs-virtual-packages">Self-Contained vs. “Virtual” Packages</a></li>
|
||
<li><a class="reference internal" href="#backwards-compatibility-and-performance">Backwards Compatibility and Performance</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#virtual-paths">Virtual Paths</a></li>
|
||
<li><a class="reference internal" href="#standard-library-changes-additions">Standard Library Changes/Additions</a></li>
|
||
<li><a class="reference internal" href="#implementation-notes">Implementation Notes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="rejection-notice">
|
||
<h2><a class="toc-backref" href="#rejection-notice" role="doc-backlink">Rejection Notice</a></h2>
|
||
<p>On the first day of sprints at US PyCon 2012 we had a long and
|
||
fruitful discussion about <a class="pep reference internal" href="../pep-0382/" title="PEP 382 – Namespace Packages">PEP 382</a> and <a class="pep reference internal" href="../pep-0402/" title="PEP 402 – Simplified Package Layout and Partitioning">PEP 402</a>. We ended up rejecting
|
||
both but a new PEP will be written to carry on in the spirit of PEP
|
||
402. Martin von Löwis wrote up a summary: <a class="footnote-reference brackets" href="#id6" id="id1">[3]</a>.</p>
|
||
</section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP proposes an enhancement to Python’s package importing
|
||
to:</p>
|
||
<ul class="simple">
|
||
<li>Surprise users of other languages less,</li>
|
||
<li>Make it easier to convert a module into a package, and</li>
|
||
<li>Support dividing packages into separately installed components
|
||
(ala “namespace packages”, as described in <a class="pep reference internal" href="../pep-0382/" title="PEP 382 – Namespace Packages">PEP 382</a>)</li>
|
||
</ul>
|
||
<p>The proposed enhancements do not change the semantics of any
|
||
currently-importable directory layouts, but make it possible for
|
||
packages to use a simplified directory layout (that is not importable
|
||
currently).</p>
|
||
<p>However, the proposed changes do NOT add any performance overhead to
|
||
the importing of existing modules or packages, and performance for the
|
||
new directory layout should be about the same as that of previous
|
||
“namespace package” solutions (such as <code class="docutils literal notranslate"><span class="pre">pkgutil.extend_path()</span></code>).</p>
|
||
</section>
|
||
<section id="the-problem">
|
||
<h2><a class="toc-backref" href="#the-problem" role="doc-backlink">The Problem</a></h2>
|
||
<blockquote class="epigraph">
|
||
<div>“Most packages are like modules. Their contents are highly
|
||
interdependent and can’t be pulled apart. [However,] some
|
||
packages exist to provide a separate namespace. … It should
|
||
be possible to distribute sub-packages or submodules of these
|
||
[namespace packages] independently.”<p class="attribution">—Jim Fulton, shortly before the release of Python 2.3 <a class="footnote-reference brackets" href="#id4" id="id2">[1]</a></p>
|
||
</div></blockquote>
|
||
<p>When new users come to Python from other languages, they are often
|
||
confused by Python’s package import semantics. At Google, for example,
|
||
Guido received complaints from “a large crowd with pitchforks” <a class="footnote-reference brackets" href="#id5" id="id3">[2]</a>
|
||
that the requirement for packages to contain an <code class="docutils literal notranslate"><span class="pre">__init__</span></code> module
|
||
was a “misfeature”, and should be dropped.</p>
|
||
<p>In addition, users coming from languages like Java or Perl are
|
||
sometimes confused by a difference in Python’s import path searching.</p>
|
||
<p>In most other languages that have a similar path mechanism to Python’s
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, a package is merely a namespace that contains modules
|
||
or classes, and can thus be spread across multiple directories in
|
||
the language’s path. In Perl, for instance, a <code class="docutils literal notranslate"><span class="pre">Foo::Bar</span></code> module
|
||
will be searched for in <code class="docutils literal notranslate"><span class="pre">Foo/</span></code> subdirectories all along the module
|
||
include path, not just in the first such subdirectory found.</p>
|
||
<p>Worse, this is not just a problem for new users: it prevents <em>anyone</em>
|
||
from easily splitting a package into separately-installable
|
||
components. In Perl terms, it would be as if every possible <code class="docutils literal notranslate"><span class="pre">Net::</span></code>
|
||
module on CPAN had to be bundled up and shipped in a single tarball!</p>
|
||
<p>For that reason, various workarounds for this latter limitation exist,
|
||
circulated under the term “namespace packages”. The Python standard
|
||
library has provided one such workaround since Python 2.3 (via the
|
||
<code class="docutils literal notranslate"><span class="pre">pkgutil.extend_path()</span></code> function), and the “setuptools” package
|
||
provides another (via <code class="docutils literal notranslate"><span class="pre">pkg_resources.declare_namespace()</span></code>).</p>
|
||
<p>The workarounds themselves, however, fall prey to a <em>third</em> issue with
|
||
Python’s way of laying out packages in the filesystem.</p>
|
||
<p>Because a package <em>must</em> contain an <code class="docutils literal notranslate"><span class="pre">__init__</span></code> module, any attempt
|
||
to distribute modules for that package must necessarily include that
|
||
<code class="docutils literal notranslate"><span class="pre">__init__</span></code> module, if those modules are to be importable.</p>
|
||
<p>However, the very fact that each distribution of modules for a package
|
||
must contain this (duplicated) <code class="docutils literal notranslate"><span class="pre">__init__</span></code> module, means that OS
|
||
vendors who package up these module distributions must somehow handle
|
||
the conflict caused by several module distributions installing that
|
||
<code class="docutils literal notranslate"><span class="pre">__init__</span></code> module to the same location in the filesystem.</p>
|
||
<p>This led to the proposing of <a class="pep reference internal" href="../pep-0382/" title="PEP 382 – Namespace Packages">PEP 382</a> (“Namespace Packages”) - a way
|
||
to signal to Python’s import machinery that a directory was
|
||
importable, using unique filenames per module distribution.</p>
|
||
<p>However, there was more than one downside to this approach.
|
||
Performance for all import operations would be affected, and the
|
||
process of designating a package became even more complex. New
|
||
terminology had to be invented to explain the solution, and so on.</p>
|
||
<p>As terminology discussions continued on the Import-SIG, it soon became
|
||
apparent that the main reason it was so difficult to explain the
|
||
concepts related to “namespace packages” was because Python’s
|
||
current way of handling packages is somewhat underpowered, when
|
||
compared to other languages.</p>
|
||
<p>That is, in other popular languages with package systems, no special
|
||
term is needed to describe “namespace packages”, because <em>all</em>
|
||
packages generally behave in the desired fashion.</p>
|
||
<p>Rather than being an isolated single directory with a special marker
|
||
module (as in Python), packages in other languages are typically just
|
||
the <em>union</em> of appropriately-named directories across the <em>entire</em>
|
||
import or inclusion path.</p>
|
||
<p>In Perl, for example, the module <code class="docutils literal notranslate"><span class="pre">Foo</span></code> is always found in a
|
||
<code class="docutils literal notranslate"><span class="pre">Foo.pm</span></code> file, and a module <code class="docutils literal notranslate"><span class="pre">Foo::Bar</span></code> is always found in a
|
||
<code class="docutils literal notranslate"><span class="pre">Foo/Bar.pm</span></code> file. (In other words, there is One Obvious Way to
|
||
find the location of a particular module.)</p>
|
||
<p>This is because Perl considers a module to be <em>different</em> from a
|
||
package: the package is purely a <em>namespace</em> in which other modules
|
||
may reside, and is only <em>coincidentally</em> the name of a module as well.</p>
|
||
<p>In current versions of Python, however, the module and the package are
|
||
more tightly bound together. <code class="docutils literal notranslate"><span class="pre">Foo</span></code> is always a module – whether it
|
||
is found in <code class="docutils literal notranslate"><span class="pre">Foo.py</span></code> or <code class="docutils literal notranslate"><span class="pre">Foo/__init__.py</span></code> – and it is tightly
|
||
linked to its submodules (if any), which <em>must</em> reside in the exact
|
||
same directory where the <code class="docutils literal notranslate"><span class="pre">__init__.py</span></code> was found.</p>
|
||
<p>On the positive side, this design choice means that a package is quite
|
||
self-contained, and can be installed, copied, etc. as a unit just by
|
||
performing an operation on the package’s root directory.</p>
|
||
<p>On the negative side, however, it is non-intuitive for beginners, and
|
||
requires a more complex step to turn a module into a package. If
|
||
<code class="docutils literal notranslate"><span class="pre">Foo</span></code> begins its life as <code class="docutils literal notranslate"><span class="pre">Foo.py</span></code>, then it must be moved and
|
||
renamed to <code class="docutils literal notranslate"><span class="pre">Foo/__init__.py</span></code>.</p>
|
||
<p>Conversely, if you intend to create a <code class="docutils literal notranslate"><span class="pre">Foo.Bar</span></code> module from the
|
||
start, but have no particular module contents to put in <code class="docutils literal notranslate"><span class="pre">Foo</span></code>
|
||
itself, then you have to create an empty and seemingly-irrelevant
|
||
<code class="docutils literal notranslate"><span class="pre">Foo/__init__.py</span></code> file, just so that <code class="docutils literal notranslate"><span class="pre">Foo.Bar</span></code> can be imported.</p>
|
||
<p>(And these issues don’t just confuse newcomers to the language,
|
||
either: they annoy many experienced developers as well.)</p>
|
||
<p>So, after some discussion on the Import-SIG, this PEP was created
|
||
as an alternative to PEP 382, in an attempt to solve <em>all</em> of the
|
||
above problems, not just the “namespace package” use cases.</p>
|
||
<p>And, as a delightful side effect, the solution proposed in this PEP
|
||
does not affect the import performance of ordinary modules or
|
||
self-contained (i.e. <code class="docutils literal notranslate"><span class="pre">__init__</span></code>-based) packages.</p>
|
||
</section>
|
||
<section id="the-solution">
|
||
<h2><a class="toc-backref" href="#the-solution" role="doc-backlink">The Solution</a></h2>
|
||
<p>In the past, various proposals have been made to allow more intuitive
|
||
approaches to package directory layout. However, most of them failed
|
||
because of an apparent backward-compatibility problem.</p>
|
||
<p>That is, if the requirement for an <code class="docutils literal notranslate"><span class="pre">__init__</span></code> module were simply
|
||
dropped, it would open up the possibility for a directory named, say,
|
||
<code class="docutils literal notranslate"><span class="pre">string</span></code> on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, to block importing of the standard library
|
||
<code class="docutils literal notranslate"><span class="pre">string</span></code> module.</p>
|
||
<p>Paradoxically, however, the failure of this approach does <em>not</em> arise
|
||
from the elimination of the <code class="docutils literal notranslate"><span class="pre">__init__</span></code> requirement!</p>
|
||
<p>Rather, the failure arises because the underlying approach takes for
|
||
granted that a package is just ONE thing, instead of two.</p>
|
||
<p>In truth, a package comprises two separate, but related entities: a
|
||
module (with its own, optional contents), and a <em>namespace</em> where
|
||
<em>other</em> modules or packages can be found.</p>
|
||
<p>In current versions of Python, however, the module part (found in
|
||
<code class="docutils literal notranslate"><span class="pre">__init__</span></code>) and the namespace for submodule imports (represented
|
||
by the <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attribute) are both initialized at the same time,
|
||
when the package is first imported.</p>
|
||
<p>And, if you assume this is the <em>only</em> way to initialize these two
|
||
things, then there is no way to drop the need for an <code class="docutils literal notranslate"><span class="pre">__init__</span></code>
|
||
module, while still being backwards-compatible with existing directory
|
||
layouts.</p>
|
||
<p>After all, as soon as you encounter a directory on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>
|
||
matching the desired name, that means you’ve “found” the package, and
|
||
must stop searching, right?</p>
|
||
<p>Well, not quite.</p>
|
||
<section id="a-thought-experiment">
|
||
<h3><a class="toc-backref" href="#a-thought-experiment" role="doc-backlink">A Thought Experiment</a></h3>
|
||
<p>Let’s hop into the time machine for a moment, and pretend we’re back
|
||
in the early 1990s, shortly before Python packages and <code class="docutils literal notranslate"><span class="pre">__init__.py</span></code>
|
||
have been invented. But, imagine that we <em>are</em> familiar with
|
||
Perl-like package imports, and we want to implement a similar system
|
||
in Python.</p>
|
||
<p>We’d still have Python’s <em>module</em> imports to build on, so we could
|
||
certainly conceive of having <code class="docutils literal notranslate"><span class="pre">Foo.py</span></code> as a parent <code class="docutils literal notranslate"><span class="pre">Foo</span></code> module
|
||
for a <code class="docutils literal notranslate"><span class="pre">Foo</span></code> package. But how would we implement submodule and
|
||
subpackage imports?</p>
|
||
<p>Well, if we didn’t have the idea of <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attributes yet,
|
||
we’d probably just search <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> looking for <code class="docutils literal notranslate"><span class="pre">Foo/Bar.py</span></code>.</p>
|
||
<p>But we’d <em>only</em> do it when someone actually tried to <em>import</em>
|
||
<code class="docutils literal notranslate"><span class="pre">Foo.Bar</span></code>.</p>
|
||
<p>NOT when they imported <code class="docutils literal notranslate"><span class="pre">Foo</span></code>.</p>
|
||
<p>And <em>that</em> lets us get rid of the backwards-compatibility problem
|
||
of dropping the <code class="docutils literal notranslate"><span class="pre">__init__</span></code> requirement, back here in 2011.</p>
|
||
<p>How?</p>
|
||
<p>Well, when we <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">Foo</span></code>, we’re not even <em>looking</em> for <code class="docutils literal notranslate"><span class="pre">Foo/</span></code>
|
||
directories on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, because we don’t <em>care</em> yet. The only
|
||
point at which we care, is the point when somebody tries to actually
|
||
import a submodule or subpackage of <code class="docutils literal notranslate"><span class="pre">Foo</span></code>.</p>
|
||
<p>That means that if <code class="docutils literal notranslate"><span class="pre">Foo</span></code> is a standard library module (for example),
|
||
and I happen to have a <code class="docutils literal notranslate"><span class="pre">Foo</span></code> directory on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> (without
|
||
an <code class="docutils literal notranslate"><span class="pre">__init__.py</span></code>, of course), then <em>nothing breaks</em>. The <code class="docutils literal notranslate"><span class="pre">Foo</span></code>
|
||
module is still just a module, and it’s still imported normally.</p>
|
||
</section>
|
||
<section id="self-contained-vs-virtual-packages">
|
||
<h3><a class="toc-backref" href="#self-contained-vs-virtual-packages" role="doc-backlink">Self-Contained vs. “Virtual” Packages</a></h3>
|
||
<p>Of course, in today’s Python, trying to <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">Foo.Bar</span></code> will
|
||
fail if <code class="docutils literal notranslate"><span class="pre">Foo</span></code> is just a <code class="docutils literal notranslate"><span class="pre">Foo.py</span></code> module (and thus lacks a
|
||
<code class="docutils literal notranslate"><span class="pre">__path__</span></code> attribute).</p>
|
||
<p>So, this PEP proposes to <em>dynamically</em> create a <code class="docutils literal notranslate"><span class="pre">__path__</span></code>, in the
|
||
case where one is missing.</p>
|
||
<p>That is, if I try to <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">Foo.Bar</span></code> the proposed change to the
|
||
import machinery will notice that the <code class="docutils literal notranslate"><span class="pre">Foo</span></code> module lacks a
|
||
<code class="docutils literal notranslate"><span class="pre">__path__</span></code>, and will therefore try to <em>build</em> one before proceeding.</p>
|
||
<p>And it will do this by making a list of all the existing <code class="docutils literal notranslate"><span class="pre">Foo/</span></code>
|
||
subdirectories of the directories listed in <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>.</p>
|
||
<p>If the list is empty, the import will fail with <code class="docutils literal notranslate"><span class="pre">ImportError</span></code>, just
|
||
like today. But if the list is <em>not</em> empty, then it is saved in
|
||
a new <code class="docutils literal notranslate"><span class="pre">Foo.__path__</span></code> attribute, making the module a “virtual
|
||
package”.</p>
|
||
<p>That is, because it now has a valid <code class="docutils literal notranslate"><span class="pre">__path__</span></code>, we can proceed
|
||
to import submodules or subpackages in the normal way.</p>
|
||
<p>Now, notice that this change does not affect “classic”, self-contained
|
||
packages that have an <code class="docutils literal notranslate"><span class="pre">__init__</span></code> module in them. Such packages
|
||
already <em>have</em> a <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attribute (initialized at import time)
|
||
so the import machinery won’t try to create another one later.</p>
|
||
<p>This means that (for example) the standard library <code class="docutils literal notranslate"><span class="pre">email</span></code> package
|
||
will not be affected in any way by you having a bunch of unrelated
|
||
directories named <code class="docutils literal notranslate"><span class="pre">email</span></code> on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>. (Even if they contain
|
||
<code class="docutils literal notranslate"><span class="pre">*.py</span></code> files.)</p>
|
||
<p>But it <em>does</em> mean that if you want to turn your <code class="docutils literal notranslate"><span class="pre">Foo</span></code> module into
|
||
a <code class="docutils literal notranslate"><span class="pre">Foo</span></code> package, all you have to do is add a <code class="docutils literal notranslate"><span class="pre">Foo/</span></code> directory
|
||
somewhere on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, and start adding modules to it.</p>
|
||
<p>But what if you only want a “namespace package”? That is, a package
|
||
that is <em>only</em> a namespace for various separately-distributed
|
||
submodules and subpackages?</p>
|
||
<p>For example, if you’re Zope Corporation, distributing dozens of
|
||
separate tools like <code class="docutils literal notranslate"><span class="pre">zc.buildout</span></code>, each in packages under the <code class="docutils literal notranslate"><span class="pre">zc</span></code>
|
||
namespace, you don’t want to have to make and include an empty
|
||
<code class="docutils literal notranslate"><span class="pre">zc.py</span></code> in every tool you ship. (And, if you’re a Linux or other
|
||
OS vendor, you don’t want to deal with the package installation
|
||
conflicts created by trying to install ten copies of <code class="docutils literal notranslate"><span class="pre">zc.py</span></code> to the
|
||
same location!)</p>
|
||
<p>No problem. All we have to do is make one more minor tweak to the
|
||
import process: if the “classic” import process fails to find a
|
||
self-contained module or package (e.g., if <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">zc</span></code> fails to find
|
||
a <code class="docutils literal notranslate"><span class="pre">zc.py</span></code> or <code class="docutils literal notranslate"><span class="pre">zc/__init__.py</span></code>), then we once more try to build a
|
||
<code class="docutils literal notranslate"><span class="pre">__path__</span></code> by searching for all the <code class="docutils literal notranslate"><span class="pre">zc/</span></code> directories on
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code>, and putting them in a list.</p>
|
||
<p>If this list is empty, we raise <code class="docutils literal notranslate"><span class="pre">ImportError</span></code>. But if it’s
|
||
non-empty, we create an empty <code class="docutils literal notranslate"><span class="pre">zc</span></code> module, and put the list in
|
||
<code class="docutils literal notranslate"><span class="pre">zc.__path__</span></code>. Congratulations: <code class="docutils literal notranslate"><span class="pre">zc</span></code> is now a namespace-only,
|
||
“pure virtual” package! It has no module contents, but you can still
|
||
import submodules and subpackages from it, regardless of where they’re
|
||
located on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>.</p>
|
||
<p>(By the way, both of these additions to the import protocol (i.e. the
|
||
dynamically-added <code class="docutils literal notranslate"><span class="pre">__path__</span></code>, and dynamically-created modules)
|
||
apply recursively to child packages, using the parent package’s
|
||
<code class="docutils literal notranslate"><span class="pre">__path__</span></code> in place of <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> as a basis for generating a
|
||
child <code class="docutils literal notranslate"><span class="pre">__path__</span></code>. This means that self-contained and virtual
|
||
packages can contain each other without limitation, with the caveat
|
||
that if you put a virtual package inside a self-contained one, it’s
|
||
gonna have a really short <code class="docutils literal notranslate"><span class="pre">__path__</span></code>!)</p>
|
||
</section>
|
||
<section id="backwards-compatibility-and-performance">
|
||
<h3><a class="toc-backref" href="#backwards-compatibility-and-performance" role="doc-backlink">Backwards Compatibility and Performance</a></h3>
|
||
<p>Notice that these two changes <em>only</em> affect import operations that
|
||
today would result in <code class="docutils literal notranslate"><span class="pre">ImportError</span></code>. As a result, the performance
|
||
of imports that do not involve virtual packages is unaffected, and
|
||
potential backward compatibility issues are very restricted.</p>
|
||
<p>Today, if you try to import submodules or subpackages from a module
|
||
with no <code class="docutils literal notranslate"><span class="pre">__path__</span></code>, it’s an immediate error. And of course, if you
|
||
don’t have a <code class="docutils literal notranslate"><span class="pre">zc.py</span></code> or <code class="docutils literal notranslate"><span class="pre">zc/__init__.py</span></code> somewhere on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>
|
||
today, <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">zc</span></code> would likewise fail.</p>
|
||
<p>Thus, the only potential backwards-compatibility issues are:</p>
|
||
<ol class="arabic">
|
||
<li>Tools that expect package directories to have an <code class="docutils literal notranslate"><span class="pre">__init__</span></code>
|
||
module, that expect directories without an <code class="docutils literal notranslate"><span class="pre">__init__</span></code> module
|
||
to be unimportable, or that expect <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attributes to be
|
||
static, will not recognize virtual packages as packages.<p>(In practice, this just means that tools will need updating to
|
||
support virtual packages, e.g. by using <code class="docutils literal notranslate"><span class="pre">pkgutil.walk_modules()</span></code>
|
||
instead of using hardcoded filesystem searches.)</p>
|
||
</li>
|
||
<li>Code that <em>expects</em> certain imports to fail may now do something
|
||
unexpected. This should be fairly rare in practice, as most sane,
|
||
non-test code does not import things that are expected not to
|
||
exist!</li>
|
||
</ol>
|
||
<p>The biggest likely exception to the above would be when a piece of
|
||
code tries to check whether some package is installed by importing
|
||
it. If this is done <em>only</em> by importing a top-level module (i.e., not
|
||
checking for a <code class="docutils literal notranslate"><span class="pre">__version__</span></code> or some other attribute), <em>and</em> there
|
||
is a directory of the same name as the sought-for package on
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code> somewhere, <em>and</em> the package is not actually installed,
|
||
then such code could be fooled into thinking a package is installed
|
||
that really isn’t.</p>
|
||
<p>For example, suppose someone writes a script (<code class="docutils literal notranslate"><span class="pre">datagen.py</span></code>)
|
||
containing the following code:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">try</span><span class="p">:</span>
|
||
<span class="kn">import</span> <span class="nn">json</span>
|
||
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
||
<span class="kn">import</span> <span class="nn">simplejson</span> <span class="k">as</span> <span class="nn">json</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>And runs it in a directory laid out like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">datagen</span><span class="o">.</span><span class="n">py</span>
|
||
<span class="n">json</span><span class="o">/</span>
|
||
<span class="n">foo</span><span class="o">.</span><span class="n">js</span>
|
||
<span class="n">bar</span><span class="o">.</span><span class="n">js</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>If <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">json</span></code> succeeded due to the mere presence of the <code class="docutils literal notranslate"><span class="pre">json/</span></code>
|
||
subdirectory, the code would incorrectly believe that the <code class="docutils literal notranslate"><span class="pre">json</span></code>
|
||
module was available, and proceed to fail with an error.</p>
|
||
<p>However, we can prevent corner cases like these from arising, simply
|
||
by making one small change to the algorithm presented so far. Instead
|
||
of allowing you to import a “pure virtual” package (like <code class="docutils literal notranslate"><span class="pre">zc</span></code>),
|
||
we allow only importing of the <em>contents</em> of virtual packages.</p>
|
||
<p>That is, a statement like <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">zc</span></code> should raise <code class="docutils literal notranslate"><span class="pre">ImportError</span></code>
|
||
if there is no <code class="docutils literal notranslate"><span class="pre">zc.py</span></code> or <code class="docutils literal notranslate"><span class="pre">zc/__init__.py</span></code> on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>. But,
|
||
doing <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">zc.buildout</span></code> should still succeed, as long as there’s
|
||
a <code class="docutils literal notranslate"><span class="pre">zc/buildout.py</span></code> or <code class="docutils literal notranslate"><span class="pre">zc/buildout/__init__.py</span></code> on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>.</p>
|
||
<p>In other words, we don’t allow pure virtual packages to be imported
|
||
directly, only modules and self-contained packages. (This is an
|
||
acceptable limitation, because there is no <em>functional</em> value to
|
||
importing such a package by itself. After all, the module object
|
||
will have no <em>contents</em> until you import at least one of its
|
||
subpackages or submodules!)</p>
|
||
<p>Once <code class="docutils literal notranslate"><span class="pre">zc.buildout</span></code> has been successfully imported, though, there
|
||
<em>will</em> be a <code class="docutils literal notranslate"><span class="pre">zc</span></code> module in <code class="docutils literal notranslate"><span class="pre">sys.modules</span></code>, and trying to import it
|
||
will of course succeed. We are only preventing an <em>initial</em> import
|
||
from succeeding, in order to prevent false-positive import successes
|
||
when clashing subdirectories are present on <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>.</p>
|
||
<p>So, with this slight change, the <code class="docutils literal notranslate"><span class="pre">datagen.py</span></code> example above will
|
||
work correctly. When it does <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">json</span></code>, the mere presence of a
|
||
<code class="docutils literal notranslate"><span class="pre">json/</span></code> directory will simply not affect the import process at all,
|
||
even if it contains <code class="docutils literal notranslate"><span class="pre">.py</span></code> files. The <code class="docutils literal notranslate"><span class="pre">json/</span></code> directory will still
|
||
only be searched in the case where an import like <code class="docutils literal notranslate"><span class="pre">import</span>
|
||
<span class="pre">json.converter</span></code> is attempted.</p>
|
||
<p>Meanwhile, tools that expect to locate packages and modules by
|
||
walking a directory tree can be updated to use the existing
|
||
<code class="docutils literal notranslate"><span class="pre">pkgutil.walk_modules()</span></code> API, and tools that need to inspect
|
||
packages in memory should use the other APIs described in the
|
||
<a class="reference internal" href="#standard-library-changes-additions">Standard Library Changes/Additions</a> section below.</p>
|
||
</section>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<p>A change is made to the existing import process, when importing
|
||
names containing at least one <code class="docutils literal notranslate"><span class="pre">.</span></code> – that is, imports of modules
|
||
that have a parent package.</p>
|
||
<p>Specifically, if the parent package does not exist, or exists but
|
||
lacks a <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attribute, an attempt is first made to create a
|
||
“virtual path” for the parent package (following the algorithm
|
||
described in the section on <a class="reference internal" href="#virtual-paths">virtual paths</a>, below).</p>
|
||
<p>If the computed “virtual path” is empty, an <code class="docutils literal notranslate"><span class="pre">ImportError</span></code> results,
|
||
just as it would today. However, if a non-empty virtual path is
|
||
obtained, the normal import of the submodule or subpackage proceeds,
|
||
using that virtual path to find the submodule or subpackage. (Just
|
||
as it would have with the parent’s <code class="docutils literal notranslate"><span class="pre">__path__</span></code>, if the parent package
|
||
had existed and had a <code class="docutils literal notranslate"><span class="pre">__path__</span></code>.)</p>
|
||
<p>When a submodule or subpackage is found (but not yet loaded),
|
||
the parent package is created and added to <code class="docutils literal notranslate"><span class="pre">sys.modules</span></code> (if it
|
||
didn’t exist before), and its <code class="docutils literal notranslate"><span class="pre">__path__</span></code> is set to the computed
|
||
virtual path (if it wasn’t already set).</p>
|
||
<p>In this way, when the actual loading of the submodule or subpackage
|
||
occurs, it will see a parent package existing, and any relative
|
||
imports will work correctly. However, if no submodule or subpackage
|
||
exists, then the parent package will <em>not</em> be created, nor will a
|
||
standalone module be converted into a package (by the addition of a
|
||
spurious <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attribute).</p>
|
||
<p>Note, by the way, that this change must be applied <em>recursively</em>: that
|
||
is, if <code class="docutils literal notranslate"><span class="pre">foo</span></code> and <code class="docutils literal notranslate"><span class="pre">foo.bar</span></code> are pure virtual packages, then
|
||
<code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">foo.bar.baz</span></code> must wait until <code class="docutils literal notranslate"><span class="pre">foo.bar.baz</span></code> is found before
|
||
creating module objects for <em>both</em> <code class="docutils literal notranslate"><span class="pre">foo</span></code> and <code class="docutils literal notranslate"><span class="pre">foo.bar</span></code>, and then
|
||
create both of them together, properly setting the <code class="docutils literal notranslate"><span class="pre">foo</span></code> module’s
|
||
<code class="docutils literal notranslate"><span class="pre">.bar</span></code> attribute to point to the <code class="docutils literal notranslate"><span class="pre">foo.bar</span></code> module.</p>
|
||
<p>In this way, pure virtual packages are never directly importable:
|
||
an <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">foo</span></code> or <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">foo.bar</span></code> by itself will fail, and the
|
||
corresponding modules will not appear in <code class="docutils literal notranslate"><span class="pre">sys.modules</span></code> until they
|
||
are needed to point to a <em>successfully</em> imported submodule or
|
||
self-contained subpackage.</p>
|
||
<section id="virtual-paths">
|
||
<h3><a class="toc-backref" href="#virtual-paths" role="doc-backlink">Virtual Paths</a></h3>
|
||
<p>A virtual path is created by obtaining a <a class="pep reference internal" href="../pep-0302/" title="PEP 302 – New Import Hooks">PEP 302</a> “importer” object for
|
||
each of the path entries found in <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> (for a top-level
|
||
module) or the parent <code class="docutils literal notranslate"><span class="pre">__path__</span></code> (for a submodule).</p>
|
||
<p>(Note: because <code class="docutils literal notranslate"><span class="pre">sys.meta_path</span></code> importers are not associated with
|
||
<code class="docutils literal notranslate"><span class="pre">sys.path</span></code> or <code class="docutils literal notranslate"><span class="pre">__path__</span></code> entry strings, such importers do <em>not</em>
|
||
participate in this process.)</p>
|
||
<p>Each importer is checked for a <code class="docutils literal notranslate"><span class="pre">get_subpath()</span></code> method, and if
|
||
present, the method is called with the full name of the module/package
|
||
the path is being constructed for. The return value is either a
|
||
string representing a subdirectory for the requested package, or
|
||
<code class="docutils literal notranslate"><span class="pre">None</span></code> if no such subdirectory exists.</p>
|
||
<p>The strings returned by the importers are added to the path list
|
||
being built, in the same order as they are found. (<code class="docutils literal notranslate"><span class="pre">None</span></code> values
|
||
and missing <code class="docutils literal notranslate"><span class="pre">get_subpath()</span></code> methods are simply skipped.)</p>
|
||
<p>The resulting list (whether empty or not) is then stored in a
|
||
<code class="docutils literal notranslate"><span class="pre">sys.virtual_package_paths</span></code> dictionary, keyed by module name.</p>
|
||
<p>This dictionary has two purposes. First, it serves as a cache, in
|
||
the event that more than one attempt is made to import a submodule
|
||
of a virtual package.</p>
|
||
<p>Second, and more importantly, the dictionary can be used by code that
|
||
extends <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> at runtime to <em>update</em> imported packages’
|
||
<code class="docutils literal notranslate"><span class="pre">__path__</span></code> attributes accordingly. (See <a class="reference internal" href="#standard-library-changes-additions">Standard Library
|
||
Changes/Additions</a> below for more details.)</p>
|
||
<p>In Python code, the virtual path construction algorithm would look
|
||
something like this:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">get_virtual_path</span><span class="p">(</span><span class="n">modulename</span><span class="p">,</span> <span class="n">parent_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
|
||
<span class="k">if</span> <span class="n">modulename</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">virtual_package_paths</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="n">sys</span><span class="o">.</span><span class="n">virtual_package_paths</span><span class="p">[</span><span class="n">modulename</span><span class="p">]</span>
|
||
|
||
<span class="k">if</span> <span class="n">parent_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">parent_path</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span>
|
||
|
||
<span class="n">path</span> <span class="o">=</span> <span class="p">[]</span>
|
||
|
||
<span class="k">for</span> <span class="n">entry</span> <span class="ow">in</span> <span class="n">parent_path</span><span class="p">:</span>
|
||
<span class="c1"># Obtain a PEP 302 importer object - see pkgutil module</span>
|
||
<span class="n">importer</span> <span class="o">=</span> <span class="n">pkgutil</span><span class="o">.</span><span class="n">get_importer</span><span class="p">(</span><span class="n">entry</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">importer</span><span class="p">,</span> <span class="s1">'get_subpath'</span><span class="p">):</span>
|
||
<span class="n">subpath</span> <span class="o">=</span> <span class="n">importer</span><span class="o">.</span><span class="n">get_subpath</span><span class="p">(</span><span class="n">modulename</span><span class="p">)</span>
|
||
<span class="k">if</span> <span class="n">subpath</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">subpath</span><span class="p">)</span>
|
||
|
||
<span class="n">sys</span><span class="o">.</span><span class="n">virtual_package_paths</span><span class="p">[</span><span class="n">modulename</span><span class="p">]</span> <span class="o">=</span> <span class="n">path</span>
|
||
<span class="k">return</span> <span class="n">path</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>And a function like this one should be exposed in the standard
|
||
library as e.g. <code class="docutils literal notranslate"><span class="pre">imp.get_virtual_path()</span></code>, so that people creating
|
||
<code class="docutils literal notranslate"><span class="pre">__import__</span></code> replacements or <code class="docutils literal notranslate"><span class="pre">sys.meta_path</span></code> hooks can reuse it.</p>
|
||
</section>
|
||
<section id="standard-library-changes-additions">
|
||
<h3><a class="toc-backref" href="#standard-library-changes-additions" role="doc-backlink">Standard Library Changes/Additions</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">pkgutil</span></code> module should be updated to handle this
|
||
specification appropriately, including any necessary changes to
|
||
<code class="docutils literal notranslate"><span class="pre">extend_path()</span></code>, <code class="docutils literal notranslate"><span class="pre">iter_modules()</span></code>, etc.</p>
|
||
<p>Specifically the proposed changes and additions to <code class="docutils literal notranslate"><span class="pre">pkgutil</span></code> are:</p>
|
||
<ul>
|
||
<li>A new <code class="docutils literal notranslate"><span class="pre">extend_virtual_paths(path_entry)</span></code> function, to extend
|
||
existing, already-imported virtual packages’ <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attributes
|
||
to include any portions found in a new <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> entry. This
|
||
function should be called by applications extending <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>
|
||
at runtime, e.g. when adding a plugin directory or an egg to the
|
||
path.<p>The implementation of this function does a simple top-down traversal
|
||
of <code class="docutils literal notranslate"><span class="pre">sys.virtual_package_paths</span></code>, and performs any necessary
|
||
<code class="docutils literal notranslate"><span class="pre">get_subpath()</span></code> calls to identify what path entries need to be
|
||
added to the virtual path for that package, given that <code class="docutils literal notranslate"><span class="pre">path_entry</span></code>
|
||
has been added to <code class="docutils literal notranslate"><span class="pre">sys.path</span></code>. (Or, in the case of sub-packages,
|
||
adding a derived subpath entry, based on their parent package’s
|
||
virtual path.)</p>
|
||
<p>(Note: this function must update both the path values in
|
||
<code class="docutils literal notranslate"><span class="pre">sys.virtual_package_paths</span></code> as well as the <code class="docutils literal notranslate"><span class="pre">__path__</span></code> attributes
|
||
of any corresponding modules in <code class="docutils literal notranslate"><span class="pre">sys.modules</span></code>, even though in the
|
||
common case they will both be the same <code class="docutils literal notranslate"><span class="pre">list</span></code> object.)</p>
|
||
</li>
|
||
<li>A new <code class="docutils literal notranslate"><span class="pre">iter_virtual_packages(parent='')</span></code> function to allow
|
||
top-down traversal of virtual packages from
|
||
<code class="docutils literal notranslate"><span class="pre">sys.virtual_package_paths</span></code>, by yielding the child virtual
|
||
packages of <code class="docutils literal notranslate"><span class="pre">parent</span></code>. For example, calling
|
||
<code class="docutils literal notranslate"><span class="pre">iter_virtual_packages("zope")</span></code> might yield <code class="docutils literal notranslate"><span class="pre">zope.app</span></code>
|
||
and <code class="docutils literal notranslate"><span class="pre">zope.products</span></code> (if they are virtual packages listed in
|
||
<code class="docutils literal notranslate"><span class="pre">sys.virtual_package_paths</span></code>), but <strong>not</strong> <code class="docutils literal notranslate"><span class="pre">zope.foo.bar</span></code>.
|
||
(This function is needed to implement <code class="docutils literal notranslate"><span class="pre">extend_virtual_paths()</span></code>,
|
||
but is also potentially useful for other code that needs to inspect
|
||
imported virtual packages.)</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">ImpImporter.iter_modules()</span></code> should be changed to also detect and
|
||
yield the names of modules found in virtual packages.</li>
|
||
</ul>
|
||
<p>In addition to the above changes, the <code class="docutils literal notranslate"><span class="pre">zipimport</span></code> importer should
|
||
have its <code class="docutils literal notranslate"><span class="pre">iter_modules()</span></code> implementation similarly changed. (Note:
|
||
current versions of Python implement this via a shim in <code class="docutils literal notranslate"><span class="pre">pkgutil</span></code>,
|
||
so technically this is also a change to <code class="docutils literal notranslate"><span class="pre">pkgutil</span></code>.)</p>
|
||
<p>Last, but not least, the <code class="docutils literal notranslate"><span class="pre">imp</span></code> module (or <code class="docutils literal notranslate"><span class="pre">importlib</span></code>, if
|
||
appropriate) should expose the algorithm described in the <a class="reference internal" href="#virtual-paths">virtual
|
||
paths</a> section above, as a
|
||
<code class="docutils literal notranslate"><span class="pre">get_virtual_path(modulename,</span> <span class="pre">parent_path=None)</span></code> function, so that
|
||
creators of <code class="docutils literal notranslate"><span class="pre">__import__</span></code> replacements can use it.</p>
|
||
</section>
|
||
<section id="implementation-notes">
|
||
<h3><a class="toc-backref" href="#implementation-notes" role="doc-backlink">Implementation Notes</a></h3>
|
||
<p>For users, developers, and distributors of virtual packages:</p>
|
||
<ul>
|
||
<li>While virtual packages are easy to set up and use, there is still
|
||
a time and place for using self-contained packages. While it’s not
|
||
strictly necessary, adding an <code class="docutils literal notranslate"><span class="pre">__init__</span></code> module to your
|
||
self-contained packages lets users of the package (and Python
|
||
itself) know that <em>all</em> of the package’s code will be found in
|
||
that single subdirectory. In addition, it lets you define
|
||
<code class="docutils literal notranslate"><span class="pre">__all__</span></code>, expose a public API, provide a package-level docstring,
|
||
and do other things that make more sense for a self-contained
|
||
project than for a mere “namespace” package.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">sys.virtual_package_paths</span></code> is allowed to contain entries for
|
||
non-existent or not-yet-imported package names; code that uses its
|
||
contents should not assume that every key in this dictionary is also
|
||
present in <code class="docutils literal notranslate"><span class="pre">sys.modules</span></code> or that importing the name will
|
||
necessarily succeed.</li>
|
||
<li>If you are changing a currently self-contained package into a
|
||
virtual one, it’s important to note that you can no longer use its
|
||
<code class="docutils literal notranslate"><span class="pre">__file__</span></code> attribute to locate data files stored in a package
|
||
directory. Instead, you must search <code class="docutils literal notranslate"><span class="pre">__path__</span></code> or use the
|
||
<code class="docutils literal notranslate"><span class="pre">__file__</span></code> of a submodule adjacent to the desired files, or
|
||
of a self-contained subpackage that contains the desired files.<p>(Note: this caveat is already true for existing users of “namespace
|
||
packages” today. That is, it is an inherent result of being able
|
||
to partition a package, that you must know <em>which</em> partition the
|
||
desired data file lives in. We mention it here simply so that
|
||
<em>new</em> users converting from self-contained to virtual packages will
|
||
also be aware of it.)</p>
|
||
</li>
|
||
<li>XXX what is the __file__ of a “pure virtual” package? <code class="docutils literal notranslate"><span class="pre">None</span></code>?
|
||
Some arbitrary string? The path of the first directory with a
|
||
trailing separator? No matter what we put, <em>some</em> code is
|
||
going to break, but the last choice might allow some code to
|
||
accidentally work. Is that good or bad?</li>
|
||
</ul>
|
||
<p>For those implementing <a class="pep reference internal" href="../pep-0302/" title="PEP 302 – New Import Hooks">PEP 302</a> importer objects:</p>
|
||
<ul>
|
||
<li>Importers that support the <code class="docutils literal notranslate"><span class="pre">iter_modules()</span></code> method (used by
|
||
<code class="docutils literal notranslate"><span class="pre">pkgutil</span></code> to locate importable modules and packages) and want to
|
||
add virtual package support should modify their <code class="docutils literal notranslate"><span class="pre">iter_modules()</span></code>
|
||
method so that it discovers and lists virtual packages as well as
|
||
standard modules and packages. To do this, the importer should
|
||
simply list all immediate subdirectory names in its jurisdiction
|
||
that are valid Python identifiers.<p>XXX This might list a lot of not-really-packages. Should we
|
||
require importable contents to exist? If so, how deep do we
|
||
search, and how do we prevent e.g. link loops, or traversing onto
|
||
different filesystems, etc.? Ick. Also, if virtual packages are
|
||
listed, they still can’t be <em>imported</em>, which is a problem for the
|
||
way that <code class="docutils literal notranslate"><span class="pre">pkgutil.walk_modules()</span></code> is currently implemented.</p>
|
||
</li>
|
||
<li>“Meta” importers (i.e., importers placed on <code class="docutils literal notranslate"><span class="pre">sys.meta_path</span></code>) do
|
||
not need to implement <code class="docutils literal notranslate"><span class="pre">get_subpath()</span></code>, because the method
|
||
is only called on importers corresponding to <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> entries
|
||
and <code class="docutils literal notranslate"><span class="pre">__path__</span></code> entries. If a meta importer wishes to support
|
||
virtual packages, it must do so entirely within its own
|
||
<code class="docutils literal notranslate"><span class="pre">find_module()</span></code> implementation.<p>Unfortunately, it is unlikely that any such implementation will be
|
||
able to merge its package subpaths with those of other meta
|
||
importers or <code class="docutils literal notranslate"><span class="pre">sys.path</span></code> importers, so the meaning of “supporting
|
||
virtual packages” for a meta importer is currently undefined!</p>
|
||
<p>(However, since the intended use case for meta importers is to
|
||
replace Python’s normal import process entirely for some subset of
|
||
modules, and the number of such importers currently implemented is
|
||
quite small, this seems unlikely to be a big issue in practice.)</p>
|
||
</li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id4" role="doc-footnote">
|
||
<dt class="label" id="id4">[<a href="#id2">1</a>]</dt>
|
||
<dd>“namespace” vs “module” packages (mailing list thread)
|
||
(<a class="reference external" href="http://mail.zope.org/pipermail/zope3-dev/2002-December/004251.html">http://mail.zope.org/pipermail/zope3-dev/2002-December/004251.html</a>)</aside>
|
||
<aside class="footnote brackets" id="id5" role="doc-footnote">
|
||
<dt class="label" id="id5">[<a href="#id3">2</a>]</dt>
|
||
<dd>“Dropping __init__.py requirement for subpackages”
|
||
(<a class="reference external" href="https://mail.python.org/pipermail/python-dev/2006-April/064400.html">https://mail.python.org/pipermail/python-dev/2006-April/064400.html</a>)</aside>
|
||
<aside class="footnote brackets" id="id6" role="doc-footnote">
|
||
<dt class="label" id="id6">[<a href="#id1">3</a>]</dt>
|
||
<dd>Namespace Packages resolution
|
||
(<a class="reference external" href="https://mail.python.org/pipermail/import-sig/2012-March/000421.html">https://mail.python.org/pipermail/import-sig/2012-March/000421.html</a>)</aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0402.rst">https://github.com/python/peps/blob/main/peps/pep-0402.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0402.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#rejection-notice">Rejection Notice</a></li>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#the-problem">The Problem</a></li>
|
||
<li><a class="reference internal" href="#the-solution">The Solution</a><ul>
|
||
<li><a class="reference internal" href="#a-thought-experiment">A Thought Experiment</a></li>
|
||
<li><a class="reference internal" href="#self-contained-vs-virtual-packages">Self-Contained vs. “Virtual” Packages</a></li>
|
||
<li><a class="reference internal" href="#backwards-compatibility-and-performance">Backwards Compatibility and Performance</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#virtual-paths">Virtual Paths</a></li>
|
||
<li><a class="reference internal" href="#standard-library-changes-additions">Standard Library Changes/Additions</a></li>
|
||
<li><a class="reference internal" href="#implementation-notes">Implementation Notes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0402.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |