296 lines
22 KiB
HTML
296 lines
22 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 547 – Running extension modules using the -m option | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0547/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 547 – Running extension modules using the -m option | peps.python.org'>
|
||
<meta property="og:description" content="This PEP proposes implementation that allows built-in and extension modules to be executed in the __main__ namespace using the PEP 489 multi-phase initialization.">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0547/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="This PEP proposes implementation that allows built-in and extension modules to be executed in the __main__ namespace using the PEP 489 multi-phase initialization.">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 547</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 547 – Running extension modules using the -m option</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Marcel Plch <gmarcel.plch at gmail.com>,
|
||
Petr Viktorin <encukou at gmail.com></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Inactive draft that may be taken up again at a later time">Deferred</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">25-May-2017</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.7</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p></p></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#deferral-notice">Deferral Notice</a></li>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#background">Background</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#extensionfileloader-changes">ExtensionFileLoader Changes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="deferral-notice">
|
||
<h2><a class="toc-backref" href="#deferral-notice" role="doc-backlink">Deferral Notice</a></h2>
|
||
<p>Cython – the most important use case for this PEP and the only explicit
|
||
one – is not ready for multi-phase initialization yet.
|
||
It keeps global state in C-level static variables.
|
||
See discussion at <a class="reference external" href="https://github.com/cython/cython/pull/1923">Cython issue 1923</a>.</p>
|
||
<p>The PEP is deferred until the situation changes.</p>
|
||
</section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP proposes implementation that allows built-in and extension
|
||
modules to be executed in the <code class="docutils literal notranslate"><span class="pre">__main__</span></code> namespace using
|
||
the <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a> multi-phase initialization.</p>
|
||
<p>With this, a multi-phase initialization enabled module can be run
|
||
using following command:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>$ python3 -m _testmultiphase
|
||
This is a test module named __main__.
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="motivation">
|
||
<h2><a class="toc-backref" href="#motivation" role="doc-backlink">Motivation</a></h2>
|
||
<p>Currently, extension modules do not support all functionality of
|
||
Python source modules.
|
||
Specifically, it is not possible to run extension modules as scripts using
|
||
Python’s <code class="docutils literal notranslate"><span class="pre">-m</span></code> option.</p>
|
||
<p>The technical groundwork to make this possible has been done for <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a>,
|
||
and enabling the <code class="docutils literal notranslate"><span class="pre">-m</span></code> option is listed in that PEP’s
|
||
“Possible Future Extensions” section.
|
||
Technically, the additional changes proposed here are relatively small.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Extension modules’ lack of support for the <code class="docutils literal notranslate"><span class="pre">-m</span></code> option has traditionally
|
||
been worked around by providing a Python wrapper.
|
||
For example, the <code class="docutils literal notranslate"><span class="pre">_pickle</span></code> module’s command line interface is in the
|
||
pure-Python <code class="docutils literal notranslate"><span class="pre">pickle</span></code> module (along with a pure-Python reimplementation).</p>
|
||
<p>This works well for standard library modules, as building command line
|
||
interfaces using the C API is cumbersome.
|
||
However, other users may want to create executable extension modules directly.</p>
|
||
<p>An important use case is Cython, a Python-like language that compiles to
|
||
C extension modules.
|
||
Cython is a (near) superset of Python, meaning that compiling a Python module
|
||
with Cython will typically not change the module’s functionality, allowing
|
||
Cython-specific features to be added gradually.
|
||
This PEP will allow Cython extension modules to behave the same as their Python
|
||
counterparts when run using the <code class="docutils literal notranslate"><span class="pre">-m</span></code> option.
|
||
Cython developers consider the feature worth implementing (see
|
||
<a class="reference external" href="https://github.com/cython/cython/issues/1715">Cython issue 1715</a>).</p>
|
||
</section>
|
||
<section id="background">
|
||
<h2><a class="toc-backref" href="#background" role="doc-backlink">Background</a></h2>
|
||
<p>Python’s <code class="docutils literal notranslate"><span class="pre">-m</span></code> option is handled by the function
|
||
<code class="docutils literal notranslate"><span class="pre">runpy._run_module_as_main</span></code>.</p>
|
||
<p>The module specified by <code class="docutils literal notranslate"><span class="pre">-m</span></code> is not imported normally.
|
||
Instead, it is executed in the namespace of the <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module,
|
||
which is created quite early in interpreter initialization.</p>
|
||
<p>For Python source modules, running in another module’s namespace is not
|
||
a problem: the code is executed with <code class="docutils literal notranslate"><span class="pre">locals</span></code> and <code class="docutils literal notranslate"><span class="pre">globals</span></code> set to the
|
||
existing module’s <code class="docutils literal notranslate"><span class="pre">__dict__</span></code>.
|
||
This is not the case for extension modules, whose <code class="docutils literal notranslate"><span class="pre">PyInit_*</span></code> entry point
|
||
traditionally both created a new module object (using <code class="docutils literal notranslate"><span class="pre">PyModule_Create</span></code>),
|
||
and initialized it.</p>
|
||
<p>Since Python 3.5, extension modules can use <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a> multi-phase initialization.
|
||
In this scenario, the <code class="docutils literal notranslate"><span class="pre">PyInit_*</span></code> entry point returns a <code class="docutils literal notranslate"><span class="pre">PyModuleDef</span></code>
|
||
structure: a description of how the module should be created and initialized.
|
||
The extension can choose to customize creation of the module object using
|
||
the <code class="docutils literal notranslate"><span class="pre">Py_mod_create</span></code> callback, or opt to use a normal module object by not
|
||
specifying <code class="docutils literal notranslate"><span class="pre">Py_mod_create</span></code>.
|
||
Another callback, <code class="docutils literal notranslate"><span class="pre">Py_mod_exec</span></code>, is then called to initialize the module
|
||
object, e.g. by populating it with methods and classes.</p>
|
||
</section>
|
||
<section id="proposal">
|
||
<h2><a class="toc-backref" href="#proposal" role="doc-backlink">Proposal</a></h2>
|
||
<p>Multi-phase initialization makes it possible to execute an extension module in
|
||
another module’s namespace: if a <code class="docutils literal notranslate"><span class="pre">Py_mod_create</span></code> callback is not specified,
|
||
the <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module can be passed to the <code class="docutils literal notranslate"><span class="pre">Py_mod_exec</span></code> callback to be
|
||
initialized, as if <code class="docutils literal notranslate"><span class="pre">__main__</span></code> was a freshly constructed module object.</p>
|
||
<p>One complication in this scheme is C-level module state.
|
||
Each module has a <code class="docutils literal notranslate"><span class="pre">md_state</span></code> pointer that points to a region of memory
|
||
allocated when an extension module is created.
|
||
The <code class="docutils literal notranslate"><span class="pre">PyModuleDef</span></code> specifies how much memory is to be allocated.</p>
|
||
<p>The implementation must take care that <code class="docutils literal notranslate"><span class="pre">md_state</span></code> memory is allocated at most
|
||
once.
|
||
Also, the <code class="docutils literal notranslate"><span class="pre">Py_mod_exec</span></code> callback should only be called once per module.
|
||
The implications of multiply-initialized modules are too subtle to require
|
||
expecting extension authors to reason about them.
|
||
The <code class="docutils literal notranslate"><span class="pre">md_state</span></code> pointer itself will serve as a guard: allocating the memory
|
||
and calling <code class="docutils literal notranslate"><span class="pre">Py_mod_exec</span></code> will always be done together, and initializing an
|
||
extension module will fail if <code class="docutils literal notranslate"><span class="pre">md_state</span></code> is already non-NULL.</p>
|
||
<p>Since the <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module is not created as an extension module,
|
||
its <code class="docutils literal notranslate"><span class="pre">md_state</span></code> is normally <code class="docutils literal notranslate"><span class="pre">NULL</span></code>.
|
||
Before initializing an extension module in <code class="docutils literal notranslate"><span class="pre">__main__</span></code>’s context, its module
|
||
state will be allocated according to the <code class="docutils literal notranslate"><span class="pre">PyModuleDef</span></code> of that module.</p>
|
||
<p>While <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a> was designed to make these changes generally possible,
|
||
it’s necessary to decouple module discovery, creation, and initialization
|
||
steps for extension modules, so that another module can be used instead of
|
||
a newly initialized one, and the functionality needs to be added to
|
||
<code class="docutils literal notranslate"><span class="pre">runpy</span></code> and <code class="docutils literal notranslate"><span class="pre">importlib</span></code>.</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<p>A new optional method for importlib loaders will be added.
|
||
This method will be called <code class="docutils literal notranslate"><span class="pre">exec_in_module</span></code> and will take two
|
||
positional arguments: module spec and an already existing module.
|
||
Any import-related attributes, such as <code class="docutils literal notranslate"><span class="pre">__spec__</span></code> or <code class="docutils literal notranslate"><span class="pre">__name__</span></code>,
|
||
already set on the module will be ignored.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">runpy._run_module_as_main</span></code> function will look for this new
|
||
loader method.
|
||
If it is present, <code class="docutils literal notranslate"><span class="pre">runpy</span></code> will execute it instead of trying to load and
|
||
run the module’s Python code.
|
||
Otherwise, <code class="docutils literal notranslate"><span class="pre">runpy</span></code> will act as before.</p>
|
||
<section id="extensionfileloader-changes">
|
||
<h3><a class="toc-backref" href="#extensionfileloader-changes" role="doc-backlink">ExtensionFileLoader Changes</a></h3>
|
||
<p>importlib’s <code class="docutils literal notranslate"><span class="pre">ExtensionFileLoader</span></code> will get an implementation of
|
||
<code class="docutils literal notranslate"><span class="pre">exec_in_module</span></code> that will call a new function, <code class="docutils literal notranslate"><span class="pre">_imp.exec_in_module</span></code>.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">_imp.exec_in_module</span></code> will use existing machinery to find and call an
|
||
extension module’s <code class="docutils literal notranslate"><span class="pre">PyInit_*</span></code> function.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">PyInit_*</span></code> function can return either a fully initialized module
|
||
(single-phase initialization) or a <code class="docutils literal notranslate"><span class="pre">PyModuleDef</span></code> (for <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a> multi-phase
|
||
initialization).</p>
|
||
<p>In the single-phase initialization case, <code class="docutils literal notranslate"><span class="pre">_imp.exec_in_module</span></code> will raise
|
||
<code class="docutils literal notranslate"><span class="pre">ImportError</span></code>.</p>
|
||
<p>In the multi-phase initialization case, the <code class="docutils literal notranslate"><span class="pre">PyModuleDef</span></code> and the module to
|
||
be initialized will be passed to a new function, <code class="docutils literal notranslate"><span class="pre">PyModule_ExecInModule</span></code>.</p>
|
||
<p>This function raises <code class="docutils literal notranslate"><span class="pre">ImportError</span></code> if the <code class="docutils literal notranslate"><span class="pre">PyModuleDef</span></code> specifies
|
||
a <code class="docutils literal notranslate"><span class="pre">Py_mod_create</span></code> slot, or if the module has already been initialized
|
||
(i.e. its <code class="docutils literal notranslate"><span class="pre">md_state</span></code> pointer is not <code class="docutils literal notranslate"><span class="pre">NULL</span></code>).
|
||
Otherwise, the function will initialize the module according to the
|
||
<code class="docutils literal notranslate"><span class="pre">PyModuleDef</span></code>.</p>
|
||
</section>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards Compatibility</a></h2>
|
||
<p>This PEP maintains backwards compatibility.
|
||
It only adds new functions, and a new loader method that is added for
|
||
a loader that previously did not support running modules as <code class="docutils literal notranslate"><span class="pre">__main__</span></code>.</p>
|
||
</section>
|
||
<section id="reference-implementation">
|
||
<h2><a class="toc-backref" href="#reference-implementation" role="doc-backlink">Reference Implementation</a></h2>
|
||
<p>The reference implementation of this PEP is available at <a class="reference external" href="https://github.com/python/cpython/pull/1761">GitHub</a>.</p>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0547.rst">https://github.com/python/peps/blob/main/peps/pep-0547.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0547.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#deferral-notice">Deferral Notice</a></li>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#background">Background</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#extensionfileloader-changes">ExtensionFileLoader Changes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0547.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |