2084 lines
175 KiB
HTML
2084 lines
175 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 554 – Multiple Interpreters in the Stdlib | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0554/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 554 – Multiple Interpreters in the Stdlib | peps.python.org'>
|
||
<meta property="og:description" content="CPython has supported multiple interpreters in the same process (AKA “subinterpreters”) since version 1.5 (1997). The feature has been available via the C-API. [c-api] Multiple interpreters operate in relative isolation from one another, which facilit...">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0554/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="CPython has supported multiple interpreters in the same process (AKA “subinterpreters”) since version 1.5 (1997). The feature has been available via the C-API. [c-api] Multiple interpreters operate in relative isolation from one another, which facilit...">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 554</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 554 – Multiple Interpreters in the Stdlib</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Eric Snow <ericsnowcurrently at gmail.com></dd>
|
||
<dt class="field-even">Discussions-To<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="https://discuss.python.org/t/pep-554-multiple-interpreters-in-the-stdlib/24855">Discourse thread</a></dd>
|
||
<dt class="field-odd">Status<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Replaced by another succeeding PEP">Superseded</abbr></dd>
|
||
<dt class="field-even">Type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-odd">Created<span class="colon">:</span></dt>
|
||
<dd class="field-odd">05-Sep-2017</dd>
|
||
<dt class="field-even">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-even">3.13</dd>
|
||
<dt class="field-odd">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://mail.python.org/archives/list/python-ideas@python.org/thread/HQQWEE527HG3ILJVKQTXVSJIQO6NUSIA/" title="Python-Ideas thread">07-Sep-2017</a>,
|
||
<a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/NBWMA6LVD22XOUYC5ZMPBFWDQOECRP77/" title="Python-Dev thread">08-Sep-2017</a>,
|
||
<a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/EG4FSFG5E3O22FTIUQOXMQ6X6B5X3DP7/" title="Python-Dev thread">13-Sep-2017</a>,
|
||
<a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/BCSRGAMCYB3NGXNU42U66J56XNZVMQP2/" title="Python-Dev thread">05-Dec-2017</a>,
|
||
<a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/X2KPCSRVBD2QD5GP5IMXXZTGZ46OXD3D/" title="Python-Dev thread">04-May-2020</a>,
|
||
<a class="reference external" href="https://discuss.python.org/t/pep-554-multiple-interpreters-in-the-stdlib/24855/2/" title="Discourse message">14-Mar-2023</a>,
|
||
<a class="reference external" href="https://discuss.python.org/t/pep-554-multiple-interpreters-in-the-stdlib/24855/26/" title="Discourse message">01-Nov-2023</a></dd>
|
||
<dt class="field-even">Superseded-By<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="../pep-0734/">734</a></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a><ul>
|
||
<li><a class="reference internal" href="#the-interpreters-module">The “interpreters” Module</a></li>
|
||
<li><a class="reference internal" href="#api-summary-for-interpreters-module">API summary for interpreters module</a></li>
|
||
<li><a class="reference internal" href="#concurrent-futures-interpreterpoolexecutor">concurrent.futures.InterpreterPoolExecutor</a></li>
|
||
<li><a class="reference internal" href="#help-for-extension-module-maintainers">Help for Extension Module Maintainers</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#examples">Examples</a><ul>
|
||
<li><a class="reference internal" href="#run-isolated-code-in-current-os-thread">Run isolated code in current OS thread</a></li>
|
||
<li><a class="reference internal" href="#run-in-a-different-thread">Run in a different thread</a></li>
|
||
<li><a class="reference internal" href="#pre-populate-an-interpreter">Pre-populate an interpreter</a></li>
|
||
<li><a class="reference internal" href="#handling-an-exception">Handling an exception</a></li>
|
||
<li><a class="reference internal" href="#re-raising-an-exception">Re-raising an exception</a></li>
|
||
<li><a class="reference internal" href="#interact-with-the-main-namespace">Interact with the __main__ namespace</a></li>
|
||
<li><a class="reference internal" href="#synchronize-using-an-os-pipe">Synchronize using an OS pipe</a></li>
|
||
<li><a class="reference internal" href="#sharing-a-file-descriptor">Sharing a file descriptor</a></li>
|
||
<li><a class="reference internal" href="#passing-objects-via-pickle">Passing objects via pickle</a></li>
|
||
<li><a class="reference internal" href="#capturing-an-interpreter-s-stdout">Capturing an interpreter’s stdout</a></li>
|
||
<li><a class="reference internal" href="#running-a-module">Running a module</a></li>
|
||
<li><a class="reference internal" href="#running-as-script-including-zip-archives-directories">Running as script (including zip archives & directories)</a></li>
|
||
<li><a class="reference internal" href="#using-a-channel-to-communicate">Using a channel to communicate</a></li>
|
||
<li><a class="reference internal" href="#sharing-a-memoryview-imagine-map-reduce">Sharing a memoryview (imagine map-reduce)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a><ul>
|
||
<li><a class="reference internal" href="#concerns">Concerns</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#about-subinterpreters">About Subinterpreters</a><ul>
|
||
<li><a class="reference internal" href="#concurrency">Concurrency</a></li>
|
||
<li><a class="reference internal" href="#shared-data">Shared Data</a></li>
|
||
<li><a class="reference internal" href="#interpreter-isolation">Interpreter Isolation</a></li>
|
||
<li><a class="reference internal" href="#existing-usage">Existing Usage</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#alternate-python-implementations">Alternate Python Implementations</a></li>
|
||
<li><a class="reference internal" href="#interpreters-module-api">“interpreters” Module API</a><ul>
|
||
<li><a class="reference internal" href="#uncaught-exceptions">Uncaught Exceptions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#interpreter-restrictions">Interpreter Restrictions</a></li>
|
||
<li><a class="reference internal" href="#api-for-communication">API For Communication</a><ul>
|
||
<li><a class="reference internal" href="#shareable-types">Shareable Types</a><ul>
|
||
<li><a class="reference internal" href="#communicating-through-os-pipes">Communicating Through OS Pipes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#channels">Channels</a></li>
|
||
<li><a class="reference internal" href="#caveats-for-shared-objects">Caveats For Shared Objects</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#documentation">Documentation</a></li>
|
||
<li><a class="reference internal" href="#alternative-solutions">Alternative Solutions</a></li>
|
||
<li><a class="reference internal" href="#open-questions">Open Questions</a></li>
|
||
<li><a class="reference internal" href="#deferred-functionality">Deferred Functionality</a><ul>
|
||
<li><a class="reference internal" href="#add-convenience-api">Add convenience API</a></li>
|
||
<li><a class="reference internal" href="#avoid-possible-confusion-about-interpreters-running-in-the-current-thread">Avoid possible confusion about interpreters running in the current thread</a></li>
|
||
<li><a class="reference internal" href="#clarify-running-vs-has-threads">Clarify “running” vs. “has threads”</a></li>
|
||
<li><a class="reference internal" href="#a-dunder-method-for-sharing">A Dunder Method For Sharing</a></li>
|
||
<li><a class="reference internal" href="#interpreter-call">Interpreter.call()</a></li>
|
||
<li><a class="reference internal" href="#interpreter-run-in-thread">Interpreter.run_in_thread()</a></li>
|
||
<li><a class="reference internal" href="#synchronization-primitives">Synchronization Primitives</a></li>
|
||
<li><a class="reference internal" href="#csp-library">CSP Library</a></li>
|
||
<li><a class="reference internal" href="#syntactic-support">Syntactic Support</a></li>
|
||
<li><a class="reference internal" href="#multiprocessing">Multiprocessing</a></li>
|
||
<li><a class="reference internal" href="#c-extension-opt-in-opt-out">C-extension opt-in/opt-out</a></li>
|
||
<li><a class="reference internal" href="#poisoning-channels">Poisoning channels</a></li>
|
||
<li><a class="reference internal" href="#resetting-main">Resetting __main__</a></li>
|
||
<li><a class="reference internal" href="#resetting-an-interpreter-s-state">Resetting an interpreter’s state</a></li>
|
||
<li><a class="reference internal" href="#copy-an-existing-interpreter-s-state">Copy an existing interpreter’s state</a></li>
|
||
<li><a class="reference internal" href="#shareable-file-descriptors-and-sockets">Shareable file descriptors and sockets</a></li>
|
||
<li><a class="reference internal" href="#integration-with-async">Integration with async</a></li>
|
||
<li><a class="reference internal" href="#support-for-iteration">Support for iteration</a></li>
|
||
<li><a class="reference internal" href="#channel-context-managers">Channel context managers</a></li>
|
||
<li><a class="reference internal" href="#pipes-and-queues">Pipes and Queues</a></li>
|
||
<li><a class="reference internal" href="#return-a-lock-from-send">Return a lock from send()</a></li>
|
||
<li><a class="reference internal" href="#support-prioritization-in-channels">Support prioritization in channels</a></li>
|
||
<li><a class="reference internal" href="#support-inheriting-settings-and-more">Support inheriting settings (and more?)</a></li>
|
||
<li><a class="reference internal" href="#make-exceptions-shareable">Make exceptions shareable</a></li>
|
||
<li><a class="reference internal" href="#make-everything-shareable-through-serialization">Make everything shareable through serialization</a></li>
|
||
<li><a class="reference internal" href="#make-runfailederror-cause-lazy">Make RunFailedError.__cause__ lazy</a></li>
|
||
<li><a class="reference internal" href="#return-a-value-from-interp-exec">Return a value from <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code></a></li>
|
||
<li><a class="reference internal" href="#add-a-shareable-synchronization-primitive">Add a shareable synchronization primitive</a></li>
|
||
<li><a class="reference internal" href="#propagate-systemexit-and-keyboardinterrupt-differently">Propagate SystemExit and KeyboardInterrupt Differently</a></li>
|
||
<li><a class="reference internal" href="#add-an-explicit-release-and-close-to-channel-end-classes">Add an explicit release() and close() to channel end classes</a></li>
|
||
<li><a class="reference internal" href="#add-sendchannel-send-buffer">Add SendChannel.send_buffer()</a></li>
|
||
<li><a class="reference internal" href="#auto-run-in-a-thread">Auto-run in a thread</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#explicit-channel-association">Explicit channel association</a></li>
|
||
<li><a class="reference internal" href="#add-an-api-based-on-pipes">Add an API based on pipes</a></li>
|
||
<li><a class="reference internal" href="#add-an-api-based-on-queues">Add an API based on queues</a></li>
|
||
<li><a class="reference internal" href="#enumerate">“enumerate”</a></li>
|
||
<li><a class="reference internal" href="#alternate-solutions-to-prevent-leaking-exceptions-across-interpreters">Alternate solutions to prevent leaking exceptions across interpreters</a></li>
|
||
<li><a class="reference internal" href="#always-associate-each-new-interpreter-with-its-own-thread">Always associate each new interpreter with its own thread</a></li>
|
||
<li><a class="reference internal" href="#only-associate-interpreters-upon-use">Only associate interpreters upon use</a></li>
|
||
<li><a class="reference internal" href="#allow-multiple-simultaneous-calls-to-interpreter-exec">Allow multiple simultaneous calls to Interpreter.exec()</a></li>
|
||
<li><a class="reference internal" href="#add-a-reraise-method-to-runfailederror">Add a “reraise” method to RunFailedError</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>This PEP effectively continues in a cleaner form in <a class="pep reference internal" href="../pep-0734/" title="PEP 734 – Multiple Interpreters in the Stdlib">PEP 734</a>.
|
||
This PEP is kept as-is for the sake of the various sections of
|
||
background information and deferred/rejected ideas that have
|
||
been stripped from <a class="pep reference internal" href="../pep-0734/" title="PEP 734 – Multiple Interpreters in the Stdlib">PEP 734</a>.</p>
|
||
</div>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>CPython has supported multiple interpreters in the same process (AKA
|
||
“subinterpreters”) since version 1.5 (1997). The feature has been
|
||
available via the C-API. <a class="reference internal" href="#c-api" id="id1"><span>[c-api]</span></a> Multiple interpreters operate in
|
||
<a class="reference internal" href="#interpreter-isolation">relative isolation from one another</a>, which
|
||
facilitates novel alternative approaches to
|
||
<a class="reference internal" href="#concurrency">concurrency</a>.</p>
|
||
<p>This proposal introduces the stdlib <code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module. It exposes
|
||
the basic functionality of multiple interpreters already provided by the
|
||
C-API, along with basic support for communicating between interpreters.
|
||
This module is especially relevant since <a class="pep reference internal" href="../pep-0684/" title="PEP 684 – A Per-Interpreter GIL">PEP 684</a> introduced a
|
||
per-interpreter GIL in Python 3.12.</p>
|
||
</section>
|
||
<section id="proposal">
|
||
<h2><a class="toc-backref" href="#proposal" role="doc-backlink">Proposal</a></h2>
|
||
<p>Summary:</p>
|
||
<ul class="simple">
|
||
<li>add a new stdlib module: “interpreters”</li>
|
||
<li>add concurrent.futures.InterpreterPoolExecutor</li>
|
||
<li>help for extension module maintainers</li>
|
||
</ul>
|
||
<section id="the-interpreters-module">
|
||
<h3><a class="toc-backref" href="#the-interpreters-module" role="doc-backlink">The “interpreters” Module</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module will provide a high-level interface
|
||
to the multiple interpreter functionality, and wrap a new low-level
|
||
<code class="docutils literal notranslate"><span class="pre">_interpreters</span></code> (in the same way as the <code class="docutils literal notranslate"><span class="pre">threading</span></code> module).
|
||
See the <a class="reference internal" href="#examples">Examples</a> section for concrete usage and use cases.</p>
|
||
<p>Along with exposing the existing (in CPython) multiple interpreter
|
||
support, the module will also support a basic mechanism for
|
||
passing data between interpreters. That involves setting “shareable”
|
||
objects in the <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module of a target subinterpreter. Some
|
||
such objects, like <code class="docutils literal notranslate"><span class="pre">os.pipe()</span></code>, may be used to communicate further.
|
||
The module will also provide a minimal implementation of “channels”
|
||
as a demonstration of cross-interpreter communication.</p>
|
||
<p>Note that <em>objects</em> are not shared between interpreters since they are
|
||
tied to the interpreter in which they were created. Instead, the
|
||
objects’ <em>data</em> is passed between interpreters. See the <a class="reference internal" href="#shared-data">Shared Data</a>
|
||
and <a class="reference internal" href="#api-for-communication">API For Communication</a> sections for more details about
|
||
sharing/communicating between interpreters.</p>
|
||
</section>
|
||
<section id="api-summary-for-interpreters-module">
|
||
<h3><a class="toc-backref" href="#api-summary-for-interpreters-module" role="doc-backlink">API summary for interpreters module</a></h3>
|
||
<p>Here is a summary of the API for the <code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module. For a
|
||
more in-depth explanation of the proposed classes and functions, see
|
||
the <a class="reference internal" href="#interpreters-module-api">“interpreters” Module API</a> section below.</p>
|
||
<p>For creating and using interpreters:</p>
|
||
<table class="docutils align-default">
|
||
<thead>
|
||
<tr class="row-odd"><th class="head">signature</th>
|
||
<th class="head">description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">list_all()</span> <span class="pre">-></span> <span class="pre">[Interpreter]</span></code></td>
|
||
<td>Get all existing interpreters.</td>
|
||
</tr>
|
||
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">get_current()</span> <span class="pre">-></span> <span class="pre">Interpreter</span></code></td>
|
||
<td>Get the currently running interpreter.</td>
|
||
</tr>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">get_main()</span> <span class="pre">-></span> <span class="pre">Interpreter</span></code></td>
|
||
<td>Get the main interpreter.</td>
|
||
</tr>
|
||
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">create()</span> <span class="pre">-></span> <span class="pre">Interpreter</span></code></td>
|
||
<td>Initialize a new (idle) Python interpreter.</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<div class="line-block">
|
||
<div class="line"><br /></div>
|
||
</div>
|
||
<table class="docutils align-default">
|
||
<thead>
|
||
<tr class="row-odd"><th class="head">signature</th>
|
||
<th class="head">description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">class</span> <span class="pre">Interpreter</span></code></td>
|
||
<td>A single interpreter.</td>
|
||
</tr>
|
||
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">.id</span></code></td>
|
||
<td>The interpreter’s ID (read-only).</td>
|
||
</tr>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">.is_running()</span> <span class="pre">-></span> <span class="pre">bool</span></code></td>
|
||
<td>Is the interpreter currently executing code?</td>
|
||
</tr>
|
||
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">.close()</span></code></td>
|
||
<td>Finalize and destroy the interpreter.</td>
|
||
</tr>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">.set_main_attrs(**kwargs)</span></code></td>
|
||
<td>Bind “shareable” objects in <code class="docutils literal notranslate"><span class="pre">__main__</span></code>.</td>
|
||
</tr>
|
||
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">.get_main_attr(name)</span></code></td>
|
||
<td>Get a “shareable” object from <code class="docutils literal notranslate"><span class="pre">__main__</span></code>.</td>
|
||
</tr>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">.exec(src_str,</span> <span class="pre">/)</span></code></td>
|
||
<td><div class="line-block">
|
||
<div class="line">Run the given source code in the interpreter</div>
|
||
<div class="line">(in the current thread).</div>
|
||
</div>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>For communicating between interpreters:</p>
|
||
<table class="docutils align-default">
|
||
<thead>
|
||
<tr class="row-odd"><th class="head">signature</th>
|
||
<th class="head">description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">is_shareable(obj)</span> <span class="pre">-></span> <span class="pre">Bool</span></code></td>
|
||
<td><div class="line-block">
|
||
<div class="line">Can the object’s data be passed</div>
|
||
<div class="line">between interpreters?</div>
|
||
</div>
|
||
</td>
|
||
</tr>
|
||
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">create_channel()</span> <span class="pre">-></span> <span class="pre">(RecvChannel,</span> <span class="pre">SendChannel)</span></code></td>
|
||
<td><div class="line-block">
|
||
<div class="line">Create a new channel for passing</div>
|
||
<div class="line">data between interpreters.</div>
|
||
</div>
|
||
</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
<section id="concurrent-futures-interpreterpoolexecutor">
|
||
<h3><a class="toc-backref" href="#concurrent-futures-interpreterpoolexecutor" role="doc-backlink">concurrent.futures.InterpreterPoolExecutor</a></h3>
|
||
<p>An executor will be added that extends <code class="docutils literal notranslate"><span class="pre">ThreadPoolExecutor</span></code> to run
|
||
per-thread tasks in subinterpreters. Initially, the only supported
|
||
tasks will be whatever <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> takes (e.g. a <code class="docutils literal notranslate"><span class="pre">str</span></code>
|
||
script). However, we may also support some functions, as well as
|
||
eventually a separate method for pickling the task and arguments,
|
||
to reduce friction (at the expense of performance
|
||
for short-running tasks).</p>
|
||
</section>
|
||
<section id="help-for-extension-module-maintainers">
|
||
<h3><a class="toc-backref" href="#help-for-extension-module-maintainers" role="doc-backlink">Help for Extension Module Maintainers</a></h3>
|
||
<p>In practice, an extension that implements multi-phase init (<a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a>)
|
||
is considered isolated and thus compatible with multiple interpreters.
|
||
Otherwise it is “incompatible”.</p>
|
||
<p>Many extension modules are still incompatible. The maintainers and
|
||
users of such extension modules will both benefit when they are updated
|
||
to support multiple interpreters. In the meantime, users may become
|
||
confused by failures when using multiple interpreters, which could
|
||
negatively impact extension maintainers. See <a class="reference internal" href="#concerns">Concerns</a> below.</p>
|
||
<p>To mitigate that impact and accelerate compatibility, we will do the
|
||
following:</p>
|
||
<ul class="simple">
|
||
<li>be clear that extension modules are <em>not</em> required to support use in
|
||
multiple interpreters</li>
|
||
<li>raise <code class="docutils literal notranslate"><span class="pre">ImportError</span></code> when an incompatible module is imported
|
||
in a subinterpreter</li>
|
||
<li>provide resources (e.g. docs) to help maintainers reach compatibility</li>
|
||
<li>reach out to the maintainers of Cython and of the most used extension
|
||
modules (on PyPI) to get feedback and possibly provide assistance</li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="examples">
|
||
<h2><a class="toc-backref" href="#examples" role="doc-backlink">Examples</a></h2>
|
||
<section id="run-isolated-code-in-current-os-thread">
|
||
<h3><a class="toc-backref" href="#run-isolated-code-in-current-os-thread" role="doc-backlink">Run isolated code in current OS thread</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'before'</span><span class="p">)</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="s1">'print("during")'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'after'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="run-in-a-different-thread">
|
||
<h3><a class="toc-backref" href="#run-in-a-different-thread" role="doc-backlink">Run in a different thread</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="k">def</span> <span class="nf">run</span><span class="p">():</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="s1">'print("during")'</span><span class="p">)</span>
|
||
<span class="n">t</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">run</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'before'</span><span class="p">)</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'after'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="pre-populate-an-interpreter">
|
||
<h3><a class="toc-backref" href="#pre-populate-an-interpreter" role="doc-backlink">Pre-populate an interpreter</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> import some_lib</span>
|
||
<span class="s2"> import an_expensive_module</span>
|
||
<span class="s2"> some_lib.set_up()</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="n">wait_for_request</span><span class="p">()</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> some_lib.handle_request()</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="handling-an-exception">
|
||
<h3><a class="toc-backref" href="#handling-an-exception" role="doc-backlink">Handling an exception</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> raise KeyError</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="k">except</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">RunFailedError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"got the error from the subinterpreter: </span><span class="si">{</span><span class="n">exc</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="re-raising-an-exception">
|
||
<h3><a class="toc-backref" href="#re-raising-an-exception" role="doc-backlink">Re-raising an exception</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> raise KeyError</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="k">except</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">RunFailedError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="n">exc</span><span class="o">.</span><span class="n">__cause__</span>
|
||
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"got a KeyError from the subinterpreter"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Note that this pattern is a candidate for later improvement.</p>
|
||
</section>
|
||
<section id="interact-with-the-main-namespace">
|
||
<h3><a class="toc-backref" href="#interact-with-the-main-namespace" role="doc-backlink">Interact with the __main__ namespace</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">set_main_attrs</span><span class="p">(</span><span class="n">a</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">b</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> res = do_something(a, b)</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="n">res</span> <span class="o">=</span> <span class="n">interp</span><span class="o">.</span><span class="n">get_main_attr</span><span class="p">(</span><span class="s1">'res'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="synchronize-using-an-os-pipe">
|
||
<h3><a class="toc-backref" href="#synchronize-using-an-os-pipe" role="doc-backlink">Synchronize using an OS pipe</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">r1</span><span class="p">,</span> <span class="n">s1</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">pipe</span><span class="p">()</span>
|
||
<span class="n">r2</span><span class="p">,</span> <span class="n">s2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">pipe</span><span class="p">()</span>
|
||
|
||
<span class="k">def</span> <span class="nf">task</span><span class="p">():</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="sa">f</span><span class="s2">"""</span>
|
||
<span class="s2"> import os</span>
|
||
<span class="s2"> os.read(</span><span class="si">{</span><span class="n">r1</span><span class="si">}</span><span class="s2">, 1)</span>
|
||
<span class="s2"> print('during B')</span>
|
||
<span class="s2"> os.write(</span><span class="si">{</span><span class="n">s2</span><span class="si">}</span><span class="s2">, '')</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
|
||
<span class="n">t</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">task</span><span class="p">)</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'before'</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s1</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'during A'</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">r2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'after'</span><span class="p">)</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="sharing-a-file-descriptor">
|
||
<h3><a class="toc-backref" href="#sharing-a-file-descriptor" role="doc-backlink">Sharing a file descriptor</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s1">'spamspamspam'</span><span class="p">)</span> <span class="k">as</span> <span class="n">infile</span><span class="p">:</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">set_main_attrs</span><span class="p">(</span><span class="n">fd</span><span class="o">=</span><span class="n">infile</span><span class="o">.</span><span class="n">fileno</span><span class="p">())</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="sa">f</span><span class="s2">"""</span>
|
||
<span class="s2"> import os</span>
|
||
<span class="s2"> for line in os.fdopen(fd):</span>
|
||
<span class="s2"> print(line)</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="passing-objects-via-pickle">
|
||
<h3><a class="toc-backref" href="#passing-objects-via-pickle" role="doc-backlink">Passing objects via pickle</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">r</span><span class="p">,</span> <span class="n">s</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">pipe</span><span class="p">()</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="sa">f</span><span class="s2">"""</span>
|
||
<span class="s2"> import os</span>
|
||
<span class="s2"> import pickle</span>
|
||
<span class="s2"> reader = </span><span class="si">{</span><span class="n">r</span><span class="si">}</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> data = b''</span>
|
||
<span class="s2"> c = os.read(reader, 1)</span>
|
||
<span class="s2"> while c != b'</span><span class="se">\x00</span><span class="s2">':</span>
|
||
<span class="s2"> while c != b'</span><span class="se">\x00</span><span class="s2">':</span>
|
||
<span class="s2"> data += c</span>
|
||
<span class="s2"> c = os.read(reader, 1)</span>
|
||
<span class="s2"> obj = pickle.loads(data)</span>
|
||
<span class="s2"> do_something(obj)</span>
|
||
<span class="s2"> c = os.read(reader, 1)</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="k">for</span> <span class="n">obj</span> <span class="ow">in</span> <span class="nb">input</span><span class="p">:</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="sa">b</span><span class="s1">'</span><span class="se">\x00</span><span class="s1">'</span><span class="p">)</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="sa">b</span><span class="s1">'</span><span class="se">\x00</span><span class="s1">'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="capturing-an-interpreter-s-stdout">
|
||
<h3><a class="toc-backref" href="#capturing-an-interpreter-s-stdout" role="doc-backlink">Capturing an interpreter’s stdout</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">stdout</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">StringIO</span><span class="p">()</span>
|
||
<span class="k">with</span> <span class="n">contextlib</span><span class="o">.</span><span class="n">redirect_stdout</span><span class="p">(</span><span class="n">stdout</span><span class="p">):</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> print('spam!')</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="k">assert</span><span class="p">(</span><span class="n">stdout</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'spam!'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># alternately:</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> import contextlib, io</span>
|
||
<span class="s2"> stdout = io.StringIO()</span>
|
||
<span class="s2"> with contextlib.redirect_stdout(stdout):</span>
|
||
<span class="s2"> print('spam!')</span>
|
||
<span class="s2"> captured = stdout.getvalue()</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="n">captured</span> <span class="o">=</span> <span class="n">interp</span><span class="o">.</span><span class="n">get_main_attr</span><span class="p">(</span><span class="s1">'captured'</span><span class="p">)</span>
|
||
<span class="k">assert</span><span class="p">(</span><span class="n">captured</span> <span class="o">==</span> <span class="s1">'spam!'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>A pipe (<code class="docutils literal notranslate"><span class="pre">os.pipe()</span></code>) could be used similarly.</p>
|
||
</section>
|
||
<section id="running-a-module">
|
||
<h3><a class="toc-backref" href="#running-a-module" role="doc-backlink">Running a module</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">main_module</span> <span class="o">=</span> <span class="n">mod_name</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="sa">f</span><span class="s1">'import runpy; runpy.run_module(</span><span class="si">{</span><span class="n">main_module</span><span class="si">!r}</span><span class="s1">)'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="running-as-script-including-zip-archives-directories">
|
||
<h3><a class="toc-backref" href="#running-as-script-including-zip-archives-directories" role="doc-backlink">Running as script (including zip archives & directories)</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">main_script</span> <span class="o">=</span> <span class="n">path_name</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="sa">f</span><span class="s2">"import runpy; runpy.run_path(</span><span class="si">{</span><span class="n">main_script</span><span class="si">!r}</span><span class="s2">)"</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="using-a-channel-to-communicate">
|
||
<h3><a class="toc-backref" href="#using-a-channel-to-communicate" role="doc-backlink">Using a channel to communicate</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">tasks_recv</span><span class="p">,</span> <span class="n">tasks</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create_channel</span><span class="p">()</span>
|
||
<span class="n">results</span><span class="p">,</span> <span class="n">results_send</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create_channel</span><span class="p">()</span>
|
||
|
||
<span class="k">def</span> <span class="nf">worker</span><span class="p">():</span>
|
||
<span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">set_main_attrs</span><span class="p">(</span><span class="n">tasks</span><span class="o">=</span><span class="n">tasks_recv</span><span class="p">,</span> <span class="n">results</span><span class="o">=</span><span class="n">results_send</span><span class="p">)</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> def handle_request(req):</span>
|
||
<span class="s2"> ...</span>
|
||
|
||
<span class="s2"> def capture_exception(exc):</span>
|
||
<span class="s2"> ...</span>
|
||
|
||
<span class="s2"> while True:</span>
|
||
<span class="s2"> try:</span>
|
||
<span class="s2"> req = tasks.recv()</span>
|
||
<span class="s2"> except Exception:</span>
|
||
<span class="s2"> # channel closed</span>
|
||
<span class="s2"> break</span>
|
||
<span class="s2"> try:</span>
|
||
<span class="s2"> res = handle_request(req)</span>
|
||
<span class="s2"> except Exception as exc:</span>
|
||
<span class="s2"> res = capture_exception(exc)</span>
|
||
<span class="s2"> results.send_nowait(res)</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="n">threads</span> <span class="o">=</span> <span class="p">[</span><span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">worker</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">20</span><span class="p">)]</span>
|
||
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||
|
||
<span class="n">requests</span> <span class="o">=</span> <span class="o">...</span>
|
||
<span class="k">for</span> <span class="n">req</span> <span class="ow">in</span> <span class="n">requests</span><span class="p">:</span>
|
||
<span class="n">tasks</span><span class="o">.</span><span class="n">send</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
|
||
<span class="n">tasks</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||
|
||
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="sharing-a-memoryview-imagine-map-reduce">
|
||
<h3><a class="toc-backref" href="#sharing-a-memoryview-imagine-map-reduce" role="doc-backlink">Sharing a memoryview (imagine map-reduce)</a></h3>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">data</span><span class="p">,</span> <span class="n">chunksize</span> <span class="o">=</span> <span class="n">read_large_data_set</span><span class="p">()</span>
|
||
<span class="n">buf</span> <span class="o">=</span> <span class="nb">memoryview</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
|
||
<span class="n">numchunks</span> <span class="o">=</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="n">chunksize</span>
|
||
<span class="n">results</span> <span class="o">=</span> <span class="nb">memoryview</span><span class="p">(</span><span class="sa">b</span><span class="s1">'</span><span class="se">\0</span><span class="s1">'</span> <span class="o">*</span> <span class="n">numchunks</span><span class="p">)</span>
|
||
|
||
<span class="n">tasks_recv</span><span class="p">,</span> <span class="n">tasks</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create_channel</span><span class="p">()</span>
|
||
|
||
<span class="k">def</span> <span class="nf">worker</span><span class="p">():</span>
|
||
<span class="n">interp</span> <span class="o">=</span> <span class="n">interpreters</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">set_main_attrs</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">buf</span><span class="p">,</span> <span class="n">results</span><span class="o">=</span><span class="n">results</span><span class="p">,</span> <span class="n">tasks</span><span class="o">=</span><span class="n">tasks_recv</span><span class="p">)</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">tw</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span>
|
||
<span class="s2"> while True:</span>
|
||
<span class="s2"> try:</span>
|
||
<span class="s2"> req = tasks.recv()</span>
|
||
<span class="s2"> except Exception:</span>
|
||
<span class="s2"> # channel closed</span>
|
||
<span class="s2"> break</span>
|
||
<span class="s2"> resindex, start, end = req</span>
|
||
<span class="s2"> chunk = data[start: end]</span>
|
||
<span class="s2"> res = reduce_chunk(chunk)</span>
|
||
<span class="s2"> results[resindex] = res</span>
|
||
<span class="s2"> """</span><span class="p">))</span>
|
||
<span class="n">t</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">worker</span><span class="p">)</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">numchunks</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">workers_running</span><span class="p">():</span>
|
||
<span class="k">raise</span> <span class="o">...</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">i</span> <span class="o">*</span> <span class="n">chunksize</span>
|
||
<span class="n">end</span> <span class="o">=</span> <span class="n">start</span> <span class="o">+</span> <span class="n">chunksize</span>
|
||
<span class="k">if</span> <span class="n">end</span> <span class="o">></span> <span class="nb">len</span><span class="p">(</span><span class="n">buf</span><span class="p">):</span>
|
||
<span class="n">end</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">buf</span><span class="p">)</span>
|
||
<span class="n">tasks</span><span class="o">.</span><span class="n">send</span><span class="p">((</span><span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">,</span> <span class="n">i</span><span class="p">))</span>
|
||
<span class="n">tasks</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||
<span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
|
||
|
||
<span class="n">use_results</span><span class="p">(</span><span class="n">results</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Running code in multiple interpreters provides a useful level of
|
||
isolation within the same process. This can be leveraged in a number
|
||
of ways. Furthermore, subinterpreters provide a well-defined framework
|
||
in which such isolation may extended. (See <a class="pep reference internal" href="../pep-0684/" title="PEP 684 – A Per-Interpreter GIL">PEP 684</a>.)</p>
|
||
<p>Alyssa (Nick) Coghlan explained some of the benefits through a comparison with
|
||
multi-processing <a class="reference internal" href="#benefits" id="id2"><span>[benefits]</span></a>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">I</span><span class="p">]</span> <span class="n">expect</span> <span class="n">that</span> <span class="n">communicating</span> <span class="n">between</span> <span class="n">subinterpreters</span> <span class="ow">is</span> <span class="n">going</span>
|
||
<span class="n">to</span> <span class="n">end</span> <span class="n">up</span> <span class="n">looking</span> <span class="n">an</span> <span class="n">awful</span> <span class="n">lot</span> <span class="n">like</span> <span class="n">communicating</span> <span class="n">between</span>
|
||
<span class="n">subprocesses</span> <span class="n">via</span> <span class="n">shared</span> <span class="n">memory</span><span class="o">.</span>
|
||
|
||
<span class="n">The</span> <span class="n">trade</span><span class="o">-</span><span class="n">off</span> <span class="n">between</span> <span class="n">the</span> <span class="n">two</span> <span class="n">models</span> <span class="n">will</span> <span class="n">then</span> <span class="n">be</span> <span class="n">that</span> <span class="n">one</span> <span class="n">still</span>
|
||
<span class="n">just</span> <span class="n">looks</span> <span class="n">like</span> <span class="n">a</span> <span class="n">single</span> <span class="n">process</span> <span class="kn">from</span> <span class="nn">the</span> <span class="n">point</span> <span class="n">of</span> <span class="n">view</span> <span class="n">of</span> <span class="n">the</span>
|
||
<span class="n">outside</span> <span class="n">world</span><span class="p">,</span> <span class="ow">and</span> <span class="n">hence</span> <span class="n">doesn</span><span class="s1">'t place any extra demands on the</span>
|
||
<span class="n">underlying</span> <span class="n">OS</span> <span class="n">beyond</span> <span class="n">those</span> <span class="n">required</span> <span class="n">to</span> <span class="n">run</span> <span class="n">CPython</span> <span class="k">with</span> <span class="n">a</span> <span class="n">single</span>
|
||
<span class="n">interpreter</span><span class="p">,</span> <span class="k">while</span> <span class="n">the</span> <span class="n">other</span> <span class="n">gives</span> <span class="n">much</span> <span class="n">stricter</span> <span class="n">isolation</span>
|
||
<span class="p">(</span><span class="n">including</span> <span class="n">isolating</span> <span class="n">C</span> <span class="nb">globals</span> <span class="ow">in</span> <span class="n">extension</span> <span class="n">modules</span><span class="p">),</span> <span class="n">but</span> <span class="n">also</span>
|
||
<span class="n">demands</span> <span class="n">much</span> <span class="n">more</span> <span class="kn">from</span> <span class="nn">the</span> <span class="n">OS</span> <span class="n">when</span> <span class="n">it</span> <span class="n">comes</span> <span class="n">to</span> <span class="n">its</span> <span class="n">IPC</span>
|
||
<span class="n">capabilities</span><span class="o">.</span>
|
||
|
||
<span class="n">The</span> <span class="n">security</span> <span class="n">risk</span> <span class="n">profiles</span> <span class="n">of</span> <span class="n">the</span> <span class="n">two</span> <span class="n">approaches</span> <span class="n">will</span> <span class="n">also</span> <span class="n">be</span> <span class="n">quite</span>
|
||
<span class="n">different</span><span class="p">,</span> <span class="n">since</span> <span class="n">using</span> <span class="n">subinterpreters</span> <span class="n">won</span><span class="s1">'t require deliberately</span>
|
||
<span class="n">poking</span> <span class="n">holes</span> <span class="ow">in</span> <span class="n">the</span> <span class="n">process</span> <span class="n">isolation</span> <span class="n">that</span> <span class="n">operating</span> <span class="n">systems</span> <span class="n">give</span>
|
||
<span class="n">you</span> <span class="n">by</span> <span class="n">default</span><span class="o">.</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>CPython has supported multiple interpreters, with increasing levels
|
||
of support, since version 1.5. While the feature has the potential
|
||
to be a powerful tool, it has suffered from neglect
|
||
because the multiple interpreter capabilities are not readily available
|
||
directly from Python. Exposing the existing functionality
|
||
in the stdlib will help reverse the situation.</p>
|
||
<p>This proposal is focused on enabling the fundamental capability of
|
||
multiple interpreters, isolated from each other,
|
||
in the same Python process. This is a
|
||
new area for Python so there is relative uncertainly about the best
|
||
tools to provide as companions to interpreters. Thus we minimize
|
||
the functionality we add in the proposal as much as possible.</p>
|
||
<section id="concerns">
|
||
<h3><a class="toc-backref" href="#concerns" role="doc-backlink">Concerns</a></h3>
|
||
<ul class="simple">
|
||
<li>“subinterpreters are not worth the trouble”</li>
|
||
</ul>
|
||
<p>Some have argued that subinterpreters do not add sufficient benefit
|
||
to justify making them an official part of Python. Adding features
|
||
to the language (or stdlib) has a cost in increasing the size of
|
||
the language. So an addition must pay for itself.</p>
|
||
<p>In this case, multiple interpreter support provide a novel concurrency
|
||
model focused on isolated threads of execution. Furthermore, they
|
||
provide an opportunity for changes in CPython that will allow
|
||
simultaneous use of multiple CPU cores (currently prevented
|
||
by the GIL–see <a class="pep reference internal" href="../pep-0684/" title="PEP 684 – A Per-Interpreter GIL">PEP 684</a>).</p>
|
||
<p>Alternatives to subinterpreters include threading, async, and
|
||
multiprocessing. Threading is limited by the GIL and async isn’t
|
||
the right solution for every problem (nor for every person).
|
||
Multiprocessing is likewise valuable in some but not all situations.
|
||
Direct IPC (rather than via the multiprocessing module) provides
|
||
similar benefits but with the same caveat.</p>
|
||
<p>Notably, subinterpreters are not intended as a replacement for any of
|
||
the above. Certainly they overlap in some areas, but the benefits of
|
||
subinterpreters include isolation and (potentially) performance. In
|
||
particular, subinterpreters provide a direct route to an alternate
|
||
concurrency model (e.g. CSP) which has found success elsewhere and
|
||
will appeal to some Python users. That is the core value that the
|
||
<code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module will provide.</p>
|
||
<ul class="simple">
|
||
<li>“stdlib support for multiple interpreters adds extra burden
|
||
on C extension authors”</li>
|
||
</ul>
|
||
<p>In the <a class="reference internal" href="#interpreter-isolation">Interpreter Isolation</a> section below we identify ways in
|
||
which isolation in CPython’s subinterpreters is incomplete. Most
|
||
notable is extension modules that use C globals to store internal
|
||
state. (<a class="pep reference internal" href="../pep-3121/" title="PEP 3121 – Extension Module Initialization and Finalization">PEP 3121</a> and <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a> provide a solution to that problem,
|
||
followed by some extra APIs that improve efficiency, e.g. <a class="pep reference internal" href="../pep-0573/" title="PEP 573 – Module State Access from C Extension Methods">PEP 573</a>).</p>
|
||
<p>Consequently, projects that publish extension modules may face an
|
||
increased maintenance burden as their users start using subinterpreters,
|
||
where their modules may break. This situation is limited to modules
|
||
that use C globals (or use libraries that use C globals) to store
|
||
internal state. For numpy, the reported-bug rate is one every 6
|
||
months. <a class="reference internal" href="#bug-rate" id="id3"><span>[bug-rate]</span></a></p>
|
||
<p>Ultimately this comes down to a question of how often it will be a
|
||
problem in practice: how many projects would be affected, how often
|
||
their users will be affected, what the additional maintenance burden
|
||
will be for projects, and what the overall benefit of subinterpreters
|
||
is to offset those costs. The position of this PEP is that the actual
|
||
extra maintenance burden will be small and well below the threshold at
|
||
which subinterpreters are worth it.</p>
|
||
<ul class="simple">
|
||
<li>“creating a new concurrency API deserves much more thought and
|
||
experimentation, so the new module shouldn’t go into the stdlib
|
||
right away, if ever”</li>
|
||
</ul>
|
||
<p>Introducing an API for a new concurrency model, like happened with
|
||
asyncio, is an extremely large project that requires a lot of careful
|
||
consideration. It is not something that can be done as simply as this
|
||
PEP proposes and likely deserves significant time on PyPI to mature.
|
||
(See <a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/message/TUEAZNZHVJGGLL4OFD32OW6JJDKM6FAS/">Nathaniel’s post</a> on python-dev.)</p>
|
||
<p>However, this PEP does not propose any new concurrency API.
|
||
At most it exposes minimal tools (e.g. subinterpreters, channels)
|
||
which may be used to write code that follows patterns associated with
|
||
(relatively) new-to-Python <a class="reference internal" href="#concurrency">concurrency models</a>.
|
||
Those tools could also be used as the basis for APIs for such
|
||
concurrency models. Again, this PEP does not propose any such API.</p>
|
||
<ul class="simple">
|
||
<li>“there is no point to exposing subinterpreters if they still share
|
||
the GIL”</li>
|
||
<li>“the effort to make the GIL per-interpreter is disruptive and risky”</li>
|
||
</ul>
|
||
<p>A common misconception is that this PEP also includes a promise that
|
||
interpreters will no longer share the GIL. When that is clarified,
|
||
the next question is “what is the point?”. This is already answered
|
||
at length in this PEP. Just to be clear, the value lies in:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">increase</span> <span class="n">exposure</span> <span class="n">of</span> <span class="n">the</span> <span class="n">existing</span> <span class="n">feature</span><span class="p">,</span> <span class="n">which</span> <span class="n">helps</span> <span class="n">improve</span>
|
||
<span class="n">the</span> <span class="n">code</span> <span class="n">health</span> <span class="n">of</span> <span class="n">the</span> <span class="n">entire</span> <span class="n">CPython</span> <span class="n">runtime</span>
|
||
<span class="o">*</span> <span class="n">expose</span> <span class="n">the</span> <span class="p">(</span><span class="n">mostly</span><span class="p">)</span> <span class="n">isolated</span> <span class="n">execution</span> <span class="n">of</span> <span class="n">interpreters</span>
|
||
<span class="o">*</span> <span class="n">preparation</span> <span class="k">for</span> <span class="n">per</span><span class="o">-</span><span class="n">interpreter</span> <span class="n">GIL</span>
|
||
<span class="o">*</span> <span class="n">encourage</span> <span class="n">experimentation</span>
|
||
</pre></div>
|
||
</div>
|
||
<ul class="simple">
|
||
<li>“data sharing can have a negative impact on cache performance
|
||
in multi-core scenarios”</li>
|
||
</ul>
|
||
<p>(See <a class="reference internal" href="#cache-line-ping-pong" id="id4"><span>[cache-line-ping-pong]</span></a>.)</p>
|
||
<p>This shouldn’t be a problem for now as we have no immediate plans
|
||
to actually share data between interpreters, instead focusing
|
||
on copying.</p>
|
||
</section>
|
||
</section>
|
||
<section id="about-subinterpreters">
|
||
<h2><a class="toc-backref" href="#about-subinterpreters" role="doc-backlink">About Subinterpreters</a></h2>
|
||
<section id="concurrency">
|
||
<h3><a class="toc-backref" href="#concurrency" role="doc-backlink">Concurrency</a></h3>
|
||
<p>Concurrency is a challenging area of software development. Decades of
|
||
research and practice have led to a wide variety of concurrency models,
|
||
each with different goals. Most center on correctness and usability.</p>
|
||
<p>One class of concurrency models focuses on isolated threads of
|
||
execution that interoperate through some message passing scheme. A
|
||
notable example is Communicating Sequential Processes <a class="reference internal" href="#csp" id="id5"><span>[CSP]</span></a> (upon
|
||
which Go’s concurrency is roughly based). The intended isolation
|
||
inherent to CPython’s interpreters makes them well-suited
|
||
to this approach.</p>
|
||
</section>
|
||
<section id="shared-data">
|
||
<h3><a class="toc-backref" href="#shared-data" role="doc-backlink">Shared Data</a></h3>
|
||
<p>CPython’s interpreters are inherently isolated (with caveats
|
||
explained below), in contrast to threads. So the same
|
||
communicate-via-shared-memory approach doesn’t work. Without an
|
||
alternative, effective use of concurrency via multiple interpreters
|
||
is significantly limited.</p>
|
||
<p>The key challenge here is that sharing objects between interpreters
|
||
faces complexity due to various constraints on object ownership,
|
||
visibility, and mutability. At a conceptual level it’s easier to
|
||
reason about concurrency when objects only exist in one interpreter
|
||
at a time. At a technical level, CPython’s current memory model
|
||
limits how Python <em>objects</em> may be shared safely between interpreters;
|
||
effectively, objects are bound to the interpreter in which they were
|
||
created. Furthermore, the complexity of <em>object</em> sharing increases as
|
||
interpreters become more isolated, e.g. after GIL removal (though this
|
||
is mitigated somewhat for some “immortal” objects (see <a class="pep reference internal" href="../pep-0683/" title="PEP 683 – Immortal Objects, Using a Fixed Refcount">PEP 683</a>).</p>
|
||
<p>Consequently, the mechanism for sharing needs to be carefully considered.
|
||
There are a number of valid solutions, several of which may be
|
||
appropriate to support in Python’s stdlib and C-API. Any such solution
|
||
is likely to share many characteristics with the others.</p>
|
||
<p>In the meantime, we propose here a minimal solution
|
||
(<code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code>), which sets some precedent for how
|
||
objects are shared. More importantly, it facilitates the introduction
|
||
of more advanced approaches later and allows them to coexist and cooperate.
|
||
In part to demonstrate that, we will provide a basic implementation of
|
||
“channels”, as a somewhat more advanced sharing solution.</p>
|
||
<p>Separate proposals may cover:</p>
|
||
<ul class="simple">
|
||
<li>the addition of a public C-API based on the implementation
|
||
<code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code></li>
|
||
<li>the addition of other sharing approaches to the “interpreters” module</li>
|
||
</ul>
|
||
<p>The fundamental enabling feature for communication is that most objects
|
||
can be converted to some encoding of underlying raw data, which is safe
|
||
to be passed between interpreters. For example, an <code class="docutils literal notranslate"><span class="pre">int</span></code> object can
|
||
be turned into a C <code class="docutils literal notranslate"><span class="pre">long</span></code> value, sent to another interpreter, and
|
||
turned back into an <code class="docutils literal notranslate"><span class="pre">int</span></code> object there. As another example,
|
||
<code class="docutils literal notranslate"><span class="pre">None</span></code> may be passed as-is.</p>
|
||
<p>Regardless, the effort to determine the best way forward here is mostly
|
||
outside the scope of this PEP. In the meantime, this proposal describes
|
||
a basic interim solution using pipes (<code class="docutils literal notranslate"><span class="pre">os.pipe()</span></code>), as well as
|
||
providing a dedicated capability (“channels”).
|
||
See <a class="reference internal" href="#api-for-communication">API For Communication</a> below.</p>
|
||
</section>
|
||
<section id="interpreter-isolation">
|
||
<h3><a class="toc-backref" href="#interpreter-isolation" role="doc-backlink">Interpreter Isolation</a></h3>
|
||
<p>CPython’s interpreters are intended to be strictly isolated from each
|
||
other. Each interpreter has its own copy of all modules, classes,
|
||
functions, and variables. The same applies to state in C, including in
|
||
extension modules. The CPython C-API docs explain more. <a class="reference internal" href="#caveats" id="id6"><span>[caveats]</span></a></p>
|
||
<p>However, there are ways in which interpreters do share some state.
|
||
First of all, some process-global state remains shared:</p>
|
||
<ul class="simple">
|
||
<li>file descriptors</li>
|
||
<li>low-level env vars</li>
|
||
<li>process memory (though allocators <em>are</em> isolated)</li>
|
||
<li>builtin types (e.g. dict, bytes)</li>
|
||
<li>singletons (e.g. None)</li>
|
||
<li>underlying static module data (e.g. functions) for
|
||
builtin/extension/frozen modules</li>
|
||
</ul>
|
||
<p>There are no plans to change this.</p>
|
||
<p>Second, some isolation is faulty due to bugs or implementations that did
|
||
not take subinterpreters into account. This includes things like
|
||
extension modules that rely on C globals. <a class="reference internal" href="#cryptography" id="id7"><span>[cryptography]</span></a> In these
|
||
cases bugs should be opened (some are already):</p>
|
||
<ul class="simple">
|
||
<li>readline module hook functions (<a class="reference external" href="http://bugs.python.org/issue4202">http://bugs.python.org/issue4202</a>)</li>
|
||
<li>memory leaks on re-init (<a class="reference external" href="http://bugs.python.org/issue21387">http://bugs.python.org/issue21387</a>)</li>
|
||
</ul>
|
||
<p>Finally, some potential isolation is missing due to the current design
|
||
of CPython. Improvements are currently going on to address gaps in this
|
||
area:</p>
|
||
<ul class="simple">
|
||
<li>extensions using the <code class="docutils literal notranslate"><span class="pre">PyGILState_*</span></code> API are somewhat incompatible <a class="reference internal" href="#gilstate" id="id8"><span>[gilstate]</span></a></li>
|
||
</ul>
|
||
</section>
|
||
<section id="existing-usage">
|
||
<h3><a class="toc-backref" href="#existing-usage" role="doc-backlink">Existing Usage</a></h3>
|
||
<p>Multiple interpreter support has not been a widely used feature.
|
||
In fact, there have been only a handful of documented cases of
|
||
widespread usage, including
|
||
<a class="reference external" href="https://github.com/GrahamDumpleton/mod_wsgi">mod_wsgi</a>,
|
||
<a class="reference external" href="https://github.com/ceph/ceph/pull/14971">OpenStack Ceph</a>, and
|
||
<a class="reference external" href="https://github.com/ninia/jep">JEP</a>. On the one hand, these cases
|
||
provide confidence that existing multiple interpreter support is
|
||
relatively stable. On the other hand, there isn’t much of a sample
|
||
size from which to judge the utility of the feature.</p>
|
||
</section>
|
||
</section>
|
||
<section id="alternate-python-implementations">
|
||
<h2><a class="toc-backref" href="#alternate-python-implementations" role="doc-backlink">Alternate Python Implementations</a></h2>
|
||
<p>I’ve solicited feedback from various Python implementors about support
|
||
for subinterpreters. Each has indicated that they would be able to
|
||
support multiple interpreters in the same process (if they choose to)
|
||
without a lot of trouble. Here are the projects I contacted:</p>
|
||
<ul class="simple">
|
||
<li>jython (<a class="reference internal" href="#jython" id="id9"><span>[jython]</span></a>)</li>
|
||
<li>ironpython (personal correspondence)</li>
|
||
<li>pypy (personal correspondence)</li>
|
||
<li>micropython (personal correspondence)</li>
|
||
</ul>
|
||
</section>
|
||
<section id="interpreters-module-api">
|
||
<span id="interpreters-is-shareable"></span><span id="interpreters-interpreter"></span><span id="interpreters-create"></span><span id="interpreters-get-current"></span><span id="interpreters-list-all"></span><h2><a class="toc-backref" href="#interpreters-module-api" role="doc-backlink">“interpreters” Module API</a></h2>
|
||
<p>The module provides the following functions:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">list_all</span><span class="p">()</span> <span class="o">-></span> <span class="p">[</span><span class="n">Interpreter</span><span class="p">]</span>
|
||
|
||
<span class="n">Return</span> <span class="n">a</span> <span class="nb">list</span> <span class="n">of</span> <span class="nb">all</span> <span class="n">existing</span> <span class="n">interpreters</span><span class="o">.</span>
|
||
|
||
<span class="n">get_current</span><span class="p">()</span> <span class="o">=></span> <span class="n">Interpreter</span>
|
||
|
||
<span class="n">Return</span> <span class="n">the</span> <span class="n">currently</span> <span class="n">running</span> <span class="n">interpreter</span><span class="o">.</span>
|
||
|
||
<span class="n">get_main</span><span class="p">()</span> <span class="o">=></span> <span class="n">Interpreter</span>
|
||
|
||
<span class="n">Return</span> <span class="n">the</span> <span class="n">main</span> <span class="n">interpreter</span><span class="o">.</span> <span class="n">If</span> <span class="n">the</span> <span class="n">Python</span> <span class="n">implementation</span>
|
||
<span class="n">has</span> <span class="n">no</span> <span class="n">concept</span> <span class="n">of</span> <span class="n">a</span> <span class="n">main</span> <span class="n">interpreter</span> <span class="n">then</span> <span class="k">return</span> <span class="kc">None</span><span class="o">.</span>
|
||
|
||
<span class="n">create</span><span class="p">()</span> <span class="o">-></span> <span class="n">Interpreter</span>
|
||
|
||
<span class="n">Initialize</span> <span class="n">a</span> <span class="n">new</span> <span class="n">Python</span> <span class="n">interpreter</span> <span class="ow">and</span> <span class="k">return</span> <span class="n">it</span><span class="o">.</span>
|
||
<span class="n">It</span> <span class="n">will</span> <span class="n">remain</span> <span class="n">idle</span> <span class="n">until</span> <span class="n">something</span> <span class="ow">is</span> <span class="n">run</span> <span class="ow">in</span> <span class="n">it</span> <span class="ow">and</span> <span class="n">always</span>
|
||
<span class="n">run</span> <span class="ow">in</span> <span class="n">its</span> <span class="n">own</span> <span class="n">thread</span><span class="o">.</span>
|
||
|
||
<span class="n">is_shareable</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||
|
||
<span class="n">Return</span> <span class="kc">True</span> <span class="k">if</span> <span class="n">the</span> <span class="nb">object</span> <span class="n">may</span> <span class="n">be</span> <span class="s2">"shared"</span> <span class="n">between</span> <span class="n">interpreters</span><span class="o">.</span>
|
||
<span class="n">This</span> <span class="n">does</span> <span class="ow">not</span> <span class="n">necessarily</span> <span class="n">mean</span> <span class="n">that</span> <span class="n">the</span> <span class="n">actual</span> <span class="n">objects</span> <span class="n">will</span> <span class="n">be</span>
|
||
<span class="n">shared</span><span class="o">.</span> <span class="n">Instead</span><span class="p">,</span> <span class="n">it</span> <span class="n">means</span> <span class="n">that</span> <span class="n">the</span> <span class="n">objects</span><span class="s1">' underlying data will</span>
|
||
<span class="n">be</span> <span class="n">shared</span> <span class="ow">in</span> <span class="n">a</span> <span class="n">cross</span><span class="o">-</span><span class="n">interpreter</span> <span class="n">way</span><span class="p">,</span> <span class="n">whether</span> <span class="n">via</span> <span class="n">a</span> <span class="n">proxy</span><span class="p">,</span> <span class="n">a</span>
|
||
<span class="n">copy</span><span class="p">,</span> <span class="ow">or</span> <span class="n">some</span> <span class="n">other</span> <span class="n">means</span><span class="o">.</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The module also provides the following class:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>class Interpreter(id):
|
||
|
||
id -> int:
|
||
|
||
The interpreter's ID. (read-only)
|
||
|
||
is_running() -> bool:
|
||
|
||
Return whether or not the interpreter's "exec()" is currently
|
||
executing code. Code running in subthreads is ignored.
|
||
Calling this on the current interpreter will always return True.
|
||
|
||
close():
|
||
|
||
Finalize and destroy the interpreter.
|
||
|
||
This may not be called on an already running interpreter.
|
||
Doing so results in a RuntimeError.
|
||
|
||
set_main_attrs(iterable_or_mapping, /):
|
||
set_main_attrs(**kwargs):
|
||
|
||
Set attributes in the interpreter's __main__ module
|
||
corresponding to the given name-value pairs. Each value
|
||
must be a "shareable" object and will be converted to a new
|
||
object (e.g. copy, proxy) in whatever way that object's type
|
||
defines. If an attribute with the same name is already set,
|
||
it will be overwritten.
|
||
|
||
This method is helpful for setting up an interpreter before
|
||
calling exec().
|
||
|
||
get_main_attr(name, default=None, /):
|
||
|
||
Return the value of the corresponding attribute of the
|
||
interpreter's __main__ module. If the attribute isn't set
|
||
then the default is returned. If it is set, but the value
|
||
isn't "shareable" then a ValueError is raised.
|
||
|
||
This may be used to introspect the __main__ module, as well
|
||
as a very basic mechanism for "returning" one or more results
|
||
from Interpreter.exec().
|
||
|
||
exec(source_str, /):
|
||
|
||
Run the provided Python source code in the interpreter,
|
||
in its __main__ module.
|
||
|
||
This may not be called on an already running interpreter.
|
||
Doing so results in a RuntimeError.
|
||
|
||
An "interp.exec()" call is similar to a builtin exec() call
|
||
(or to calling a function that returns None). Once
|
||
"interp.exec()" completes, the code that called "exec()"
|
||
continues executing (in the original interpreter). Likewise,
|
||
if there is any uncaught exception then it effectively
|
||
(see below) propagates into the code where ``interp.exec()``
|
||
was called. Like exec() (and threads), but unlike function
|
||
calls, there is no return value. If any "return" value from
|
||
the code is needed, send the data out via a pipe (os.pipe())
|
||
or channel or other cross-interpreter communication mechanism.
|
||
|
||
The big difference from exec() or functions is that
|
||
"interp.exec()" executes the code in an entirely different
|
||
interpreter, with entirely separate state. The interpreters
|
||
are completely isolated from each other, so the state of the
|
||
original interpreter (including the code it was executing in
|
||
the current OS thread) does not affect the state of the target
|
||
interpreter (the one that will execute the code). Likewise,
|
||
the target does not affect the original, nor any of its other
|
||
threads.
|
||
|
||
Instead, the state of the original interpreter (for this thread)
|
||
is frozen, and the code it's executing code completely blocks.
|
||
At that point, the target interpreter is given control of the
|
||
OS thread. Then, when it finishes executing, the original
|
||
interpreter gets control back and continues executing.
|
||
|
||
So calling "interp.exec()" will effectively cause the current
|
||
Python thread to completely pause. Sometimes you won't want
|
||
that pause, in which case you should make the "exec()" call in
|
||
another thread. To do so, add a function that calls
|
||
"interp.exec()" and then run that function in a normal
|
||
"threading.Thread".
|
||
|
||
Note that the interpreter's state is never reset, neither
|
||
before "interp.exec()" executes the code nor after. Thus the
|
||
interpreter state is preserved between calls to
|
||
"interp.exec()". This includes "sys.modules", the "builtins"
|
||
module, and the internal state of C extension modules.
|
||
|
||
Also note that "interp.exec()" executes in the namespace of the
|
||
"__main__" module, just like scripts, the REPL, "-m", and
|
||
"-c". Just as the interpreter's state is not ever reset, the
|
||
"__main__" module is never reset. You can imagine
|
||
concatenating the code from each "interp.exec()" call into one
|
||
long script. This is the same as how the REPL operates.
|
||
|
||
Supported code: source text.
|
||
</pre></div>
|
||
</div>
|
||
<p>In addition to the functionality of <code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code>,
|
||
the module provides a related way to pass data between interpreters:
|
||
channels. See <a class="reference internal" href="#channels">Channels</a> below.</p>
|
||
<section id="uncaught-exceptions">
|
||
<h3><a class="toc-backref" href="#uncaught-exceptions" role="doc-backlink">Uncaught Exceptions</a></h3>
|
||
<p>Regarding uncaught exceptions in <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code>, we noted that
|
||
they are “effectively” propagated into the code where <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code>
|
||
was called. To prevent leaking exceptions (and tracebacks) between
|
||
interpreters, we create a surrogate of the exception and its traceback
|
||
(see <a class="reference external" href="https://docs.python.org/3/library/traceback.html#traceback.TracebackException" title="(in Python v3.12)"><code class="xref py py-class docutils literal notranslate"><span class="pre">traceback.TracebackException</span></code></a>), set it to <code class="docutils literal notranslate"><span class="pre">__cause__</span></code>
|
||
on a new <code class="docutils literal notranslate"><span class="pre">interpreters.RunFailedError</span></code>, and raise that.</p>
|
||
<p>Directly raising (a proxy of) the exception is problematic since it’s
|
||
harder to distinguish between an error in the <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> call
|
||
and an uncaught exception from the subinterpreter.</p>
|
||
</section>
|
||
</section>
|
||
<section id="interpreter-restrictions">
|
||
<h2><a class="toc-backref" href="#interpreter-restrictions" role="doc-backlink">Interpreter Restrictions</a></h2>
|
||
<p>Every new interpreter created by <code class="docutils literal notranslate"><span class="pre">interpreters.create()</span></code>
|
||
now has specific restrictions on any code it runs. This includes the
|
||
following:</p>
|
||
<ul class="simple">
|
||
<li>importing an extension module fails if it does not implement
|
||
multi-phase init</li>
|
||
<li>daemon threads may not be created</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">os.fork()</span></code> is not allowed (so no <code class="docutils literal notranslate"><span class="pre">multiprocessing</span></code>)</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">os.exec*()</span></code> is not allowed
|
||
(but “fork+exec”, a la <code class="docutils literal notranslate"><span class="pre">subprocess</span></code> is okay)</li>
|
||
</ul>
|
||
<p>Note that interpreters created with the existing C-API do not have these
|
||
restrictions. The same is true for the “main” interpreter, so
|
||
existing use of Python will not change.</p>
|
||
<p>We may choose to later loosen some of the above restrictions or provide
|
||
a way to enable/disable granular restrictions individually. Regardless,
|
||
requiring multi-phase init from extension modules will always be a
|
||
default restriction.</p>
|
||
</section>
|
||
<section id="api-for-communication">
|
||
<h2><a class="toc-backref" href="#api-for-communication" role="doc-backlink">API For Communication</a></h2>
|
||
<p>As discussed in <a class="reference internal" href="#shared-data">Shared Data</a> above, multiple interpreter support
|
||
is less useful without a mechanism for sharing data (communicating)
|
||
between them. Sharing actual Python objects between interpreters,
|
||
however, has enough potential problems that we are avoiding support
|
||
for that in this proposal. Nor, as mentioned earlier, are we adding
|
||
anything more than a basic mechanism for communication.</p>
|
||
<p>That mechanism is the <code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code> method.
|
||
It may be used to set up global variables before <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code>
|
||
is called. The name-value pairs passed to <code class="docutils literal notranslate"><span class="pre">set_main_attrs()</span></code> are
|
||
bound as attributes of the interpreter’s <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module.
|
||
The values must be “shareable”. See <a class="reference internal" href="#shareable-types">Shareable Types</a> below.</p>
|
||
<p>Additional approaches to communicating and sharing objects are enabled
|
||
through <code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code>. A shareable object could be
|
||
implemented which works like a queue, but with cross-interpreter safety.
|
||
In fact, this PEP does include an example of such an approach: channels.</p>
|
||
<section id="shareable-types">
|
||
<h3><a class="toc-backref" href="#shareable-types" role="doc-backlink">Shareable Types</a></h3>
|
||
<p>An object is “shareable” if its type supports shareable instances.
|
||
The type must implement a new internal protocol, which is used to
|
||
convert an object to interpreter-independent data and then converted
|
||
back to an object on the other side. Also see
|
||
<a class="reference internal" href="#interpreters-is-shareable">is_shareable()</a> above.</p>
|
||
<p>A minimal set of simple, immutable builtin types will be supported
|
||
initially, including:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">None</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">bool</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">bytes</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">str</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">int</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">float</span></code></li>
|
||
</ul>
|
||
<p>We will also support a small number of complex types initially:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">memoryview</span></code>, to allow sharing <a class="pep reference internal" href="../pep-3118/" title="PEP 3118 – Revising the buffer protocol">PEP 3118</a> buffers</li>
|
||
<li><a class="reference internal" href="#channels">channels</a></li>
|
||
</ul>
|
||
<p>Further builtin types may be supported later, complex or not.
|
||
Limiting the initial shareable types is a practical matter, reducing
|
||
the potential complexity of the initial implementation. There are a
|
||
number of strategies we may pursue in the future to expand supported
|
||
objects, once we have more experience with interpreter isolation.</p>
|
||
<p>In the meantime, a separate proposal will discuss making the internal
|
||
protocol (and C-API) used by <code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code> public.
|
||
With that protocol, support for other types could be added
|
||
by extension modules.</p>
|
||
<section id="communicating-through-os-pipes">
|
||
<h4><a class="toc-backref" href="#communicating-through-os-pipes" role="doc-backlink">Communicating Through OS Pipes</a></h4>
|
||
<p>Even without a dedicated object for communication, users may already
|
||
use existing tools. For example, one basic approach for sending data
|
||
between interpreters is to use a pipe (see <code class="docutils literal notranslate"><span class="pre">os.pipe()</span></code>):</p>
|
||
<ol class="arabic simple">
|
||
<li>interpreter A calls <code class="docutils literal notranslate"><span class="pre">os.pipe()</span></code> to get a read/write pair
|
||
of file descriptors (both <code class="docutils literal notranslate"><span class="pre">int</span></code> objects)</li>
|
||
<li>interpreter A calls <code class="docutils literal notranslate"><span class="pre">interp.set_main_attrs()</span></code>, binding the read FD
|
||
(or embeds it using string formatting)</li>
|
||
<li>interpreter A calls <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> on interpreter B</li>
|
||
<li>interpreter A writes some bytes to the write FD</li>
|
||
<li>interpreter B reads those bytes</li>
|
||
</ol>
|
||
<p>Several of the earlier examples demonstrate this, such as
|
||
<a class="reference internal" href="#synchronize-using-an-os-pipe">Synchronize using an OS pipe</a>.</p>
|
||
</section>
|
||
</section>
|
||
<section id="channels">
|
||
<span id="interpreters-sendchannel"></span><span id="interpreters-recvchannel"></span><span id="interpreters-create-channel"></span><h3><a class="toc-backref" href="#channels" role="doc-backlink">Channels</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module will include a dedicated solution for
|
||
passing object data between interpreters: channels. They are included
|
||
in the module in part to provide an easier mechanism than using
|
||
<code class="docutils literal notranslate"><span class="pre">os.pipe()</span></code> and in part to demonstrate how libraries may take
|
||
advantage of <code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code>
|
||
and the protocol it uses.</p>
|
||
<p>A channel is a simplex FIFO. It is a basic, opt-in data sharing
|
||
mechanism that draws inspiration from pipes, queues, and CSP’s
|
||
channels. <a class="reference internal" href="#fifo" id="id10"><span>[fifo]</span></a> The main difference from pipes is that channels can
|
||
be associated with zero or more interpreters on either end. Like
|
||
queues, which are also many-to-many, channels are buffered (though
|
||
they also offer methods with unbuffered semantics).</p>
|
||
<p>Channels have two operations: send and receive. A key characteristic
|
||
of those operations is that channels transmit data derived from Python
|
||
objects rather than the objects themselves. When objects are sent,
|
||
their data is extracted. When the “object” is received in the other
|
||
interpreter, the data is converted back into an object owned by that
|
||
interpreter.</p>
|
||
<p>To make this work, the mutable shared state will be managed by the
|
||
Python runtime, not by any of the interpreters. Initially we will
|
||
support only one type of objects for shared state: the channels provided
|
||
by <code class="docutils literal notranslate"><span class="pre">interpreters.create_channel()</span></code>. Channels, in turn, will carefully
|
||
manage passing objects between interpreters.</p>
|
||
<p>This approach, including keeping the API minimal, helps us avoid further
|
||
exposing any underlying complexity to Python users.</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module provides the following function related
|
||
to channels:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">create_channel</span><span class="p">()</span> <span class="o">-></span> <span class="p">(</span><span class="n">RecvChannel</span><span class="p">,</span> <span class="n">SendChannel</span><span class="p">):</span>
|
||
|
||
<span class="n">Create</span> <span class="n">a</span> <span class="n">new</span> <span class="n">channel</span> <span class="ow">and</span> <span class="k">return</span> <span class="p">(</span><span class="n">recv</span><span class="p">,</span> <span class="n">send</span><span class="p">),</span> <span class="n">the</span> <span class="n">RecvChannel</span>
|
||
<span class="ow">and</span> <span class="n">SendChannel</span> <span class="n">corresponding</span> <span class="n">to</span> <span class="n">the</span> <span class="n">ends</span> <span class="n">of</span> <span class="n">the</span> <span class="n">channel</span><span class="o">.</span>
|
||
|
||
<span class="n">Both</span> <span class="n">ends</span> <span class="n">of</span> <span class="n">the</span> <span class="n">channel</span> <span class="n">are</span> <span class="n">supported</span> <span class="s2">"shared"</span> <span class="n">objects</span> <span class="p">(</span><span class="n">i</span><span class="o">.</span><span class="n">e</span><span class="o">.</span>
|
||
<span class="n">may</span> <span class="n">be</span> <span class="n">safely</span> <span class="n">shared</span> <span class="n">by</span> <span class="n">different</span> <span class="n">interpreters</span><span class="o">.</span> <span class="n">Thus</span> <span class="n">they</span>
|
||
<span class="n">may</span> <span class="n">be</span> <span class="nb">set</span> <span class="n">using</span> <span class="s2">"Interpreter.set_main_attrs()"</span><span class="o">.</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The module also provides the following channel-related classes:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">RecvChannel</span><span class="p">(</span><span class="nb">id</span><span class="p">):</span>
|
||
|
||
<span class="n">The</span> <span class="n">receiving</span> <span class="n">end</span> <span class="n">of</span> <span class="n">a</span> <span class="n">channel</span><span class="o">.</span> <span class="n">An</span> <span class="n">interpreter</span> <span class="n">may</span> <span class="n">use</span> <span class="n">this</span> <span class="n">to</span>
|
||
<span class="n">receive</span> <span class="n">objects</span> <span class="kn">from</span> <span class="nn">another</span> <span class="n">interpreter</span><span class="o">.</span> <span class="n">Any</span> <span class="nb">type</span> <span class="n">supported</span> <span class="n">by</span>
|
||
<span class="n">Interpreter</span><span class="o">.</span><span class="n">set_main_attrs</span><span class="p">()</span> <span class="n">will</span> <span class="n">be</span> <span class="n">supported</span> <span class="n">here</span><span class="p">,</span> <span class="n">though</span> <span class="n">at</span>
|
||
<span class="n">first</span> <span class="n">only</span> <span class="n">a</span> <span class="n">few</span> <span class="n">of</span> <span class="n">the</span> <span class="n">simple</span><span class="p">,</span> <span class="n">immutable</span> <span class="n">builtin</span> <span class="n">types</span>
|
||
<span class="n">will</span> <span class="n">be</span> <span class="n">supported</span><span class="o">.</span>
|
||
|
||
<span class="nb">id</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||
|
||
<span class="n">The</span> <span class="n">channel</span><span class="s1">'s unique ID. The "send" end has the same one.</span>
|
||
|
||
<span class="n">recv</span><span class="p">(</span><span class="o">*</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
|
||
<span class="n">Return</span> <span class="n">the</span> <span class="nb">next</span> <span class="nb">object</span> <span class="kn">from</span> <span class="nn">the</span> <span class="n">channel</span><span class="o">.</span> <span class="n">If</span> <span class="n">none</span> <span class="n">have</span> <span class="n">been</span>
|
||
<span class="n">sent</span> <span class="n">then</span> <span class="n">wait</span> <span class="n">until</span> <span class="n">the</span> <span class="nb">next</span> <span class="n">send</span> <span class="p">(</span><span class="ow">or</span> <span class="n">until</span> <span class="n">the</span> <span class="n">timeout</span> <span class="ow">is</span> <span class="n">hit</span><span class="p">)</span><span class="o">.</span>
|
||
|
||
<span class="n">At</span> <span class="n">the</span> <span class="n">least</span><span class="p">,</span> <span class="n">the</span> <span class="nb">object</span> <span class="n">will</span> <span class="n">be</span> <span class="n">equivalent</span> <span class="n">to</span> <span class="n">the</span> <span class="n">sent</span> <span class="nb">object</span><span class="o">.</span>
|
||
<span class="n">That</span> <span class="n">will</span> <span class="n">almost</span> <span class="n">always</span> <span class="n">mean</span> <span class="n">the</span> <span class="n">same</span> <span class="nb">type</span> <span class="k">with</span> <span class="n">the</span> <span class="n">same</span> <span class="n">data</span><span class="p">,</span>
|
||
<span class="n">though</span> <span class="n">it</span> <span class="n">could</span> <span class="n">also</span> <span class="n">be</span> <span class="n">a</span> <span class="n">compatible</span> <span class="n">proxy</span><span class="o">.</span> <span class="n">Regardless</span><span class="p">,</span> <span class="n">it</span> <span class="n">may</span>
|
||
<span class="n">use</span> <span class="n">a</span> <span class="n">copy</span> <span class="n">of</span> <span class="n">that</span> <span class="n">data</span> <span class="ow">or</span> <span class="n">actually</span> <span class="n">share</span> <span class="n">the</span> <span class="n">data</span><span class="o">.</span> <span class="n">That</span><span class="s1">'s up</span>
|
||
<span class="n">to</span> <span class="n">the</span> <span class="nb">object</span><span class="s1">'s type.</span>
|
||
|
||
<span class="n">recv_nowait</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
|
||
<span class="n">Return</span> <span class="n">the</span> <span class="nb">next</span> <span class="nb">object</span> <span class="kn">from</span> <span class="nn">the</span> <span class="n">channel</span><span class="o">.</span> <span class="n">If</span> <span class="n">none</span> <span class="n">have</span> <span class="n">been</span>
|
||
<span class="n">sent</span> <span class="n">then</span> <span class="k">return</span> <span class="n">the</span> <span class="n">default</span><span class="o">.</span> <span class="n">Otherwise</span><span class="p">,</span> <span class="n">this</span> <span class="ow">is</span> <span class="n">the</span> <span class="n">same</span>
|
||
<span class="k">as</span> <span class="n">the</span> <span class="s2">"recv()"</span> <span class="n">method</span><span class="o">.</span>
|
||
|
||
|
||
<span class="k">class</span> <span class="nc">SendChannel</span><span class="p">(</span><span class="nb">id</span><span class="p">):</span>
|
||
|
||
<span class="n">The</span> <span class="n">sending</span> <span class="n">end</span> <span class="n">of</span> <span class="n">a</span> <span class="n">channel</span><span class="o">.</span> <span class="n">An</span> <span class="n">interpreter</span> <span class="n">may</span> <span class="n">use</span> <span class="n">this</span> <span class="n">to</span>
|
||
<span class="n">send</span> <span class="n">objects</span> <span class="n">to</span> <span class="n">another</span> <span class="n">interpreter</span><span class="o">.</span> <span class="n">Any</span> <span class="nb">type</span> <span class="n">supported</span> <span class="n">by</span>
|
||
<span class="n">Interpreter</span><span class="o">.</span><span class="n">set_main_attrs</span><span class="p">()</span> <span class="n">will</span> <span class="n">be</span> <span class="n">supported</span> <span class="n">here</span><span class="p">,</span> <span class="n">though</span>
|
||
<span class="n">at</span> <span class="n">first</span> <span class="n">only</span> <span class="n">a</span> <span class="n">few</span> <span class="n">of</span> <span class="n">the</span> <span class="n">simple</span><span class="p">,</span> <span class="n">immutable</span> <span class="n">builtin</span> <span class="n">types</span>
|
||
<span class="n">will</span> <span class="n">be</span> <span class="n">supported</span><span class="o">.</span>
|
||
|
||
<span class="nb">id</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||
|
||
<span class="n">The</span> <span class="n">channel</span><span class="s1">'s unique ID. The "recv" end has the same one.</span>
|
||
|
||
<span class="n">send</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||
|
||
<span class="n">Send</span> <span class="n">the</span> <span class="nb">object</span> <span class="p">(</span><span class="n">i</span><span class="o">.</span><span class="n">e</span><span class="o">.</span> <span class="n">its</span> <span class="n">data</span><span class="p">)</span> <span class="n">to</span> <span class="n">the</span> <span class="s2">"recv"</span> <span class="n">end</span> <span class="n">of</span> <span class="n">the</span>
|
||
<span class="n">channel</span><span class="o">.</span> <span class="n">Wait</span> <span class="n">until</span> <span class="n">the</span> <span class="nb">object</span> <span class="ow">is</span> <span class="n">received</span><span class="o">.</span> <span class="n">If</span> <span class="n">the</span> <span class="nb">object</span>
|
||
<span class="ow">is</span> <span class="ow">not</span> <span class="n">shareable</span> <span class="n">then</span> <span class="ne">ValueError</span> <span class="ow">is</span> <span class="n">raised</span><span class="o">.</span>
|
||
|
||
<span class="n">The</span> <span class="n">builtin</span> <span class="nb">memoryview</span> <span class="ow">is</span> <span class="n">supported</span><span class="p">,</span> <span class="n">so</span> <span class="n">sending</span> <span class="n">a</span> <span class="n">buffer</span>
|
||
<span class="n">across</span> <span class="n">involves</span> <span class="n">first</span> <span class="n">wrapping</span> <span class="n">the</span> <span class="nb">object</span> <span class="ow">in</span> <span class="n">a</span> <span class="nb">memoryview</span>
|
||
<span class="ow">and</span> <span class="n">then</span> <span class="n">sending</span> <span class="n">that</span><span class="o">.</span>
|
||
|
||
<span class="n">send_nowait</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||
|
||
<span class="n">Send</span> <span class="n">the</span> <span class="nb">object</span> <span class="n">to</span> <span class="n">the</span> <span class="s2">"recv"</span> <span class="n">end</span> <span class="n">of</span> <span class="n">the</span> <span class="n">channel</span><span class="o">.</span> <span class="n">This</span>
|
||
<span class="n">behaves</span> <span class="n">the</span> <span class="n">same</span> <span class="k">as</span> <span class="s2">"send()"</span><span class="p">,</span> <span class="k">except</span> <span class="k">for</span> <span class="n">the</span> <span class="n">waiting</span> <span class="n">part</span><span class="o">.</span>
|
||
<span class="n">If</span> <span class="n">no</span> <span class="n">interpreter</span> <span class="ow">is</span> <span class="n">currently</span> <span class="n">receiving</span> <span class="p">(</span><span class="n">waiting</span> <span class="n">on</span> <span class="n">the</span>
|
||
<span class="n">other</span> <span class="n">end</span><span class="p">)</span> <span class="n">then</span> <span class="n">queue</span> <span class="n">the</span> <span class="nb">object</span> <span class="ow">and</span> <span class="k">return</span> <span class="kc">False</span><span class="o">.</span> <span class="n">Otherwise</span>
|
||
<span class="k">return</span> <span class="kc">True</span><span class="o">.</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="caveats-for-shared-objects">
|
||
<h3><a class="toc-backref" href="#caveats-for-shared-objects" role="doc-backlink">Caveats For Shared Objects</a></h3>
|
||
<p>Again, Python objects are not shared between interpreters.
|
||
However, in some cases data those objects wrap is actually shared
|
||
and not just copied. One example might be <a class="pep reference internal" href="../pep-3118/" title="PEP 3118 – Revising the buffer protocol">PEP 3118</a> buffers.</p>
|
||
<p>In those cases the object in the original interpreter is kept alive
|
||
until the shared data in the other interpreter is no longer used.
|
||
Then object destruction can happen like normal in the original
|
||
interpreter, along with the previously shared data.</p>
|
||
</section>
|
||
</section>
|
||
<section id="documentation">
|
||
<h2><a class="toc-backref" href="#documentation" role="doc-backlink">Documentation</a></h2>
|
||
<p>The new stdlib docs page for the <code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module will include
|
||
the following:</p>
|
||
<ul class="simple">
|
||
<li>(at the top) a clear note that support for multiple interpreters
|
||
is not required from extension modules</li>
|
||
<li>some explanation about what subinterpreters are</li>
|
||
<li>brief examples of how to use multiple interpreters
|
||
(and communicating between them)</li>
|
||
<li>a summary of the limitations of using multiple interpreters</li>
|
||
<li>(for extension maintainers) a link to the resources for ensuring
|
||
multiple interpreters compatibility</li>
|
||
<li>much of the API information in this PEP</li>
|
||
</ul>
|
||
<p>Docs about resources for extension maintainers already exist on the
|
||
<a class="reference external" href="https://docs.python.org/3/howto/isolating-extensions.html">Isolating Extension Modules</a> howto page. Any
|
||
extra help will be added there. For example, it may prove helpful
|
||
to discuss strategies for dealing with linked libraries that keep
|
||
their own subinterpreter-incompatible global state.</p>
|
||
<p>Note that the documentation will play a large part in mitigating any
|
||
negative impact that the new <code class="docutils literal notranslate"><span class="pre">interpreters</span></code> module might have on
|
||
extension module maintainers.</p>
|
||
<p>Also, the <code class="docutils literal notranslate"><span class="pre">ImportError</span></code> for incompatible extension modules will be
|
||
updated to clearly say it is due to missing multiple interpreters
|
||
compatibility and that extensions are not required to provide it. This
|
||
will help set user expectations properly.</p>
|
||
</section>
|
||
<section id="alternative-solutions">
|
||
<h2><a class="toc-backref" href="#alternative-solutions" role="doc-backlink">Alternative Solutions</a></h2>
|
||
<p>One possible alternative to a new module is to add support for interpreters
|
||
to <code class="docutils literal notranslate"><span class="pre">concurrent.futures</span></code>. There are several reasons why that wouldn’t work:</p>
|
||
<ul class="simple">
|
||
<li>the obvious place to look for multiple interpreters support
|
||
is an “interpreters” module, much as with “threading”, etc.</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">concurrent.futures</span></code> is all about executing functions
|
||
but currently we don’t have a good way to run a function
|
||
from one interpreter in another</li>
|
||
</ul>
|
||
<p>Similar reasoning applies for support in the <code class="docutils literal notranslate"><span class="pre">multiprocessing</span></code> module.</p>
|
||
</section>
|
||
<section id="open-questions">
|
||
<h2><a class="toc-backref" href="#open-questions" role="doc-backlink">Open Questions</a></h2>
|
||
<ul class="simple">
|
||
<li>will is be too confusing that <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> runs in the current thread?</li>
|
||
<li>should we add pickling fallbacks right now for <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code>, and/or
|
||
<code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code> and <code class="docutils literal notranslate"><span class="pre">Interpreter.get_main_attr()</span></code>?</li>
|
||
<li>should we support (limited) functions in <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> right now?</li>
|
||
<li>rename <code class="docutils literal notranslate"><span class="pre">Interpreter.close()</span></code> to <code class="docutils literal notranslate"><span class="pre">Interpreter.destroy()</span></code>?</li>
|
||
<li>drop <code class="docutils literal notranslate"><span class="pre">Interpreter.get_main_attr()</span></code>, since we have channels?</li>
|
||
<li>should channels be its own PEP?</li>
|
||
</ul>
|
||
</section>
|
||
<section id="deferred-functionality">
|
||
<h2><a class="toc-backref" href="#deferred-functionality" role="doc-backlink">Deferred Functionality</a></h2>
|
||
<p>In the interest of keeping this proposal minimal, the following
|
||
functionality has been left out for future consideration. Note that
|
||
this is not a judgement against any of said capability, but rather a
|
||
deferment. That said, each is arguably valid.</p>
|
||
<section id="add-convenience-api">
|
||
<h3><a class="toc-backref" href="#add-convenience-api" role="doc-backlink">Add convenience API</a></h3>
|
||
<p>There are a number of things I can imagine would smooth out
|
||
<em>hypothetical</em> rough edges with the new module:</p>
|
||
<ul class="simple">
|
||
<li>add something like <code class="docutils literal notranslate"><span class="pre">Interpreter.run()</span></code> or <code class="docutils literal notranslate"><span class="pre">Interpreter.call()</span></code>
|
||
that calls <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> and falls back to pickle</li>
|
||
<li>fall back to pickle in <code class="docutils literal notranslate"><span class="pre">Interpreter.set_main_attrs()</span></code>
|
||
and <code class="docutils literal notranslate"><span class="pre">Interpreter.get_main_attr()</span></code></li>
|
||
</ul>
|
||
<p>These would be easy to do if this proves to be a pain point.</p>
|
||
</section>
|
||
<section id="avoid-possible-confusion-about-interpreters-running-in-the-current-thread">
|
||
<h3><a class="toc-backref" href="#avoid-possible-confusion-about-interpreters-running-in-the-current-thread" role="doc-backlink">Avoid possible confusion about interpreters running in the current thread</a></h3>
|
||
<p>One regular point of confusion has been that <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code>
|
||
executes in the current OS thread, temporarily blocking the current
|
||
Python thread. It may be worth doing something to avoid that confusion.</p>
|
||
<p>Some possible solutions for this hypothetical problem:</p>
|
||
<ul class="simple">
|
||
<li>by default, run in a new thread?</li>
|
||
<li>add <code class="docutils literal notranslate"><span class="pre">Interpreter.exec_in_thread()</span></code>?</li>
|
||
<li>add <code class="docutils literal notranslate"><span class="pre">Interpreter.exec_in_current_thread()</span></code>?</li>
|
||
</ul>
|
||
<p>In earlier versions of this PEP the method was <code class="docutils literal notranslate"><span class="pre">interp.run()</span></code>.
|
||
The simple change to <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> alone will probably reduce
|
||
confusion sufficiently, when coupled with educating users via
|
||
the docs. It it turns out to be a real problem, we can pursue
|
||
one of the alternatives at that point.</p>
|
||
</section>
|
||
<section id="clarify-running-vs-has-threads">
|
||
<h3><a class="toc-backref" href="#clarify-running-vs-has-threads" role="doc-backlink">Clarify “running” vs. “has threads”</a></h3>
|
||
<p><code class="docutils literal notranslate"><span class="pre">Interpreter.is_running()</span></code> refers specifically to whether or not
|
||
<code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> (or similar) is running somewhere. It does not
|
||
say anything about if the interpreter has any subthreads running. That
|
||
information might be helpful.</p>
|
||
<p>Some things we could do:</p>
|
||
<ul class="simple">
|
||
<li>rename <code class="docutils literal notranslate"><span class="pre">Interpreter.is_running()</span></code> to <code class="docutils literal notranslate"><span class="pre">Interpreter.is_running_main()</span></code></li>
|
||
<li>add <code class="docutils literal notranslate"><span class="pre">Interpreter.has_threads()</span></code>, to complement <code class="docutils literal notranslate"><span class="pre">Interpreter.is_running()</span></code></li>
|
||
<li>expand to <code class="docutils literal notranslate"><span class="pre">Interpreter.is_running(main=True,</span> <span class="pre">threads=False)</span></code></li>
|
||
</ul>
|
||
<p>None of these are urgent and any could be done later, if desired.</p>
|
||
</section>
|
||
<section id="a-dunder-method-for-sharing">
|
||
<h3><a class="toc-backref" href="#a-dunder-method-for-sharing" role="doc-backlink">A Dunder Method For Sharing</a></h3>
|
||
<p>We could add a special method, like <code class="docutils literal notranslate"><span class="pre">__xid__</span></code> to correspond to <code class="docutils literal notranslate"><span class="pre">tp_xid</span></code>.
|
||
At the very least, it would allow Python types to convert their instances
|
||
to some other type that implements <code class="docutils literal notranslate"><span class="pre">tp_xid</span></code>.</p>
|
||
<p>The problem is that exposing this capability to Python code presents
|
||
a degree of complixity that hasn’t been explored yet, nor is there
|
||
a compelling case to investigate that complexity.</p>
|
||
</section>
|
||
<section id="interpreter-call">
|
||
<h3><a class="toc-backref" href="#interpreter-call" role="doc-backlink">Interpreter.call()</a></h3>
|
||
<p>It would be convenient to run existing functions in subinterpreters
|
||
directly. <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> could be adjusted to support this or
|
||
a <code class="docutils literal notranslate"><span class="pre">call()</span></code> method could be added:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Interpreter</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This suffers from the same problem as sharing objects between
|
||
interpreters via queues. The minimal solution (running a source string)
|
||
is sufficient for us to get the feature out where it can be explored.</p>
|
||
</section>
|
||
<section id="interpreter-run-in-thread">
|
||
<h3><a class="toc-backref" href="#interpreter-run-in-thread" role="doc-backlink">Interpreter.run_in_thread()</a></h3>
|
||
<p>This method would make a <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> call for you in a thread.
|
||
Doing this using only <code class="docutils literal notranslate"><span class="pre">threading.Thread</span></code> and <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> is
|
||
relatively trivial so we’ve left it out.</p>
|
||
</section>
|
||
<section id="synchronization-primitives">
|
||
<h3><a class="toc-backref" href="#synchronization-primitives" role="doc-backlink">Synchronization Primitives</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">threading</span></code> module provides a number of synchronization primitives
|
||
for coordinating concurrent operations. This is especially necessary
|
||
due to the shared-state nature of threading. In contrast,
|
||
interpreters do not share state. Data sharing is restricted to the
|
||
runtime’s shareable objects capability, which does away with the need
|
||
for explicit synchronization. If any sort of opt-in shared state
|
||
support is added to CPython’s interpreters in the future, that same
|
||
effort can introduce synchronization primitives to meet that need.</p>
|
||
</section>
|
||
<section id="csp-library">
|
||
<h3><a class="toc-backref" href="#csp-library" role="doc-backlink">CSP Library</a></h3>
|
||
<p>A <code class="docutils literal notranslate"><span class="pre">csp</span></code> module would not be a large step away from the functionality
|
||
provided by this PEP. However, adding such a module is outside the
|
||
minimalist goals of this proposal.</p>
|
||
</section>
|
||
<section id="syntactic-support">
|
||
<h3><a class="toc-backref" href="#syntactic-support" role="doc-backlink">Syntactic Support</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">Go</span></code> language provides a concurrency model based on CSP,
|
||
so it’s similar to the concurrency model that multiple interpreters
|
||
support. However, <code class="docutils literal notranslate"><span class="pre">Go</span></code> also provides syntactic support, as well as
|
||
several builtin concurrency primitives, to make concurrency a
|
||
first-class feature. Conceivably, similar syntactic (and builtin)
|
||
support could be added to Python using interpreters. However,
|
||
that is <em>way</em> outside the scope of this PEP!</p>
|
||
</section>
|
||
<section id="multiprocessing">
|
||
<h3><a class="toc-backref" href="#multiprocessing" role="doc-backlink">Multiprocessing</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">multiprocessing</span></code> module could support interpreters in the same
|
||
way it supports threads and processes. In fact, the module’s
|
||
maintainer, Davin Potts, has indicated this is a reasonable feature
|
||
request. However, it is outside the narrow scope of this PEP.</p>
|
||
</section>
|
||
<section id="c-extension-opt-in-opt-out">
|
||
<h3><a class="toc-backref" href="#c-extension-opt-in-opt-out" role="doc-backlink">C-extension opt-in/opt-out</a></h3>
|
||
<p>By using the <code class="docutils literal notranslate"><span class="pre">PyModuleDef_Slot</span></code> introduced by <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a>, we could
|
||
easily add a mechanism by which C-extension modules could opt out of
|
||
multiple interpreter support. Then the import machinery, when operating
|
||
in a subinterpreter, would need to check the module for support.
|
||
It would raise an ImportError if unsupported.</p>
|
||
<p>Alternately we could support opting in to multiple interpreters support.
|
||
However, that would probably exclude many more modules (unnecessarily)
|
||
than the opt-out approach. Also, note that <a class="pep reference internal" href="../pep-0489/" title="PEP 489 – Multi-phase extension module initialization">PEP 489</a> defined that an
|
||
extension’s use of the PEP’s machinery implies multiple interpreters
|
||
support.</p>
|
||
<p>The scope of adding the ModuleDef slot and fixing up the import
|
||
machinery is non-trivial, but could be worth it. It all depends on
|
||
how many extension modules break under subinterpreters. Given that
|
||
there are relatively few cases we know of through mod_wsgi, we can
|
||
leave this for later.</p>
|
||
</section>
|
||
<section id="poisoning-channels">
|
||
<h3><a class="toc-backref" href="#poisoning-channels" role="doc-backlink">Poisoning channels</a></h3>
|
||
<p>CSP has the concept of poisoning a channel. Once a channel has been
|
||
poisoned, any <code class="docutils literal notranslate"><span class="pre">send()</span></code> or <code class="docutils literal notranslate"><span class="pre">recv()</span></code> call on it would raise a special
|
||
exception, effectively ending execution in the interpreter that tried
|
||
to use the poisoned channel.</p>
|
||
<p>This could be accomplished by adding a <code class="docutils literal notranslate"><span class="pre">poison()</span></code> method to both ends
|
||
of the channel. The <code class="docutils literal notranslate"><span class="pre">close()</span></code> method can be used in this way
|
||
(mostly), but these semantics are relatively specialized and can wait.</p>
|
||
</section>
|
||
<section id="resetting-main">
|
||
<h3><a class="toc-backref" href="#resetting-main" role="doc-backlink">Resetting __main__</a></h3>
|
||
<p>As proposed, every call to <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> will execute in the
|
||
namespace of the interpreter’s existing <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module. This means
|
||
that data persists there between <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> calls. Sometimes
|
||
this isn’t desirable and you want to execute in a fresh <code class="docutils literal notranslate"><span class="pre">__main__</span></code>.
|
||
Also, you don’t necessarily want to leak objects there that you aren’t
|
||
using any more.</p>
|
||
<p>Note that the following won’t work right because it will clear too much
|
||
(e.g. <code class="docutils literal notranslate"><span class="pre">__name__</span></code> and the other “__dunder__” attributes:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="s1">'globals().clear()'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Possible solutions include:</p>
|
||
<ul class="simple">
|
||
<li>a <code class="docutils literal notranslate"><span class="pre">create()</span></code> arg to indicate resetting <code class="docutils literal notranslate"><span class="pre">__main__</span></code> after each
|
||
<code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> call</li>
|
||
<li>an <code class="docutils literal notranslate"><span class="pre">Interpreter.reset_main</span></code> flag to support opting in or out
|
||
after the fact</li>
|
||
<li>an <code class="docutils literal notranslate"><span class="pre">Interpreter.reset_main()</span></code> method to opt in when desired</li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">importlib.util.reset_globals()</span></code> <a class="reference internal" href="#reset-globals" id="id11"><span>[reset_globals]</span></a></li>
|
||
</ul>
|
||
<p>Also note that resetting <code class="docutils literal notranslate"><span class="pre">__main__</span></code> does nothing about state stored
|
||
in other modules. So any solution would have to be clear about the
|
||
scope of what is being reset. Conceivably we could invent a mechanism
|
||
by which any (or every) module could be reset, unlike <code class="docutils literal notranslate"><span class="pre">reload()</span></code>
|
||
which does not clear the module before loading into it.</p>
|
||
<p>Regardless, since <code class="docutils literal notranslate"><span class="pre">__main__</span></code> is the execution namespace of the
|
||
interpreter, resetting it has a much more direct correlation to
|
||
interpreters and their dynamic state than does resetting other modules.
|
||
So a more generic module reset mechanism may prove unnecessary.</p>
|
||
<p>This isn’t a critical feature initially. It can wait until later
|
||
if desirable.</p>
|
||
</section>
|
||
<section id="resetting-an-interpreter-s-state">
|
||
<h3><a class="toc-backref" href="#resetting-an-interpreter-s-state" role="doc-backlink">Resetting an interpreter’s state</a></h3>
|
||
<p>It may be nice to re-use an existing subinterpreter instead of
|
||
spinning up a new one. Since an interpreter has substantially more
|
||
state than just the <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module, it isn’t so easy to put an
|
||
interpreter back into a pristine/fresh state. In fact, there <em>may</em>
|
||
be parts of the state that cannot be reset from Python code.</p>
|
||
<p>A possible solution is to add an <code class="docutils literal notranslate"><span class="pre">Interpreter.reset()</span></code> method. This
|
||
would put the interpreter back into the state it was in when newly
|
||
created. If called on a running interpreter it would fail (hence the
|
||
main interpreter could never be reset). This would likely be more
|
||
efficient than creating a new interpreter, though that depends on
|
||
what optimizations will be made later to interpreter creation.</p>
|
||
<p>While this would potentially provide functionality that is not
|
||
otherwise available from Python code, it isn’t a fundamental
|
||
functionality. So in the spirit of minimalism here, this can wait.
|
||
Regardless, I doubt it would be controversial to add it post-PEP.</p>
|
||
</section>
|
||
<section id="copy-an-existing-interpreter-s-state">
|
||
<h3><a class="toc-backref" href="#copy-an-existing-interpreter-s-state" role="doc-backlink">Copy an existing interpreter’s state</a></h3>
|
||
<p>Relatedly, it may be useful to support creating a new interpreter
|
||
based on an existing one, e.g. <code class="docutils literal notranslate"><span class="pre">Interpreter.copy()</span></code>. This ties
|
||
into the idea that a snapshot could be made of an interpreter’s memory,
|
||
which would make starting up CPython, or creating new interpreters,
|
||
faster in general. The same mechanism could be used for a
|
||
hypothetical <code class="docutils literal notranslate"><span class="pre">Interpreter.reset()</span></code>, as described previously.</p>
|
||
</section>
|
||
<section id="shareable-file-descriptors-and-sockets">
|
||
<h3><a class="toc-backref" href="#shareable-file-descriptors-and-sockets" role="doc-backlink">Shareable file descriptors and sockets</a></h3>
|
||
<p>Given that file descriptors and sockets are process-global resources,
|
||
making them shareable is a reasonable idea. They would be a good
|
||
candidate for the first effort at expanding the supported shareable
|
||
types. They aren’t strictly necessary for the initial API.</p>
|
||
</section>
|
||
<section id="integration-with-async">
|
||
<h3><a class="toc-backref" href="#integration-with-async" role="doc-backlink">Integration with async</a></h3>
|
||
<p>Per Antoine Pitrou <a class="reference internal" href="#async" id="id12"><span>[async]</span></a>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Has any thought been given to how FIFOs could integrate with async
|
||
code driven by an event loop (e.g. asyncio)? I think the model of
|
||
executing several asyncio (or Tornado) applications each in their
|
||
own subinterpreter may prove quite interesting to reconcile multi-
|
||
core concurrency with ease of programming. That would require the
|
||
FIFOs to be able to synchronize on something an event loop can wait
|
||
on (probably a file descriptor?).
|
||
</pre></div>
|
||
</div>
|
||
<p>The basic functionality of multiple interpreters support does not depend
|
||
on async and can be added later.</p>
|
||
<p>A possible solution is to provide async implementations of the blocking
|
||
channel methods (<code class="docutils literal notranslate"><span class="pre">recv()</span></code>, and <code class="docutils literal notranslate"><span class="pre">send()</span></code>).</p>
|
||
<p>Alternately, “readiness callbacks” could be used to simplify use in
|
||
async scenarios. This would mean adding an optional <code class="docutils literal notranslate"><span class="pre">callback</span></code>
|
||
(kw-only) parameter to the <code class="docutils literal notranslate"><span class="pre">recv_nowait()</span></code> and <code class="docutils literal notranslate"><span class="pre">send_nowait()</span></code>
|
||
channel methods. The callback would be called once the object was sent
|
||
or received (respectively).</p>
|
||
<p>(Note that making channels buffered makes readiness callbacks less
|
||
important.)</p>
|
||
</section>
|
||
<section id="support-for-iteration">
|
||
<h3><a class="toc-backref" href="#support-for-iteration" role="doc-backlink">Support for iteration</a></h3>
|
||
<p>Supporting iteration on <code class="docutils literal notranslate"><span class="pre">RecvChannel</span></code> (via <code class="docutils literal notranslate"><span class="pre">__iter__()</span></code> or
|
||
<code class="docutils literal notranslate"><span class="pre">_next__()</span></code>) may be useful. A trivial implementation would use the
|
||
<code class="docutils literal notranslate"><span class="pre">recv()</span></code> method, similar to how files do iteration. Since this isn’t
|
||
a fundamental capability and has a simple analog, adding iteration
|
||
support can wait until later.</p>
|
||
</section>
|
||
<section id="channel-context-managers">
|
||
<h3><a class="toc-backref" href="#channel-context-managers" role="doc-backlink">Channel context managers</a></h3>
|
||
<p>Context manager support on <code class="docutils literal notranslate"><span class="pre">RecvChannel</span></code> and <code class="docutils literal notranslate"><span class="pre">SendChannel</span></code> may be
|
||
helpful. The implementation would be simple, wrapping a call to
|
||
<code class="docutils literal notranslate"><span class="pre">close()</span></code> (or maybe <code class="docutils literal notranslate"><span class="pre">release()</span></code>) like files do. As with iteration,
|
||
this can wait.</p>
|
||
</section>
|
||
<section id="pipes-and-queues">
|
||
<h3><a class="toc-backref" href="#pipes-and-queues" role="doc-backlink">Pipes and Queues</a></h3>
|
||
<p>With the proposed object passing mechanism of “os.pipe()”, other similar
|
||
basic types aren’t strictly required to achieve the minimal useful
|
||
functionality of multiple interpreters. Such types include pipes
|
||
(like unbuffered channels, but one-to-one) and queues (like channels,
|
||
but more generic). See below in <a class="reference internal" href="#rejected-ideas">Rejected Ideas</a> for more information.</p>
|
||
<p>Even though these types aren’t part of this proposal, they may still
|
||
be useful in the context of concurrency. Adding them later is entirely
|
||
reasonable. The could be trivially implemented as wrappers around
|
||
channels. Alternatively they could be implemented for efficiency at the
|
||
same low level as channels.</p>
|
||
</section>
|
||
<section id="return-a-lock-from-send">
|
||
<h3><a class="toc-backref" href="#return-a-lock-from-send" role="doc-backlink">Return a lock from send()</a></h3>
|
||
<p>When sending an object through a channel, you don’t have a way of knowing
|
||
when the object gets received on the other end. One way to work around
|
||
this is to return a locked <code class="docutils literal notranslate"><span class="pre">threading.Lock</span></code> from <code class="docutils literal notranslate"><span class="pre">SendChannel.send()</span></code>
|
||
that unlocks once the object is received.</p>
|
||
<p>Alternately, the proposed <code class="docutils literal notranslate"><span class="pre">SendChannel.send()</span></code> (blocking) and
|
||
<code class="docutils literal notranslate"><span class="pre">SendChannel.send_nowait()</span></code> provide an explicit distinction that is
|
||
less likely to confuse users.</p>
|
||
<p>Note that returning a lock would matter for buffered channels
|
||
(i.e. queues). For unbuffered channels it is a non-issue.</p>
|
||
</section>
|
||
<section id="support-prioritization-in-channels">
|
||
<h3><a class="toc-backref" href="#support-prioritization-in-channels" role="doc-backlink">Support prioritization in channels</a></h3>
|
||
<p>A simple example is <code class="docutils literal notranslate"><span class="pre">queue.PriorityQueue</span></code> in the stdlib.</p>
|
||
</section>
|
||
<section id="support-inheriting-settings-and-more">
|
||
<h3><a class="toc-backref" href="#support-inheriting-settings-and-more" role="doc-backlink">Support inheriting settings (and more?)</a></h3>
|
||
<p>Folks might find it useful, when creating a new interpreter, to be
|
||
able to indicate that they would like some things “inherited” by the
|
||
new interpreter. The mechanism could be a strict copy or it could be
|
||
copy-on-write. The motivating example is with the warnings module
|
||
(e.g. copy the filters).</p>
|
||
<p>The feature isn’t critical, nor would it be widely useful, so it
|
||
can wait until there’s interest. Notably, both suggested solutions
|
||
will require significant work, especially when it comes to complex
|
||
objects and most especially for mutable containers of mutable
|
||
complex objects.</p>
|
||
</section>
|
||
<section id="make-exceptions-shareable">
|
||
<h3><a class="toc-backref" href="#make-exceptions-shareable" role="doc-backlink">Make exceptions shareable</a></h3>
|
||
<p>Exceptions are propagated out of <code class="docutils literal notranslate"><span class="pre">run()</span></code> calls, so it isn’t a big
|
||
leap to make them shareable. However, as noted elsewhere,
|
||
it isn’t essential or (particularly common) so we can wait on doing
|
||
that.</p>
|
||
</section>
|
||
<section id="make-everything-shareable-through-serialization">
|
||
<h3><a class="toc-backref" href="#make-everything-shareable-through-serialization" role="doc-backlink">Make everything shareable through serialization</a></h3>
|
||
<p>We could use pickle (or marshal) to serialize everything and thus
|
||
make them shareable. Doing this is potentially inefficient,
|
||
but it may be a matter of convenience in the end.
|
||
We can add it later, but trying to remove it later
|
||
would be significantly more painful.</p>
|
||
</section>
|
||
<section id="make-runfailederror-cause-lazy">
|
||
<h3><a class="toc-backref" href="#make-runfailederror-cause-lazy" role="doc-backlink">Make RunFailedError.__cause__ lazy</a></h3>
|
||
<p>An uncaught exception in a subinterpreter (from <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code>) is
|
||
copied to the calling interpreter and set as <code class="docutils literal notranslate"><span class="pre">__cause__</span></code> on a
|
||
<code class="docutils literal notranslate"><span class="pre">RunFailedError</span></code> which is then raised. That copying part involves
|
||
some sort of deserialization in the calling interpreter, which can be
|
||
expensive (e.g. due to imports) yet is not always necessary.</p>
|
||
<p>So it may be useful to use an <code class="docutils literal notranslate"><span class="pre">ExceptionProxy</span></code> type to wrap the
|
||
serialized exception and only deserialize it when needed. That could
|
||
be via <code class="docutils literal notranslate"><span class="pre">ExceptionProxy__getattribute__()</span></code> or perhaps through
|
||
<code class="docutils literal notranslate"><span class="pre">RunFailedError.resolve()</span></code> (which would raise the deserialized
|
||
exception and set <code class="docutils literal notranslate"><span class="pre">RunFailedError.__cause__</span></code> to the exception.</p>
|
||
<p>It may also make sense to have <code class="docutils literal notranslate"><span class="pre">RunFailedError.__cause__</span></code> be a
|
||
descriptor that does the lazy deserialization (and set <code class="docutils literal notranslate"><span class="pre">__cause__</span></code>)
|
||
on the <code class="docutils literal notranslate"><span class="pre">RunFailedError</span></code> instance.</p>
|
||
</section>
|
||
<section id="return-a-value-from-interp-exec">
|
||
<h3><a class="toc-backref" href="#return-a-value-from-interp-exec" role="doc-backlink">Return a value from <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code></a></h3>
|
||
<p>Currently <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> always returns None. One idea is to return
|
||
the return value from whatever the subinterpreter ran. However, for now
|
||
it doesn’t make sense. The only thing folks can run is a string of
|
||
code (i.e. a script). This is equivalent to <code class="docutils literal notranslate"><span class="pre">PyRun_StringFlags()</span></code>,
|
||
<code class="docutils literal notranslate"><span class="pre">exec()</span></code>, or a module body. None of those “return” anything. We can
|
||
revisit this once <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> supports functions, etc.</p>
|
||
</section>
|
||
<section id="add-a-shareable-synchronization-primitive">
|
||
<h3><a class="toc-backref" href="#add-a-shareable-synchronization-primitive" role="doc-backlink">Add a shareable synchronization primitive</a></h3>
|
||
<p>This would be <code class="docutils literal notranslate"><span class="pre">_threading.Lock</span></code> (or something like it) where
|
||
interpreters would actually share the underlying mutex. The main
|
||
concern is that locks and isolated interpreters may not mix well
|
||
(as learned in Go).</p>
|
||
<p>We can add this later if it proves desirable without much trouble.</p>
|
||
</section>
|
||
<section id="propagate-systemexit-and-keyboardinterrupt-differently">
|
||
<h3><a class="toc-backref" href="#propagate-systemexit-and-keyboardinterrupt-differently" role="doc-backlink">Propagate SystemExit and KeyboardInterrupt Differently</a></h3>
|
||
<p>The exception types that inherit from <code class="docutils literal notranslate"><span class="pre">BaseException</span></code> (aside from
|
||
<code class="docutils literal notranslate"><span class="pre">Exception</span></code>) are usually treated specially. These types are:
|
||
<code class="docutils literal notranslate"><span class="pre">KeyboardInterrupt</span></code>, <code class="docutils literal notranslate"><span class="pre">SystemExit</span></code>, and <code class="docutils literal notranslate"><span class="pre">GeneratorExit</span></code>. It may
|
||
make sense to treat them specially when it comes to propagation from
|
||
<code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code>. Here are some options:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="o">*</span> <span class="n">propagate</span> <span class="n">like</span> <span class="n">normal</span> <span class="n">via</span> <span class="n">RunFailedError</span>
|
||
<span class="o">*</span> <span class="n">do</span> <span class="ow">not</span> <span class="n">propagate</span> <span class="p">(</span><span class="n">handle</span> <span class="n">them</span> <span class="n">somehow</span> <span class="ow">in</span> <span class="n">the</span> <span class="n">subinterpreter</span><span class="p">)</span>
|
||
<span class="o">*</span> <span class="n">propagate</span> <span class="n">them</span> <span class="n">directly</span> <span class="p">(</span><span class="n">avoid</span> <span class="n">RunFailedError</span><span class="p">)</span>
|
||
<span class="o">*</span> <span class="n">propagate</span> <span class="n">them</span> <span class="n">directly</span> <span class="p">(</span><span class="nb">set</span> <span class="n">RunFailedError</span> <span class="k">as</span> <span class="n">__cause__</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>We aren’t going to worry about handling them differently. Threads
|
||
already ignore <code class="docutils literal notranslate"><span class="pre">SystemExit</span></code>, so for now we will follow that pattern.</p>
|
||
</section>
|
||
<section id="add-an-explicit-release-and-close-to-channel-end-classes">
|
||
<h3><a class="toc-backref" href="#add-an-explicit-release-and-close-to-channel-end-classes" role="doc-backlink">Add an explicit release() and close() to channel end classes</a></h3>
|
||
<p>It can be convenient to have an explicit way to close a channel against
|
||
further global use. Likewise it could be useful to have an explicit
|
||
way to release one of the channel ends relative to the current
|
||
interpreter. Among other reasons, such a mechanism is useful for
|
||
communicating overall state between interpreters without the extra
|
||
boilerplate that passing objects through a channel directly would
|
||
require.</p>
|
||
<p>The challenge is getting automatic release/close right without making
|
||
it hard to understand. This is especially true when dealing with a
|
||
non-empty channel. We should be able to get by without release/close
|
||
for now.</p>
|
||
</section>
|
||
<section id="add-sendchannel-send-buffer">
|
||
<h3><a class="toc-backref" href="#add-sendchannel-send-buffer" role="doc-backlink">Add SendChannel.send_buffer()</a></h3>
|
||
<p>This method would allow no-copy sending of an object through a channel
|
||
if it supports the <a class="pep reference internal" href="../pep-3118/" title="PEP 3118 – Revising the buffer protocol">PEP 3118</a> buffer protocol (e.g. memoryview).</p>
|
||
<p>Support for this is not fundamental to channels and can be added on
|
||
later without much disruption.</p>
|
||
</section>
|
||
<section id="auto-run-in-a-thread">
|
||
<h3><a class="toc-backref" href="#auto-run-in-a-thread" role="doc-backlink">Auto-run in a thread</a></h3>
|
||
<p>The PEP proposes a hard separation between subinterpreters and threads:
|
||
if you want to run in a thread you must create the thread yourself and
|
||
call <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> in it. However, it might be convenient if
|
||
<code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> could do that for you, meaning there would be less
|
||
boilerplate.</p>
|
||
<p>Furthermore, we anticipate that users will want to run in a thread much
|
||
more often than not. So it would make sense to make this the default
|
||
behavior. We would add a kw-only param “threaded” (default <code class="docutils literal notranslate"><span class="pre">True</span></code>)
|
||
to <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> to allow the run-in-the-current-thread operation.</p>
|
||
</section>
|
||
</section>
|
||
<section id="rejected-ideas">
|
||
<h2><a class="toc-backref" href="#rejected-ideas" role="doc-backlink">Rejected Ideas</a></h2>
|
||
<section id="explicit-channel-association">
|
||
<h3><a class="toc-backref" href="#explicit-channel-association" role="doc-backlink">Explicit channel association</a></h3>
|
||
<p>Interpreters are implicitly associated with channels upon <code class="docutils literal notranslate"><span class="pre">recv()</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">send()</span></code> calls. They are de-associated with <code class="docutils literal notranslate"><span class="pre">release()</span></code> calls. The
|
||
alternative would be explicit methods. It would be either
|
||
<code class="docutils literal notranslate"><span class="pre">add_channel()</span></code> and <code class="docutils literal notranslate"><span class="pre">remove_channel()</span></code> methods on <code class="docutils literal notranslate"><span class="pre">Interpreter</span></code>
|
||
objects or something similar on channel objects.</p>
|
||
<p>In practice, this level of management shouldn’t be necessary for users.
|
||
So adding more explicit support would only add clutter to the API.</p>
|
||
</section>
|
||
<section id="add-an-api-based-on-pipes">
|
||
<h3><a class="toc-backref" href="#add-an-api-based-on-pipes" role="doc-backlink">Add an API based on pipes</a></h3>
|
||
<p>A pipe would be a simplex FIFO between exactly two interpreters. For
|
||
most use cases this would be sufficient. It could potentially simplify
|
||
the implementation as well. However, it isn’t a big step to supporting
|
||
a many-to-many simplex FIFO via channels. Also, with pipes the API
|
||
ends up being slightly more complicated, requiring naming the pipes.</p>
|
||
</section>
|
||
<section id="add-an-api-based-on-queues">
|
||
<h3><a class="toc-backref" href="#add-an-api-based-on-queues" role="doc-backlink">Add an API based on queues</a></h3>
|
||
<p>Queues and buffered channels are almost the same thing. The main
|
||
difference is that channels have a stronger relationship with context
|
||
(i.e. the associated interpreter).</p>
|
||
<p>The name “Channel” was used instead of “Queue” to avoid confusion with
|
||
the stdlib <code class="docutils literal notranslate"><span class="pre">queue.Queue</span></code>.</p>
|
||
</section>
|
||
<section id="enumerate">
|
||
<h3><a class="toc-backref" href="#enumerate" role="doc-backlink">“enumerate”</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">list_all()</span></code> function provides the list of all interpreters.
|
||
In the threading module, which partly inspired the proposed API, the
|
||
function is called <code class="docutils literal notranslate"><span class="pre">enumerate()</span></code>. The name is different here to
|
||
avoid confusing Python users that are not already familiar with the
|
||
threading API. For them “enumerate” is rather unclear, whereas
|
||
“list_all” is clear.</p>
|
||
</section>
|
||
<section id="alternate-solutions-to-prevent-leaking-exceptions-across-interpreters">
|
||
<h3><a class="toc-backref" href="#alternate-solutions-to-prevent-leaking-exceptions-across-interpreters" role="doc-backlink">Alternate solutions to prevent leaking exceptions across interpreters</a></h3>
|
||
<p>In function calls, uncaught exceptions propagate to the calling frame.
|
||
The same approach could be taken with <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code>. However, this
|
||
would mean that exception objects would leak across the inter-interpreter
|
||
boundary. Likewise, the frames in the traceback would potentially leak.</p>
|
||
<p>While that might not be a problem currently, it would be a problem once
|
||
interpreters get better isolation relative to memory management (which
|
||
is necessary to stop sharing the GIL between interpreters). We’ve
|
||
resolved the semantics of how the exceptions propagate by raising a
|
||
<code class="docutils literal notranslate"><span class="pre">RunFailedError</span></code> instead, for which <code class="docutils literal notranslate"><span class="pre">__cause__</span></code> wraps a safe proxy
|
||
for the original exception and traceback.</p>
|
||
<p>Rejected possible solutions:</p>
|
||
<ul class="simple">
|
||
<li>reproduce the exception and traceback in the original interpreter
|
||
and raise that.</li>
|
||
<li>raise a subclass of RunFailedError that proxies the original
|
||
exception and traceback.</li>
|
||
<li>raise RuntimeError instead of RunFailedError</li>
|
||
<li>convert at the boundary (a la <code class="docutils literal notranslate"><span class="pre">subprocess.CalledProcessError</span></code>)
|
||
(requires a cross-interpreter representation)</li>
|
||
<li>support customization via <code class="docutils literal notranslate"><span class="pre">Interpreter.excepthook</span></code>
|
||
(requires a cross-interpreter representation)</li>
|
||
<li>wrap in a proxy at the boundary (including with support for
|
||
something like <code class="docutils literal notranslate"><span class="pre">err.raise()</span></code> to propagate the traceback).</li>
|
||
<li>return the exception (or its proxy) from <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code> instead of
|
||
raising it</li>
|
||
<li>return a result object (like <code class="docutils literal notranslate"><span class="pre">subprocess</span></code> does) <a class="reference internal" href="#result-object" id="id13"><span>[result-object]</span></a>
|
||
(unnecessary complexity?)</li>
|
||
<li>throw the exception away and expect users to deal with unhandled
|
||
exceptions explicitly in the script they pass to <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code>
|
||
(they can pass error info out via channels);
|
||
with threads you have to do something similar</li>
|
||
</ul>
|
||
</section>
|
||
<section id="always-associate-each-new-interpreter-with-its-own-thread">
|
||
<h3><a class="toc-backref" href="#always-associate-each-new-interpreter-with-its-own-thread" role="doc-backlink">Always associate each new interpreter with its own thread</a></h3>
|
||
<p>As implemented in the C-API, an interpreter is not inherently tied to
|
||
any thread. Furthermore, it will run in any existing thread, whether
|
||
created by Python or not. You only have to activate one of its thread
|
||
states (<code class="docutils literal notranslate"><span class="pre">PyThreadState</span></code>) in the thread first. This means that the
|
||
same thread may run more than one interpreter (though obviously
|
||
not at the same time).</p>
|
||
<p>The proposed module maintains this behavior. Interpreters are not
|
||
tied to threads. Only calls to <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> are. However,
|
||
one of the key objectives of this PEP is to provide a more
|
||
human-centric concurrency model. With that in mind, from a conceptual
|
||
standpoint the module <em>might</em> be easier to understand if each
|
||
interpreter were associated with its own thread.</p>
|
||
<p>That would mean <code class="docutils literal notranslate"><span class="pre">interpreters.create()</span></code> would create a new thread
|
||
and <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> would only execute in that thread (and
|
||
nothing else would). The benefit is that users would not have to
|
||
wrap <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> calls in a new <code class="docutils literal notranslate"><span class="pre">threading.Thread</span></code>. Nor
|
||
would they be in a position to accidentally pause the current
|
||
interpreter (in the current thread) while their interpreter
|
||
executes.</p>
|
||
<p>The idea is rejected because the benefit is small and the cost is high.
|
||
The difference from the capability in the C-API would be potentially
|
||
confusing. The implicit creation of threads is magical. The early
|
||
creation of threads is potentially wasteful. The inability to run
|
||
arbitrary interpreters in an existing thread would prevent some valid
|
||
use cases, frustrating users. Tying interpreters to threads would
|
||
require extra runtime modifications. It would also make the module’s
|
||
implementation overly complicated. Finally, it might not even make
|
||
the module easier to understand.</p>
|
||
</section>
|
||
<section id="only-associate-interpreters-upon-use">
|
||
<h3><a class="toc-backref" href="#only-associate-interpreters-upon-use" role="doc-backlink">Only associate interpreters upon use</a></h3>
|
||
<p>Associate interpreters with channel ends only once <code class="docutils literal notranslate"><span class="pre">recv()</span></code>,
|
||
<code class="docutils literal notranslate"><span class="pre">send()</span></code>, etc. are called.</p>
|
||
<p>Doing this is potentially confusing and also can lead to unexpected
|
||
races where a channel is auto-closed before it can be used in the
|
||
original (creating) interpreter.</p>
|
||
</section>
|
||
<section id="allow-multiple-simultaneous-calls-to-interpreter-exec">
|
||
<h3><a class="toc-backref" href="#allow-multiple-simultaneous-calls-to-interpreter-exec" role="doc-backlink">Allow multiple simultaneous calls to Interpreter.exec()</a></h3>
|
||
<p>This would make sense especially if <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> were to
|
||
manage new threads for you (which we’ve rejected). Essentially,
|
||
each call would run independently, which would be mostly fine
|
||
from a narrow technical standpoint, since each interpreter
|
||
can have multiple threads.</p>
|
||
<p>The problem is that the interpreter has only one <code class="docutils literal notranslate"><span class="pre">__main__</span></code> module
|
||
and simultaneous <code class="docutils literal notranslate"><span class="pre">Interpreter.exec()</span></code> calls would have to sort out
|
||
sharing <code class="docutils literal notranslate"><span class="pre">__main__</span></code> or we’d have to invent a new mechanism. Neither
|
||
would be simple enough to be worth doing.</p>
|
||
</section>
|
||
<section id="add-a-reraise-method-to-runfailederror">
|
||
<h3><a class="toc-backref" href="#add-a-reraise-method-to-runfailederror" role="doc-backlink">Add a “reraise” method to RunFailedError</a></h3>
|
||
<p>While having <code class="docutils literal notranslate"><span class="pre">__cause__</span></code> set on <code class="docutils literal notranslate"><span class="pre">RunFailedError</span></code> helps produce a
|
||
more useful traceback, it’s less helpful when handling the original
|
||
error. To help facilitate this, we could add
|
||
<code class="docutils literal notranslate"><span class="pre">RunFailedError.reraise()</span></code>. This method would enable the following
|
||
pattern:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">try</span><span class="p">:</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">script</span><span class="p">)</span>
|
||
<span class="k">except</span> <span class="n">RunFailedError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="n">exc</span><span class="o">.</span><span class="n">reraise</span><span class="p">()</span>
|
||
<span class="k">except</span> <span class="n">MyException</span><span class="p">:</span>
|
||
<span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This would be made even simpler if there existed a <code class="docutils literal notranslate"><span class="pre">__reraise__</span></code>
|
||
protocol.</p>
|
||
<p>All that said, this is completely unnecessary. Using <code class="docutils literal notranslate"><span class="pre">__cause__</span></code>
|
||
is good enough:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">try</span><span class="p">:</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">script</span><span class="p">)</span>
|
||
<span class="k">except</span> <span class="n">RunFailedError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="n">exc</span><span class="o">.</span><span class="n">__cause__</span>
|
||
<span class="k">except</span> <span class="n">MyException</span><span class="p">:</span>
|
||
<span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Note that in extreme cases it may require a little extra boilerplate:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">try</span><span class="p">:</span>
|
||
<span class="k">try</span><span class="p">:</span>
|
||
<span class="n">interp</span><span class="o">.</span><span class="n">exec</span><span class="p">(</span><span class="n">script</span><span class="p">)</span>
|
||
<span class="k">except</span> <span class="n">RunFailedError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||
<span class="k">if</span> <span class="n">exc</span><span class="o">.</span><span class="n">__cause__</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="k">raise</span> <span class="n">exc</span><span class="o">.</span><span class="n">__cause__</span>
|
||
<span class="k">raise</span> <span class="c1"># re-raise</span>
|
||
<span class="k">except</span> <span class="n">MyException</span><span class="p">:</span>
|
||
<span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="implementation">
|
||
<h2><a class="toc-backref" href="#implementation" role="doc-backlink">Implementation</a></h2>
|
||
<p>The implementation of the PEP has 4 parts:</p>
|
||
<ul class="simple">
|
||
<li>the high-level module described in this PEP (mostly a light wrapper
|
||
around a low-level C extension</li>
|
||
<li>the low-level C extension module</li>
|
||
<li>additions to the internal C-API needed by the low-level module</li>
|
||
<li>secondary fixes/changes in the CPython runtime that facilitate
|
||
the low-level module (among other benefits)</li>
|
||
</ul>
|
||
<p>These are at various levels of completion, with more done the lower
|
||
you go:</p>
|
||
<ul class="simple">
|
||
<li>the high-level module has been, at best, roughly implemented.
|
||
However, fully implementing it will be almost trivial.</li>
|
||
<li>the low-level module is mostly complete. The bulk of the
|
||
implementation was merged into master in December 2018 as the
|
||
“_xxsubinterpreters” module (for the sake of testing multiple
|
||
interpreters functionality). Only the exception propagation
|
||
implementation remains to be finished, which will not require
|
||
extensive work.</li>
|
||
<li>all necessary C-API work has been finished</li>
|
||
<li>all anticipated work in the runtime has been finished</li>
|
||
</ul>
|
||
<p>The implementation effort for <a class="pep reference internal" href="../pep-0554/" title="PEP 554 – Multiple Interpreters in the Stdlib">PEP 554</a> is being tracked as part of
|
||
a larger project aimed at improving multi-core support in CPython.
|
||
<a class="reference internal" href="#multi-core-project" id="id14"><span>[multi-core-project]</span></a></p>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<div role="list" class="citation-list">
|
||
<div class="citation" id="c-api" role="doc-biblioentry">
|
||
<dt class="label" id="c-api">[<a href="#id1">c-api</a>]</dt>
|
||
<dd><a class="reference external" href="https://docs.python.org/3/c-api/init.html#sub-interpreter-support">https://docs.python.org/3/c-api/init.html#sub-interpreter-support</a></div>
|
||
<div class="citation" id="csp" role="doc-biblioentry">
|
||
<dt class="label" id="csp">[<a href="#id5">CSP</a>]</dt>
|
||
<dd><a class="reference external" href="https://en.wikipedia.org/wiki/Communicating_sequential_processes">https://en.wikipedia.org/wiki/Communicating_sequential_processes</a>
|
||
<a class="reference external" href="https://github.com/futurecore/python-csp">https://github.com/futurecore/python-csp</a></div>
|
||
<div class="citation" id="fifo" role="doc-biblioentry">
|
||
<dt class="label" id="fifo">[<a href="#id10">fifo</a>]</dt>
|
||
<dd><a class="reference external" href="https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Pipe">https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Pipe</a>
|
||
<a class="reference external" href="https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue">https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue</a>
|
||
<a class="reference external" href="https://docs.python.org/3/library/queue.html#module-queue">https://docs.python.org/3/library/queue.html#module-queue</a>
|
||
<a class="reference external" href="http://stackless.readthedocs.io/en/2.7-slp/library/stackless/channels.html">http://stackless.readthedocs.io/en/2.7-slp/library/stackless/channels.html</a>
|
||
<a class="reference external" href="https://golang.org/doc/effective_go.html#sharing">https://golang.org/doc/effective_go.html#sharing</a>
|
||
<a class="reference external" href="http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/">http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/</a></div>
|
||
<div class="citation" id="caveats" role="doc-biblioentry">
|
||
<dt class="label" id="caveats">[<a href="#id6">caveats</a>]</dt>
|
||
<dd><a class="reference external" href="https://docs.python.org/3/c-api/init.html#bugs-and-caveats">https://docs.python.org/3/c-api/init.html#bugs-and-caveats</a></div>
|
||
<div class="citation" id="cryptography" role="doc-biblioentry">
|
||
<dt class="label" id="cryptography">[<a href="#id7">cryptography</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/pyca/cryptography/issues/2299">https://github.com/pyca/cryptography/issues/2299</a></div>
|
||
<div class="citation" id="gilstate" role="doc-biblioentry">
|
||
<dt class="label" id="gilstate">[<a href="#id8">gilstate</a>]</dt>
|
||
<dd><a class="reference external" href="https://bugs.python.org/issue10915">https://bugs.python.org/issue10915</a>
|
||
<a class="reference external" href="http://bugs.python.org/issue15751">http://bugs.python.org/issue15751</a></div>
|
||
<div class="citation" id="bug-rate" role="doc-biblioentry">
|
||
<dt class="label" id="bug-rate">[<a href="#id3">bug-rate</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-September/047094.html">https://mail.python.org/pipermail/python-ideas/2017-September/047094.html</a></div>
|
||
<div class="citation" id="benefits" role="doc-biblioentry">
|
||
<dt class="label" id="benefits">[<a href="#id2">benefits</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-September/047122.html">https://mail.python.org/pipermail/python-ideas/2017-September/047122.html</a></div>
|
||
<div class="citation" id="reset-globals" role="doc-biblioentry">
|
||
<dt class="label" id="reset-globals">[<a href="#id11">reset_globals</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-dev/2017-September/149545.html">https://mail.python.org/pipermail/python-dev/2017-September/149545.html</a></div>
|
||
<div class="citation" id="async" role="doc-biblioentry">
|
||
<dt class="label" id="async">[<a href="#id12">async</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-dev/2017-September/149420.html">https://mail.python.org/pipermail/python-dev/2017-September/149420.html</a>
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-dev/2017-September/149585.html">https://mail.python.org/pipermail/python-dev/2017-September/149585.html</a></div>
|
||
<div class="citation" id="result-object" role="doc-biblioentry">
|
||
<dt class="label" id="result-object">[<a href="#id13">result-object</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-dev/2017-September/149562.html">https://mail.python.org/pipermail/python-dev/2017-September/149562.html</a></div>
|
||
<div class="citation" id="jython" role="doc-biblioentry">
|
||
<dt class="label" id="jython">[<a href="#id9">jython</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-May/045771.html">https://mail.python.org/pipermail/python-ideas/2017-May/045771.html</a></div>
|
||
<div class="citation" id="multi-core-project" role="doc-biblioentry">
|
||
<dt class="label" id="multi-core-project">[<a href="#id14">multi-core-project</a>]</dt>
|
||
<dd><a class="reference external" href="https://github.com/ericsnowcurrently/multi-core-python">https://github.com/ericsnowcurrently/multi-core-python</a></div>
|
||
<div class="citation" id="cache-line-ping-pong" role="doc-biblioentry">
|
||
<dt class="label" id="cache-line-ping-pong">[<a href="#id4">cache-line-ping-pong</a>]</dt>
|
||
<dd><a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/message/3HVRFWHDMWPNR367GXBILZ4JJAUQ2STZ/">https://mail.python.org/archives/list/python-dev@python.org/message/3HVRFWHDMWPNR367GXBILZ4JJAUQ2STZ/</a></div>
|
||
</div>
|
||
<ul class="simple">
|
||
<li><dl class="simple">
|
||
<dt>mp-conn</dt><dd><a class="reference external" href="https://docs.python.org/3/library/multiprocessing.html#connection-objects">https://docs.python.org/3/library/multiprocessing.html#connection-objects</a></dd>
|
||
</dl>
|
||
</li>
|
||
<li><dl class="simple">
|
||
<dt>main-thread</dt><dd><a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2017-September/047144.html">https://mail.python.org/pipermail/python-ideas/2017-September/047144.html</a>
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-dev/2017-September/149566.html">https://mail.python.org/pipermail/python-dev/2017-September/149566.html</a></dd>
|
||
</dl>
|
||
</li>
|
||
<li><dl class="simple">
|
||
<dt>petr-c-ext</dt><dd><a class="reference external" href="https://mail.python.org/pipermail/import-sig/2016-June/001062.html">https://mail.python.org/pipermail/import-sig/2016-June/001062.html</a>
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-ideas/2016-April/039748.html">https://mail.python.org/pipermail/python-ideas/2016-April/039748.html</a></dd>
|
||
</dl>
|
||
</li>
|
||
</ul>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0554.rst">https://github.com/python/peps/blob/main/peps/pep-0554.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0554.rst">2023-11-28 02:32:35 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a><ul>
|
||
<li><a class="reference internal" href="#the-interpreters-module">The “interpreters” Module</a></li>
|
||
<li><a class="reference internal" href="#api-summary-for-interpreters-module">API summary for interpreters module</a></li>
|
||
<li><a class="reference internal" href="#concurrent-futures-interpreterpoolexecutor">concurrent.futures.InterpreterPoolExecutor</a></li>
|
||
<li><a class="reference internal" href="#help-for-extension-module-maintainers">Help for Extension Module Maintainers</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#examples">Examples</a><ul>
|
||
<li><a class="reference internal" href="#run-isolated-code-in-current-os-thread">Run isolated code in current OS thread</a></li>
|
||
<li><a class="reference internal" href="#run-in-a-different-thread">Run in a different thread</a></li>
|
||
<li><a class="reference internal" href="#pre-populate-an-interpreter">Pre-populate an interpreter</a></li>
|
||
<li><a class="reference internal" href="#handling-an-exception">Handling an exception</a></li>
|
||
<li><a class="reference internal" href="#re-raising-an-exception">Re-raising an exception</a></li>
|
||
<li><a class="reference internal" href="#interact-with-the-main-namespace">Interact with the __main__ namespace</a></li>
|
||
<li><a class="reference internal" href="#synchronize-using-an-os-pipe">Synchronize using an OS pipe</a></li>
|
||
<li><a class="reference internal" href="#sharing-a-file-descriptor">Sharing a file descriptor</a></li>
|
||
<li><a class="reference internal" href="#passing-objects-via-pickle">Passing objects via pickle</a></li>
|
||
<li><a class="reference internal" href="#capturing-an-interpreter-s-stdout">Capturing an interpreter’s stdout</a></li>
|
||
<li><a class="reference internal" href="#running-a-module">Running a module</a></li>
|
||
<li><a class="reference internal" href="#running-as-script-including-zip-archives-directories">Running as script (including zip archives & directories)</a></li>
|
||
<li><a class="reference internal" href="#using-a-channel-to-communicate">Using a channel to communicate</a></li>
|
||
<li><a class="reference internal" href="#sharing-a-memoryview-imagine-map-reduce">Sharing a memoryview (imagine map-reduce)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a><ul>
|
||
<li><a class="reference internal" href="#concerns">Concerns</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#about-subinterpreters">About Subinterpreters</a><ul>
|
||
<li><a class="reference internal" href="#concurrency">Concurrency</a></li>
|
||
<li><a class="reference internal" href="#shared-data">Shared Data</a></li>
|
||
<li><a class="reference internal" href="#interpreter-isolation">Interpreter Isolation</a></li>
|
||
<li><a class="reference internal" href="#existing-usage">Existing Usage</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#alternate-python-implementations">Alternate Python Implementations</a></li>
|
||
<li><a class="reference internal" href="#interpreters-module-api">“interpreters” Module API</a><ul>
|
||
<li><a class="reference internal" href="#uncaught-exceptions">Uncaught Exceptions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#interpreter-restrictions">Interpreter Restrictions</a></li>
|
||
<li><a class="reference internal" href="#api-for-communication">API For Communication</a><ul>
|
||
<li><a class="reference internal" href="#shareable-types">Shareable Types</a><ul>
|
||
<li><a class="reference internal" href="#communicating-through-os-pipes">Communicating Through OS Pipes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#channels">Channels</a></li>
|
||
<li><a class="reference internal" href="#caveats-for-shared-objects">Caveats For Shared Objects</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#documentation">Documentation</a></li>
|
||
<li><a class="reference internal" href="#alternative-solutions">Alternative Solutions</a></li>
|
||
<li><a class="reference internal" href="#open-questions">Open Questions</a></li>
|
||
<li><a class="reference internal" href="#deferred-functionality">Deferred Functionality</a><ul>
|
||
<li><a class="reference internal" href="#add-convenience-api">Add convenience API</a></li>
|
||
<li><a class="reference internal" href="#avoid-possible-confusion-about-interpreters-running-in-the-current-thread">Avoid possible confusion about interpreters running in the current thread</a></li>
|
||
<li><a class="reference internal" href="#clarify-running-vs-has-threads">Clarify “running” vs. “has threads”</a></li>
|
||
<li><a class="reference internal" href="#a-dunder-method-for-sharing">A Dunder Method For Sharing</a></li>
|
||
<li><a class="reference internal" href="#interpreter-call">Interpreter.call()</a></li>
|
||
<li><a class="reference internal" href="#interpreter-run-in-thread">Interpreter.run_in_thread()</a></li>
|
||
<li><a class="reference internal" href="#synchronization-primitives">Synchronization Primitives</a></li>
|
||
<li><a class="reference internal" href="#csp-library">CSP Library</a></li>
|
||
<li><a class="reference internal" href="#syntactic-support">Syntactic Support</a></li>
|
||
<li><a class="reference internal" href="#multiprocessing">Multiprocessing</a></li>
|
||
<li><a class="reference internal" href="#c-extension-opt-in-opt-out">C-extension opt-in/opt-out</a></li>
|
||
<li><a class="reference internal" href="#poisoning-channels">Poisoning channels</a></li>
|
||
<li><a class="reference internal" href="#resetting-main">Resetting __main__</a></li>
|
||
<li><a class="reference internal" href="#resetting-an-interpreter-s-state">Resetting an interpreter’s state</a></li>
|
||
<li><a class="reference internal" href="#copy-an-existing-interpreter-s-state">Copy an existing interpreter’s state</a></li>
|
||
<li><a class="reference internal" href="#shareable-file-descriptors-and-sockets">Shareable file descriptors and sockets</a></li>
|
||
<li><a class="reference internal" href="#integration-with-async">Integration with async</a></li>
|
||
<li><a class="reference internal" href="#support-for-iteration">Support for iteration</a></li>
|
||
<li><a class="reference internal" href="#channel-context-managers">Channel context managers</a></li>
|
||
<li><a class="reference internal" href="#pipes-and-queues">Pipes and Queues</a></li>
|
||
<li><a class="reference internal" href="#return-a-lock-from-send">Return a lock from send()</a></li>
|
||
<li><a class="reference internal" href="#support-prioritization-in-channels">Support prioritization in channels</a></li>
|
||
<li><a class="reference internal" href="#support-inheriting-settings-and-more">Support inheriting settings (and more?)</a></li>
|
||
<li><a class="reference internal" href="#make-exceptions-shareable">Make exceptions shareable</a></li>
|
||
<li><a class="reference internal" href="#make-everything-shareable-through-serialization">Make everything shareable through serialization</a></li>
|
||
<li><a class="reference internal" href="#make-runfailederror-cause-lazy">Make RunFailedError.__cause__ lazy</a></li>
|
||
<li><a class="reference internal" href="#return-a-value-from-interp-exec">Return a value from <code class="docutils literal notranslate"><span class="pre">interp.exec()</span></code></a></li>
|
||
<li><a class="reference internal" href="#add-a-shareable-synchronization-primitive">Add a shareable synchronization primitive</a></li>
|
||
<li><a class="reference internal" href="#propagate-systemexit-and-keyboardinterrupt-differently">Propagate SystemExit and KeyboardInterrupt Differently</a></li>
|
||
<li><a class="reference internal" href="#add-an-explicit-release-and-close-to-channel-end-classes">Add an explicit release() and close() to channel end classes</a></li>
|
||
<li><a class="reference internal" href="#add-sendchannel-send-buffer">Add SendChannel.send_buffer()</a></li>
|
||
<li><a class="reference internal" href="#auto-run-in-a-thread">Auto-run in a thread</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#explicit-channel-association">Explicit channel association</a></li>
|
||
<li><a class="reference internal" href="#add-an-api-based-on-pipes">Add an API based on pipes</a></li>
|
||
<li><a class="reference internal" href="#add-an-api-based-on-queues">Add an API based on queues</a></li>
|
||
<li><a class="reference internal" href="#enumerate">“enumerate”</a></li>
|
||
<li><a class="reference internal" href="#alternate-solutions-to-prevent-leaking-exceptions-across-interpreters">Alternate solutions to prevent leaking exceptions across interpreters</a></li>
|
||
<li><a class="reference internal" href="#always-associate-each-new-interpreter-with-its-own-thread">Always associate each new interpreter with its own thread</a></li>
|
||
<li><a class="reference internal" href="#only-associate-interpreters-upon-use">Only associate interpreters upon use</a></li>
|
||
<li><a class="reference internal" href="#allow-multiple-simultaneous-calls-to-interpreter-exec">Allow multiple simultaneous calls to Interpreter.exec()</a></li>
|
||
<li><a class="reference internal" href="#add-a-reraise-method-to-runfailederror">Add a “reraise” method to RunFailedError</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#implementation">Implementation</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0554.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |