436 lines
31 KiB
HTML
436 lines
31 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 709 – Inlined comprehensions | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0709/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 709 – Inlined comprehensions | peps.python.org'>
|
||
<meta property="og:description" content="Comprehensions are currently compiled as nested functions, which provides isolation of the comprehension’s iteration variable, but is inefficient at runtime. This PEP proposes to inline list, dictionary, and set comprehensions into the code where they a...">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0709/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Comprehensions are currently compiled as nested functions, which provides isolation of the comprehension’s iteration variable, but is inefficient at runtime. This PEP proposes to inline list, dictionary, and set comprehensions into the code where they a...">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 709</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 709 – Inlined comprehensions</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Carl Meyer <carl at oddbird.net></dd>
|
||
<dt class="field-even">Sponsor<span class="colon">:</span></dt>
|
||
<dd class="field-even">Guido van Rossum <guido at python.org></dd>
|
||
<dt class="field-odd">Discussions-To<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://discuss.python.org/t/pep-709-inlined-comprehensions/24240">Discourse thread</a></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Accepted and implementation complete, or no longer active">Final</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">24-Feb-2023</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.12</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="https://discuss.python.org/t/pep-709-inlined-comprehensions/24240" title="Discourse thread">25-Feb-2023</a></dd>
|
||
<dt class="field-odd">Resolution<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://discuss.python.org/t/pep-709-inlined-comprehensions/24240/36">Discourse message</a></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a></li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a><ul>
|
||
<li><a class="reference internal" href="#locals-includes-outer-variables">locals() includes outer variables</a></li>
|
||
<li><a class="reference internal" href="#no-comprehension-frame-in-tracebacks">No comprehension frame in tracebacks</a></li>
|
||
<li><a class="reference internal" href="#tracing-profiling-will-no-longer-show-a-call-return-for-the-comprehension">Tracing/profiling will no longer show a call/return for the comprehension</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#impact-on-other-python-implementations">Impact on other Python implementations</a></li>
|
||
<li><a class="reference internal" href="#how-to-teach-this">How to Teach This</a></li>
|
||
<li><a class="reference internal" href="#security-implications">Security Implications</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#more-efficient-comprehension-calling-without-inlining">More efficient comprehension calling, without inlining</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>Comprehensions are currently compiled as nested functions, which provides
|
||
isolation of the comprehension’s iteration variable, but is inefficient at
|
||
runtime. This PEP proposes to inline list, dictionary, and set comprehensions
|
||
into the code where they are defined, and provide the expected isolation by
|
||
pushing/popping clashing locals on the stack. This change makes comprehensions
|
||
much faster: up to 2x faster for a microbenchmark of a comprehension alone,
|
||
translating to an 11% speedup for one sample benchmark derived from real-world
|
||
code that makes heavy use of comprehensions in the context of doing actual work.</p>
|
||
</section>
|
||
<section id="motivation">
|
||
<h2><a class="toc-backref" href="#motivation" role="doc-backlink">Motivation</a></h2>
|
||
<p>Comprehensions are a popular and widely-used feature of the Python language.
|
||
The nested-function compilation of comprehensions optimizes for compiler
|
||
simplicity at the expense of performance of user code. It is possible to
|
||
provide near-identical semantics (see <a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a>) with much
|
||
better runtime performance for all users of comprehensions, with only a small
|
||
increase in compiler complexity.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Inlining is a common compiler optimization in many languages. Generalized
|
||
inlining of function calls at compile time in Python is near-impossible, since
|
||
call targets may be patched at runtime. Comprehensions are a special case,
|
||
where we have a call target known statically in the compiler that can neither
|
||
be patched (barring undocumented and unsupported fiddling with bytecode
|
||
directly) nor escape.</p>
|
||
<p>Inlining also permits other compiler optimizations of bytecode to be more
|
||
effective, because they can now “see through” the comprehension bytecode,
|
||
instead of it being an opaque call.</p>
|
||
<p>Normally a performance improvement would not require a PEP. In this case, the
|
||
simplest and most efficient implementation results in some user-visible effects,
|
||
so this is not just a performance improvement, it is a (small) change to the
|
||
language.</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<p>Given a simple comprehension:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">f</span><span class="p">(</span><span class="n">lst</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">lst</span><span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The compiler currently emits the following bytecode for the function <code class="docutils literal notranslate"><span class="pre">f</span></code>:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>1 0 RESUME 0
|
||
|
||
2 2 LOAD_CONST 1 (<code object <listcomp> at 0x...)
|
||
4 MAKE_FUNCTION 0
|
||
6 LOAD_FAST 0 (lst)
|
||
8 GET_ITER
|
||
10 CALL 0
|
||
20 RETURN_VALUE
|
||
|
||
Disassembly of <code object <listcomp> at 0x...>:
|
||
2 0 RESUME 0
|
||
2 BUILD_LIST 0
|
||
4 LOAD_FAST 0 (.0)
|
||
>> 6 FOR_ITER 4 (to 18)
|
||
10 STORE_FAST 1 (x)
|
||
12 LOAD_FAST 1 (x)
|
||
14 LIST_APPEND 2
|
||
16 JUMP_BACKWARD 6 (to 6)
|
||
>> 18 END_FOR
|
||
20 RETURN_VALUE
|
||
</pre></div>
|
||
</div>
|
||
<p>The bytecode for the comprehension is in a separate code object. Each time
|
||
<code class="docutils literal notranslate"><span class="pre">f()</span></code> is called, a new single-use function object is allocated (by
|
||
<code class="docutils literal notranslate"><span class="pre">MAKE_FUNCTION</span></code>), called (allocating and then destroying a new frame on the
|
||
Python stack), and then immediately thrown away.</p>
|
||
<p>Under this PEP, the compiler will emit the following bytecode for <code class="docutils literal notranslate"><span class="pre">f()</span></code>
|
||
instead:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>1 0 RESUME 0
|
||
|
||
2 2 LOAD_FAST 0 (lst)
|
||
4 GET_ITER
|
||
6 LOAD_FAST_AND_CLEAR 1 (x)
|
||
8 SWAP 2
|
||
10 BUILD_LIST 0
|
||
12 SWAP 2
|
||
>> 14 FOR_ITER 4 (to 26)
|
||
18 STORE_FAST 1 (x)
|
||
20 LOAD_FAST 1 (x)
|
||
22 LIST_APPEND 2
|
||
24 JUMP_BACKWARD 6 (to 14)
|
||
>> 26 END_FOR
|
||
28 SWAP 2
|
||
30 STORE_FAST 1 (x)
|
||
32 RETURN_VALUE
|
||
</pre></div>
|
||
</div>
|
||
<p>There is no longer a separate code object, nor creation of a single-use function
|
||
object, nor any need to create and destroy a Python frame.</p>
|
||
<p>Isolation of the <code class="docutils literal notranslate"><span class="pre">x</span></code> iteration variable is achieved by the combination of the
|
||
new <code class="docutils literal notranslate"><span class="pre">LOAD_FAST_AND_CLEAR</span></code> opcode at offset <code class="docutils literal notranslate"><span class="pre">6</span></code>, which saves any outer value
|
||
of <code class="docutils literal notranslate"><span class="pre">x</span></code> on the stack before running the comprehension, and <code class="docutils literal notranslate"><span class="pre">30</span> <span class="pre">STORE_FAST</span></code>,
|
||
which restores the outer value of <code class="docutils literal notranslate"><span class="pre">x</span></code> (if any) after running the
|
||
comprehension.</p>
|
||
<p>If the comprehension accesses variables from the outer scope, inlining avoids
|
||
the need to place these variables in a cell, allowing the comprehension (and all
|
||
other code in the outer function) to access them as normal fast locals instead.
|
||
This provides further performance gains.</p>
|
||
<p>In some cases, the comprehension iteration variable may be a global or cellvar
|
||
or freevar, rather than a simple function local, in the outer scope. In these
|
||
cases, the compiler also internally pushes and pops the scope information for
|
||
the variable when entering/leaving the comprehension, so that semantics are
|
||
maintained. For example, if the variable is a global outside the comprehension,
|
||
<code class="docutils literal notranslate"><span class="pre">LOAD_GLOBAL</span></code> will still be used where it is referenced outside the
|
||
comprehension, but <code class="docutils literal notranslate"><span class="pre">LOAD_FAST</span></code> / <code class="docutils literal notranslate"><span class="pre">STORE_FAST</span></code> will be used within the
|
||
comprehension. If it is a cellvar/freevar outside the comprehension, the
|
||
<code class="docutils literal notranslate"><span class="pre">LOAD_FAST_AND_CLEAR</span></code> / <code class="docutils literal notranslate"><span class="pre">STORE_FAST</span></code> used to save/restore it do not change
|
||
(there is no <code class="docutils literal notranslate"><span class="pre">LOAD_DEREF_AND_CLEAR</span></code>), meaning that the entire cell (not just
|
||
the value within it) is saved/restored, so the comprehension does not write to
|
||
the outer cell.</p>
|
||
<p>Comprehensions occurring in module or class scope are also inlined. In this
|
||
case, the comprehension will introduce usage of fast-locals (<code class="docutils literal notranslate"><span class="pre">LOAD_FAST</span></code> /
|
||
<code class="docutils literal notranslate"><span class="pre">STORE_FAST</span></code>) for the comprehension iteration variable within the
|
||
comprehension only, in a scope where otherwise only <code class="docutils literal notranslate"><span class="pre">LOAD_NAME</span></code> /
|
||
<code class="docutils literal notranslate"><span class="pre">STORE_NAME</span></code> would be used, maintaining isolation.</p>
|
||
<p>In effect, comprehensions introduce a sub-scope where local variables are fully
|
||
isolated, but without the performance cost or stack frame entry of a call.</p>
|
||
<p>Generator expressions are currently not inlined in the reference implementation
|
||
of this PEP. In the future, some generator expressions may be inlined, where the
|
||
returned generator object does not leak.</p>
|
||
<p>Asynchronous comprehensions are inlined the same as synchronous ones; no special
|
||
handling is needed.</p>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards Compatibility</a></h2>
|
||
<p>Comprehension inlining will cause the following visible behavior changes. No
|
||
changes in the standard library or test suite were necessary to adapt to these
|
||
changes in the implementation, suggesting the impact in user code is likely to
|
||
be minimal.</p>
|
||
<p>Specialized tools depending on undocumented details of compiler bytecode output
|
||
may of course be affected in ways beyond the below, but these tools already must
|
||
adapt to bytecode changes in each Python version.</p>
|
||
<section id="locals-includes-outer-variables">
|
||
<h3><a class="toc-backref" href="#locals-includes-outer-variables" role="doc-backlink">locals() includes outer variables</a></h3>
|
||
<p>Calling <code class="docutils literal notranslate"><span class="pre">locals()</span></code> within a comprehension will include all locals of the
|
||
function containing the comprehension. E.g. given the following function:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">f</span><span class="p">(</span><span class="n">lst</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="p">[</span><span class="nb">locals</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">lst</span><span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Calling <code class="docutils literal notranslate"><span class="pre">f([1])</span></code> in current Python will return:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="s1">'.0'</span><span class="p">:</span> <span class="o"><</span><span class="n">list_iterator</span> <span class="nb">object</span> <span class="n">at</span> <span class="mh">0x7f8d37170460</span><span class="o">></span><span class="p">,</span> <span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>where <code class="docutils literal notranslate"><span class="pre">.0</span></code> is an internal implementation detail: the synthetic sole argument
|
||
to the comprehension “function”.</p>
|
||
<p>Under this PEP, it will instead return:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[{</span><span class="s1">'lst'</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="s1">'x'</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This now includes the outer <code class="docutils literal notranslate"><span class="pre">lst</span></code> variable as a local, and eliminates the
|
||
synthetic <code class="docutils literal notranslate"><span class="pre">.0</span></code>.</p>
|
||
</section>
|
||
<section id="no-comprehension-frame-in-tracebacks">
|
||
<h3><a class="toc-backref" href="#no-comprehension-frame-in-tracebacks" role="doc-backlink">No comprehension frame in tracebacks</a></h3>
|
||
<p>Under this PEP, a comprehension will no longer have its own dedicated frame in
|
||
a stack trace. For example, given this function:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">g</span><span class="p">():</span>
|
||
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"boom"</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">f</span><span class="p">():</span>
|
||
<span class="k">return</span> <span class="p">[</span><span class="n">g</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">1</span><span class="p">]]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Currently, calling <code class="docutils literal notranslate"><span class="pre">f()</span></code> results in the following traceback:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>Traceback (most recent call last):
|
||
File "<stdin>", line 1, in <module>
|
||
File "<stdin>", line 5, in f
|
||
File "<stdin>", line 5, in <listcomp>
|
||
File "<stdin>", line 2, in g
|
||
RuntimeError: boom
|
||
</pre></div>
|
||
</div>
|
||
<p>Note the dedicated frame for <code class="docutils literal notranslate"><span class="pre"><listcomp></span></code>.</p>
|
||
<p>Under this PEP, the traceback looks like this instead:</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>Traceback (most recent call last):
|
||
File "<stdin>", line 1, in <module>
|
||
File "<stdin>", line 5, in f
|
||
File "<stdin>", line 2, in g
|
||
RuntimeError: boom
|
||
</pre></div>
|
||
</div>
|
||
<p>There is no longer an extra frame for the list comprehension. The frame for the
|
||
<code class="docutils literal notranslate"><span class="pre">f</span></code> function has the correct line number for the comprehension, however, so
|
||
this simply makes the traceback more compact without losing any useful
|
||
information.</p>
|
||
<p>It is theoretically possible that code using warnings with the <code class="docutils literal notranslate"><span class="pre">stacklevel</span></code>
|
||
argument could observe a behavior change due to the frame stack change. In
|
||
practice, however, this seems unlikely. It would require a warning raised in
|
||
library code that is always called through a comprehension in that same
|
||
library, where the warning is using a <code class="docutils literal notranslate"><span class="pre">stacklevel</span></code> of 3+ to bypass the
|
||
comprehension and its containing function and point to a calling frame outside
|
||
the library. In such a scenario it would usually be simpler and more reliable
|
||
to raise the warning closer to the calling code and bypass fewer frames.</p>
|
||
</section>
|
||
<section id="tracing-profiling-will-no-longer-show-a-call-return-for-the-comprehension">
|
||
<h3><a class="toc-backref" href="#tracing-profiling-will-no-longer-show-a-call-return-for-the-comprehension" role="doc-backlink">Tracing/profiling will no longer show a call/return for the comprehension</a></h3>
|
||
<p>Naturally, since list/dict/set comprehensions will no longer be implemented as a
|
||
call to a nested function, tracing/profiling using <code class="docutils literal notranslate"><span class="pre">sys.settrace</span></code> or
|
||
<code class="docutils literal notranslate"><span class="pre">sys.setprofile</span></code> will also no longer reflect that a call and return have
|
||
occurred.</p>
|
||
</section>
|
||
</section>
|
||
<section id="impact-on-other-python-implementations">
|
||
<h2><a class="toc-backref" href="#impact-on-other-python-implementations" role="doc-backlink">Impact on other Python implementations</a></h2>
|
||
<p>Per comments from representatives of <a class="reference external" href="https://discuss.python.org/t/pep-709-inlined-comprehensions/24240/20">GraalPython</a> and
|
||
<a class="reference external" href="https://discuss.python.org/t/pep-709-inlined-comprehensions/24240/22">PyPy</a>,
|
||
they would likely feel the need to adapt to the observable behavior changes
|
||
here, given the likelihood that someone, at some point, will depend on them.
|
||
Thus, all else equal, fewer observable changes would be less work. But these
|
||
changes (at least in the case of GraalPython) should be manageable “without much
|
||
headache”.</p>
|
||
</section>
|
||
<section id="how-to-teach-this">
|
||
<h2><a class="toc-backref" href="#how-to-teach-this" role="doc-backlink">How to Teach This</a></h2>
|
||
<p>It is not intuitively obvious that comprehension syntax will or should result
|
||
in creation and call of a nested function. For new users not already accustomed
|
||
to the prior behavior, I suspect the new behavior in this PEP will be more
|
||
intuitive and require less explanation. (“Why is there a <code class="docutils literal notranslate"><span class="pre"><listcomp></span></code> line in
|
||
my traceback when I didn’t define any such function? What is this <code class="docutils literal notranslate"><span class="pre">.0</span></code>
|
||
variable I see in <code class="docutils literal notranslate"><span class="pre">locals()</span></code>?”)</p>
|
||
</section>
|
||
<section id="security-implications">
|
||
<h2><a class="toc-backref" href="#security-implications" role="doc-backlink">Security Implications</a></h2>
|
||
<p>None known.</p>
|
||
</section>
|
||
<section id="reference-implementation">
|
||
<h2><a class="toc-backref" href="#reference-implementation" role="doc-backlink">Reference Implementation</a></h2>
|
||
<p>This PEP has a reference implementation in the form of <a class="reference external" href="https://github.com/python/cpython/pull/101441">a PR against the CPython main
|
||
branch</a> which passes all tests.</p>
|
||
<p>The reference implementation performs the micro-benchmark <code class="docutils literal notranslate"><span class="pre">./python</span> <span class="pre">-m</span> <span class="pre">pyperf</span>
|
||
<span class="pre">timeit</span> <span class="pre">-s</span> <span class="pre">'l</span> <span class="pre">=</span> <span class="pre">[1]'</span> <span class="pre">'[x</span> <span class="pre">for</span> <span class="pre">x</span> <span class="pre">in</span> <span class="pre">l]'</span></code> 1.96x faster than the <code class="docutils literal notranslate"><span class="pre">main</span></code> branch (in a
|
||
build compiled with <code class="docutils literal notranslate"><span class="pre">--enable-optimizations</span></code>.)</p>
|
||
<p>The reference implementation performs the <code class="docutils literal notranslate"><span class="pre">comprehensions</span></code> benchmark in the
|
||
<a class="reference external" href="https://github.com/python/pyperformance">pyperformance</a> benchmark suite
|
||
(which is not a micro-benchmark of comprehensions alone, but tests
|
||
real-world-derived code doing realistic work using comprehensions) 11% faster
|
||
than <code class="docutils literal notranslate"><span class="pre">main</span></code> branch (again in optimized builds). Other benchmarks in
|
||
pyperformance (none of which use comprehensions heavily) don’t show any impact
|
||
outside the noise.</p>
|
||
<p>The implementation has no impact on non-comprehension code.</p>
|
||
</section>
|
||
<section id="rejected-ideas">
|
||
<h2><a class="toc-backref" href="#rejected-ideas" role="doc-backlink">Rejected Ideas</a></h2>
|
||
<section id="more-efficient-comprehension-calling-without-inlining">
|
||
<h3><a class="toc-backref" href="#more-efficient-comprehension-calling-without-inlining" role="doc-backlink">More efficient comprehension calling, without inlining</a></h3>
|
||
<p>An <a class="reference external" href="https://github.com/python/cpython/pull/101310">alternate approach</a>
|
||
introduces a new opcode for “calling” a comprehension in streamlined fashion
|
||
without the need to create a throwaway function object, but still creating a new
|
||
Python frame. This avoids all of the visible effects listed under <a class="reference internal" href="#backwards-compatibility">Backwards
|
||
Compatibility</a>, and provides roughly half of the performance benefit (1.5x
|
||
improvement on the microbenchmark, 4% improvement on <code class="docutils literal notranslate"><span class="pre">comprehensions</span></code>
|
||
benchmark in pyperformance.) It also requires adding a new pointer to the
|
||
<code class="docutils literal notranslate"><span class="pre">_PyInterpreterFrame</span></code> struct and a new <code class="docutils literal notranslate"><span class="pre">Py_INCREF</span></code> on each frame
|
||
construction, meaning (unlike this PEP) it has a (very small) performance cost
|
||
for all code. It also provides less scope for future optimizations.</p>
|
||
<p>This PEP takes the position that full inlining offers sufficient additional
|
||
performance to more than justify the behavior changes.</p>
|
||
</section>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document is placed in the public domain or under the
|
||
CC0-1.0-Universal license, whichever is more permissive.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0709.rst">https://github.com/python/peps/blob/main/peps/pep-0709.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0709.rst">2023-12-15 15:06:12 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#motivation">Motivation</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a></li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a><ul>
|
||
<li><a class="reference internal" href="#locals-includes-outer-variables">locals() includes outer variables</a></li>
|
||
<li><a class="reference internal" href="#no-comprehension-frame-in-tracebacks">No comprehension frame in tracebacks</a></li>
|
||
<li><a class="reference internal" href="#tracing-profiling-will-no-longer-show-a-call-return-for-the-comprehension">Tracing/profiling will no longer show a call/return for the comprehension</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#impact-on-other-python-implementations">Impact on other Python implementations</a></li>
|
||
<li><a class="reference internal" href="#how-to-teach-this">How to Teach This</a></li>
|
||
<li><a class="reference internal" href="#security-implications">Security Implications</a></li>
|
||
<li><a class="reference internal" href="#reference-implementation">Reference Implementation</a></li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#more-efficient-comprehension-calling-without-inlining">More efficient comprehension calling, without inlining</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0709.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |