406 lines
31 KiB
HTML
406 lines
31 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 289 – Generator Expressions | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0289/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 289 – Generator Expressions | peps.python.org'>
|
||
<meta property="og:description" content="This PEP introduces generator expressions as a high performance, memory efficient generalization of list comprehensions PEP 202 and generators PEP 255.">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0289/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="This PEP introduces generator expressions as a high performance, memory efficient generalization of list comprehensions PEP 202 and generators PEP 255.">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 289</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 289 – Generator Expressions</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Raymond Hettinger <python at rcn.com></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Accepted and implementation complete, or no longer active">Final</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">30-Jan-2002</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">2.4</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">22-Oct-2003</dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#bdfl-pronouncements">BDFL Pronouncements</a></li>
|
||
<li><a class="reference internal" href="#the-details">The Details</a></li>
|
||
<li><a class="reference internal" href="#early-binding-versus-late-binding">Early Binding versus Late Binding</a></li>
|
||
<li><a class="reference internal" href="#reduction-functions">Reduction Functions</a></li>
|
||
<li><a class="reference internal" href="#acknowledgements">Acknowledgements</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP introduces generator expressions as a high performance,
|
||
memory efficient generalization of list comprehensions <a class="pep reference internal" href="../pep-0202/" title="PEP 202 – List Comprehensions">PEP 202</a> and
|
||
generators <a class="pep reference internal" href="../pep-0255/" title="PEP 255 – Simple Generators">PEP 255</a>.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>Experience with list comprehensions has shown their widespread
|
||
utility throughout Python. However, many of the use cases do
|
||
not need to have a full list created in memory. Instead, they
|
||
only need to iterate over the elements one at a time.</p>
|
||
<p>For instance, the following summation code will build a full list of
|
||
squares in memory, iterate over those values, and, when the reference
|
||
is no longer needed, delete the list:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">sum</span><span class="p">([</span><span class="n">x</span><span class="o">*</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">)])</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Memory is conserved by using a generator expression instead:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">sum</span><span class="p">(</span><span class="n">x</span><span class="o">*</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Similar benefits are conferred on constructors for container objects:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">s</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">word</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">page</span> <span class="k">for</span> <span class="n">word</span> <span class="ow">in</span> <span class="n">line</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>
|
||
<span class="n">d</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span> <span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">func</span><span class="p">(</span><span class="n">k</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">keylist</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Generator expressions are especially useful with functions like sum(),
|
||
min(), and max() that reduce an iterable input to a single value:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">max</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">line</span><span class="p">)</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">file</span> <span class="k">if</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Generator expressions also address some examples of functionals coded
|
||
with lambda:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">,</span> <span class="n">a</span><span class="p">:</span> <span class="n">s</span> <span class="o">+</span> <span class="n">a</span><span class="o">.</span><span class="n">myattr</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">,</span> <span class="n">a</span><span class="p">:</span> <span class="n">s</span> <span class="o">+</span> <span class="n">a</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span> <span class="n">data</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>These simplify to:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">sum</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">myattr</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">data</span><span class="p">)</span>
|
||
<span class="nb">sum</span><span class="p">(</span><span class="n">a</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">data</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>List comprehensions greatly reduced the need for filter() and map().
|
||
Likewise, generator expressions are expected to minimize the need
|
||
for itertools.ifilter() and itertools.imap(). In contrast, the
|
||
utility of other itertools will be enhanced by generator expressions:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">dotproduct</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="n">x</span><span class="o">*</span><span class="n">y</span> <span class="k">for</span> <span class="n">x</span><span class="p">,</span><span class="n">y</span> <span class="ow">in</span> <span class="n">itertools</span><span class="o">.</span><span class="n">izip</span><span class="p">(</span><span class="n">x_vector</span><span class="p">,</span> <span class="n">y_vector</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Having a syntax similar to list comprehensions also makes it easy to
|
||
convert existing code into a generator expression when scaling up
|
||
application.</p>
|
||
<p>Early timings showed that generators had a significant performance
|
||
advantage over list comprehensions. However, the latter were highly
|
||
optimized for Py2.4 and now the performance is roughly comparable
|
||
for small to mid-sized data sets. As the data volumes grow larger,
|
||
generator expressions tend to perform better because they do not
|
||
exhaust cache memory and they allow Python to re-use objects between
|
||
iterations.</p>
|
||
</section>
|
||
<section id="bdfl-pronouncements">
|
||
<h2><a class="toc-backref" href="#bdfl-pronouncements" role="doc-backlink">BDFL Pronouncements</a></h2>
|
||
<p>This PEP is ACCEPTED for Py2.4.</p>
|
||
</section>
|
||
<section id="the-details">
|
||
<h2><a class="toc-backref" href="#the-details" role="doc-backlink">The Details</a></h2>
|
||
<p>(None of this is exact enough in the eye of a reader from Mars, but I
|
||
hope the examples convey the intention well enough for a discussion in
|
||
c.l.py. The Python Reference Manual should contain a 100% exact
|
||
semantic and syntactic specification.)</p>
|
||
<ol class="arabic">
|
||
<li>The semantics of a generator expression are equivalent to creating
|
||
an anonymous generator function and calling it. For example:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">g</span> <span class="o">=</span> <span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
|
||
<span class="nb">print</span> <span class="n">g</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>is equivalent to:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">__gen</span><span class="p">(</span><span class="n">exp</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">exp</span><span class="p">:</span>
|
||
<span class="k">yield</span> <span class="n">x</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">g</span> <span class="o">=</span> <span class="n">__gen</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">)))</span>
|
||
<span class="nb">print</span> <span class="n">g</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Only the outermost for-expression is evaluated immediately, the other
|
||
expressions are deferred until the generator is run:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">g</span> <span class="o">=</span> <span class="p">(</span><span class="n">tgtexp</span> <span class="k">for</span> <span class="n">var1</span> <span class="ow">in</span> <span class="n">exp1</span> <span class="k">if</span> <span class="n">exp2</span> <span class="k">for</span> <span class="n">var2</span> <span class="ow">in</span> <span class="n">exp3</span> <span class="k">if</span> <span class="n">exp4</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>is equivalent to:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">__gen</span><span class="p">(</span><span class="n">bound_exp</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">var1</span> <span class="ow">in</span> <span class="n">bound_exp</span><span class="p">:</span>
|
||
<span class="k">if</span> <span class="n">exp2</span><span class="p">:</span>
|
||
<span class="k">for</span> <span class="n">var2</span> <span class="ow">in</span> <span class="n">exp3</span><span class="p">:</span>
|
||
<span class="k">if</span> <span class="n">exp4</span><span class="p">:</span>
|
||
<span class="k">yield</span> <span class="n">tgtexp</span>
|
||
<span class="n">g</span> <span class="o">=</span> <span class="n">__gen</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">exp1</span><span class="p">))</span>
|
||
<span class="k">del</span> <span class="n">__gen</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li>The syntax requires that a generator expression always needs to be
|
||
directly inside a set of parentheses and cannot have a comma on
|
||
either side. With reference to the file Grammar/Grammar in CVS,
|
||
two rules change:<ol class="loweralpha">
|
||
<li>The rule:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">atom</span><span class="p">:</span> <span class="s1">'('</span> <span class="p">[</span><span class="n">testlist</span><span class="p">]</span> <span class="s1">')'</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>changes to:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">atom</span><span class="p">:</span> <span class="s1">'('</span> <span class="p">[</span><span class="n">testlist_gexp</span><span class="p">]</span> <span class="s1">')'</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>where testlist_gexp is almost the same as listmaker, but only
|
||
allows a single test after ‘for’ … ‘in’:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">testlist_gexp</span><span class="p">:</span> <span class="n">test</span> <span class="p">(</span> <span class="n">gen_for</span> <span class="o">|</span> <span class="p">(</span><span class="s1">','</span> <span class="n">test</span><span class="p">)</span><span class="o">*</span> <span class="p">[</span><span class="s1">','</span><span class="p">]</span> <span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li>The rule for arglist needs similar changes.</li>
|
||
</ol>
|
||
<p>This means that you can write:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">sum</span><span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>but you would have to write:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">reduce</span><span class="p">(</span><span class="n">operator</span><span class="o">.</span><span class="n">add</span><span class="p">,</span> <span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">)))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>and also:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">g</span> <span class="o">=</span> <span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>i.e. if a function call has a single positional argument, it can be
|
||
a generator expression without extra parentheses, but in all other
|
||
cases you have to parenthesize it.</p>
|
||
<p>The exact details were checked in to Grammar/Grammar version 1.49.</p>
|
||
</li>
|
||
<li>The loop variable (if it is a simple variable or a tuple of simple
|
||
variables) is not exposed to the surrounding function. This
|
||
facilitates the implementation and makes typical use cases more
|
||
reliable. In some future version of Python, list comprehensions
|
||
will also hide the induction variable from the surrounding code
|
||
(and, in Py2.4, warnings will be issued for code accessing the
|
||
induction variable).<p>For example:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">x</span> <span class="o">=</span> <span class="s2">"hello"</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="s2">"abc"</span><span class="p">)</span>
|
||
<span class="nb">print</span> <span class="n">x</span> <span class="c1"># prints "hello", not "c"</span>
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li>List comprehensions will remain unchanged. For example:<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">S</span><span class="p">]</span> <span class="c1"># This is a list comprehension.</span>
|
||
<span class="p">[(</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">S</span><span class="p">)]</span> <span class="c1"># This is a list containing one generator</span>
|
||
<span class="c1"># expression.</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Unfortunately, there is currently a slight syntactic difference.
|
||
The expression:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>is legal, meaning:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)]</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>But generator expressions will not allow the former version:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">(</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>is illegal.</p>
|
||
<p>The former list comprehension syntax will become illegal in Python
|
||
3.0, and should be deprecated in Python 2.4 and beyond.</p>
|
||
<p>List comprehensions also “leak” their loop variable into the
|
||
surrounding scope. This will also change in Python 3.0, so that
|
||
the semantic definition of a list comprehension in Python 3.0 will
|
||
be equivalent to list(<generator expression>). Python 2.4 and
|
||
beyond should issue a deprecation warning if a list comprehension’s
|
||
loop variable has the same name as a variable used in the
|
||
immediately surrounding scope.</p>
|
||
</li>
|
||
</ol>
|
||
</section>
|
||
<section id="early-binding-versus-late-binding">
|
||
<h2><a class="toc-backref" href="#early-binding-versus-late-binding" role="doc-backlink">Early Binding versus Late Binding</a></h2>
|
||
<p>After much discussion, it was decided that the first (outermost)
|
||
for-expression should be evaluated immediately and that the remaining
|
||
expressions be evaluated when the generator is executed.</p>
|
||
<p>Asked to summarize the reasoning for binding the first expression,
|
||
Guido offered <a class="footnote-reference brackets" href="#id5" id="id1">[1]</a>:</p>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>Consider sum(x for x in foo()). Now suppose there's a bug in foo()
|
||
that raises an exception, and a bug in sum() that raises an
|
||
exception before it starts iterating over its argument. Which
|
||
exception would you expect to see? I'd be surprised if the one in
|
||
sum() was raised rather the one in foo(), since the call to foo()
|
||
is part of the argument to sum(), and I expect arguments to be
|
||
processed before the function is called.
|
||
|
||
OTOH, in sum(bar(x) for x in foo()), where sum() and foo()
|
||
are bugfree, but bar() raises an exception, we have no choice but
|
||
to delay the call to bar() until sum() starts iterating -- that's
|
||
part of the contract of generators. (They do nothing until their
|
||
next() method is first called.)
|
||
</pre></div>
|
||
</div>
|
||
<p>Various use cases were proposed for binding all free variables when
|
||
the generator is defined. And some proponents felt that the resulting
|
||
expressions would be easier to understand and debug if bound immediately.</p>
|
||
<p>However, Python takes a late binding approach to lambda expressions and
|
||
has no precedent for automatic, early binding. It was felt that
|
||
introducing a new paradigm would unnecessarily introduce complexity.</p>
|
||
<p>After exploring many possibilities, a consensus emerged that binding
|
||
issues were hard to understand and that users should be strongly
|
||
encouraged to use generator expressions inside functions that consume
|
||
their arguments immediately. For more complex applications, full
|
||
generator definitions are always superior in terms of being obvious
|
||
about scope, lifetime, and binding <a class="footnote-reference brackets" href="#id6" id="id2">[2]</a>.</p>
|
||
</section>
|
||
<section id="reduction-functions">
|
||
<h2><a class="toc-backref" href="#reduction-functions" role="doc-backlink">Reduction Functions</a></h2>
|
||
<p>The utility of generator expressions is greatly enhanced when combined
|
||
with reduction functions like sum(), min(), and max(). The heapq
|
||
module in Python 2.4 includes two new reduction functions: nlargest()
|
||
and nsmallest(). Both work well with generator expressions and keep
|
||
no more than n items in memory at one time.</p>
|
||
</section>
|
||
<section id="acknowledgements">
|
||
<h2><a class="toc-backref" href="#acknowledgements" role="doc-backlink">Acknowledgements</a></h2>
|
||
<ul class="simple">
|
||
<li>Raymond Hettinger first proposed the idea of “generator
|
||
comprehensions” in January 2002.</li>
|
||
<li>Peter Norvig resurrected the discussion in his proposal for
|
||
Accumulation Displays.</li>
|
||
<li>Alex Martelli provided critical measurements that proved the
|
||
performance benefits of generator expressions. He also provided
|
||
strong arguments that they were a desirable thing to have.</li>
|
||
<li>Phillip Eby suggested “iterator expressions” as the name.</li>
|
||
<li>Subsequently, Tim Peters suggested the name “generator expressions”.</li>
|
||
<li>Armin Rigo, Tim Peters, Guido van Rossum, Samuele Pedroni,
|
||
Hye-Shik Chang and Raymond Hettinger teased out the issues surrounding
|
||
early versus late binding <a class="footnote-reference brackets" href="#id5" id="id3">[1]</a>.</li>
|
||
<li>Jiwon Seo single-handedly implemented various versions of the proposal
|
||
including the final version loaded into CVS. Along the way, there
|
||
were periodic code reviews by Hye-Shik Chang and Raymond Hettinger.
|
||
Guido van Rossum made the key design decisions after comments from
|
||
Armin Rigo and newsgroup discussions. Raymond Hettinger provided
|
||
the test suite, documentation, tutorial, and examples <a class="footnote-reference brackets" href="#id6" id="id4">[2]</a>.</li>
|
||
</ul>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id5" role="doc-footnote">
|
||
<dt class="label" id="id5">[1]<em> (<a href='#id1'>1</a>, <a href='#id3'>2</a>) </em></dt>
|
||
<dd>Discussion over the relative merits of early versus late binding
|
||
<a class="reference external" href="https://mail.python.org/pipermail/python-dev/2004-April/044555.html">https://mail.python.org/pipermail/python-dev/2004-April/044555.html</a></aside>
|
||
<aside class="footnote brackets" id="id6" role="doc-footnote">
|
||
<dt class="label" id="id6">[2]<em> (<a href='#id2'>1</a>, <a href='#id4'>2</a>) </em></dt>
|
||
<dd>Patch discussion and alternative patches on Source Forge
|
||
<a class="reference external" href="https://bugs.python.org/issue872326">https://bugs.python.org/issue872326</a></aside>
|
||
</aside>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0289.rst">https://github.com/python/peps/blob/main/peps/pep-0289.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0289.rst">2023-09-09 17:39:29 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#bdfl-pronouncements">BDFL Pronouncements</a></li>
|
||
<li><a class="reference internal" href="#the-details">The Details</a></li>
|
||
<li><a class="reference internal" href="#early-binding-versus-late-binding">Early Binding versus Late Binding</a></li>
|
||
<li><a class="reference internal" href="#reduction-functions">Reduction Functions</a></li>
|
||
<li><a class="reference internal" href="#acknowledgements">Acknowledgements</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0289.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |