768 lines
58 KiB
HTML
768 lines
58 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 670 – Convert macros to functions in the Python C API | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0670/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 670 – Convert macros to functions in the Python C API | peps.python.org'>
|
||
<meta property="og:description" content="Macros in the C API will be converted to static inline functions or regular functions. This will help avoid macro pitfalls in C/C++, and make the functions usable from other programming languages.">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0670/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="Macros in the C API will be converted to static inline functions or regular functions. This will help avoid macro pitfalls in C/C++, and make the functions usable from other programming languages.">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 670</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 670 – Convert macros to functions in the Python C API</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Erlend Egeberg Aasland <erlend at python.org>,
|
||
Victor Stinner <vstinner at python.org></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Accepted and implementation complete, or no longer active">Final</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP with a new feature for Python, implementation change for CPython or interoperability standard for the ecosystem">Standards Track</abbr></dd>
|
||
<dt class="field-even">Created<span class="colon">:</span></dt>
|
||
<dd class="field-even">19-Oct-2021</dd>
|
||
<dt class="field-odd">Python-Version<span class="colon">:</span></dt>
|
||
<dd class="field-odd">3.11</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/2GN646CGWGTO6ZHHU7JTA5XWDF4ULM77/" title="Python-Dev thread">20-Oct-2021</a>,
|
||
<a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/IJ3IBVY3JDPROKX55YNDT6XZTVTTPGOP/" title="Python-Dev thread">08-Feb-2022</a>,
|
||
<a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/VM6I3UHVMME6QRSUOYLK6N2OZHP454W6/" title="Python-Dev thread">22-Feb-2022</a></dd>
|
||
<dt class="field-odd">Resolution<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/QQFCJ7LR36RUZSC3WI6WZZMQVQ3ZI4MS/">Python-Dev thread</a></dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#convert-macros-to-static-inline-functions">Convert macros to static inline functions</a></li>
|
||
<li><a class="reference internal" href="#convert-static-inline-functions-to-regular-functions">Convert static inline functions to regular functions</a></li>
|
||
<li><a class="reference internal" href="#cast-pointer-arguments">Cast pointer arguments</a><ul>
|
||
<li><a class="reference internal" href="#avoid-the-cast-in-the-limited-c-api-version-3-11">Avoid the cast in the limited C API version 3.11</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#return-type-is-not-changed">Return type is not changed</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#examples-of-macro-pitfalls">Examples of Macro Pitfalls</a><ul>
|
||
<li><a class="reference internal" href="#duplication-of-side-effects">Duplication of side effects</a></li>
|
||
<li><a class="reference internal" href="#misnesting">Misnesting</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#examples-of-hard-to-read-macros">Examples of hard to read macros</a><ul>
|
||
<li><a class="reference internal" href="#pyobject-init">PyObject_INIT()</a></li>
|
||
<li><a class="reference internal" href="#py-newreference">_Py_NewReference()</a></li>
|
||
<li><a class="reference internal" href="#pyunicode-read-char">PyUnicode_READ_CHAR()</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#macros-converted-to-functions-since-python-3-8">Macros converted to functions since Python 3.8</a><ul>
|
||
<li><a class="reference internal" href="#macros-converted-to-static-inline-functions">Macros converted to static inline functions</a></li>
|
||
<li><a class="reference internal" href="#macros-converted-to-regular-functions">Macros converted to regular functions</a></li>
|
||
<li><a class="reference internal" href="#static-inline-functions-converted-to-regular-functions">Static inline functions converted to regular functions</a></li>
|
||
<li><a class="reference internal" href="#incompatible-changes">Incompatible changes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#performance-concerns-and-benchmarks">Performance concerns and benchmarks</a><ul>
|
||
<li><a class="reference internal" href="#static-inline-functions">Static inline functions</a></li>
|
||
<li><a class="reference internal" href="#debug-build">Debug build</a></li>
|
||
<li><a class="reference internal" href="#force-inlining">Force inlining</a></li>
|
||
<li><a class="reference internal" href="#disabling-inlining">Disabling inlining</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#keep-macros-but-fix-some-macro-issues">Keep macros, but fix some macro issues</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#post-history">Post History</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#version-history">Version History</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>Macros in the C API will be converted to static inline functions or
|
||
regular functions. This will help avoid macro pitfalls in C/C++, and
|
||
make the functions usable from other programming languages.</p>
|
||
<p>To avoid compiler warnings, function arguments of pointer types
|
||
will be cast to appropriate types using additional macros.
|
||
The cast will not be done in the limited C API version 3.11:
|
||
users who opt in to the new limited API may need to add casts to
|
||
the exact expected type.</p>
|
||
<p>To avoid introducing incompatible changes, macros which can be used as
|
||
l-value in an assignment will not be converted.</p>
|
||
</section>
|
||
<section id="rationale">
|
||
<h2><a class="toc-backref" href="#rationale" role="doc-backlink">Rationale</a></h2>
|
||
<p>The use of macros may have unintended adverse effects that are hard to
|
||
avoid, even for experienced C developers. Some issues have been known
|
||
for years, while others have been discovered recently in Python.
|
||
Working around macro pitfalls makes the macro code harder to read and
|
||
to maintain.</p>
|
||
<p>Converting macros to functions has multiple advantages:</p>
|
||
<ul>
|
||
<li>Functions don’t suffer from macro pitfalls, for example the following
|
||
ones described in <a class="reference external" href="https://gcc.gnu.org/onlinedocs/cpp/Macro-Pitfalls.html">GCC documentation</a>:<ul class="simple">
|
||
<li>Misnesting</li>
|
||
<li>Operator precedence problems</li>
|
||
<li>Swallowing the semicolon</li>
|
||
<li>Duplication of side effects</li>
|
||
<li>Self-referential macros</li>
|
||
<li>Argument prescan</li>
|
||
<li>Newlines in arguments</li>
|
||
</ul>
|
||
<p>Functions don’t need the following workarounds for macro
|
||
pitfalls, making them usually easier to read and to maintain than
|
||
similar macro code:</p>
|
||
<ul class="simple">
|
||
<li>Adding parentheses around arguments.</li>
|
||
<li>Using line continuation characters if the function is written on
|
||
multiple lines.</li>
|
||
<li>Adding commas to execute multiple expressions.</li>
|
||
<li>Using <code class="docutils literal notranslate"><span class="pre">do</span> <span class="pre">{</span> <span class="pre">...</span> <span class="pre">}</span> <span class="pre">while</span> <span class="pre">(0)</span></code> to write multiple statements.</li>
|
||
</ul>
|
||
</li>
|
||
<li>Argument types and the return type of functions are well defined.</li>
|
||
<li>Debuggers and profilers can retrieve the name of inlined functions.</li>
|
||
<li>Debuggers can put breakpoints on inlined functions.</li>
|
||
<li>Variables have a well-defined scope.</li>
|
||
</ul>
|
||
<p>Converting macros and static inline functions to regular functions makes
|
||
these regular functions accessible to projects which use Python but
|
||
cannot use macros and static inline functions.</p>
|
||
</section>
|
||
<section id="specification">
|
||
<h2><a class="toc-backref" href="#specification" role="doc-backlink">Specification</a></h2>
|
||
<section id="convert-macros-to-static-inline-functions">
|
||
<h3><a class="toc-backref" href="#convert-macros-to-static-inline-functions" role="doc-backlink">Convert macros to static inline functions</a></h3>
|
||
<p>Most macros will be converted to static inline functions.</p>
|
||
<p>The following macros will not be converted:</p>
|
||
<ul class="simple">
|
||
<li>Object-like macros (i.e. those which don’t need parentheses and
|
||
arguments). For example:<ul>
|
||
<li>Empty macros. Example: <code class="docutils literal notranslate"><span class="pre">#define</span> <span class="pre">Py_HAVE_CONDVAR</span></code>.</li>
|
||
<li>Macros only defining a value, even if a constant with a well defined
|
||
type would be better. Example: <code class="docutils literal notranslate"><span class="pre">#define</span> <span class="pre">METH_VARARGS</span> <span class="pre">0x0001</span></code>.</li>
|
||
</ul>
|
||
</li>
|
||
<li>Compatibility layer for different C compilers, C language extensions,
|
||
or recent C features.
|
||
Example: <code class="docutils literal notranslate"><span class="pre">Py_GCC_ATTRIBUTE()</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_ALWAYS_INLINE</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_MEMCPY()</span></code>.</li>
|
||
<li>Macros used for definitions rather than behavior.
|
||
Example: <code class="docutils literal notranslate"><span class="pre">PyAPI_FUNC</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_DEPRECATED</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_PYTHON_H</span></code>.</li>
|
||
<li>Macros that need C preprocessor features, like stringification and
|
||
concatenation. Example: <code class="docutils literal notranslate"><span class="pre">Py_STRINGIFY()</span></code>.</li>
|
||
<li>Macros which cannot be converted to functions. Examples:
|
||
<code class="docutils literal notranslate"><span class="pre">Py_BEGIN_ALLOW_THREADS</span></code> (contains an unpaired <code class="docutils literal notranslate"><span class="pre">}</span></code>), <code class="docutils literal notranslate"><span class="pre">Py_VISIT</span></code>
|
||
(relies on specific variable names), Py_RETURN_RICHCOMPARE (returns
|
||
from the calling function).</li>
|
||
<li>Macros which can be used as an l-value in assignments. This would be
|
||
an incompatible change and is out of the scope of this PEP.
|
||
Example: <code class="docutils literal notranslate"><span class="pre">PyBytes_AS_STRING()</span></code>.</li>
|
||
<li>Macros which have different return types depending on the code path
|
||
or arguments.</li>
|
||
</ul>
|
||
</section>
|
||
<section id="convert-static-inline-functions-to-regular-functions">
|
||
<h3><a class="toc-backref" href="#convert-static-inline-functions-to-regular-functions" role="doc-backlink">Convert static inline functions to regular functions</a></h3>
|
||
<p>Static inline functions in the public C API may be converted to regular
|
||
functions, but only if there is no measurable performance impact of
|
||
changing the function.
|
||
The performance impact should be measured with benchmarks.</p>
|
||
</section>
|
||
<section id="cast-pointer-arguments">
|
||
<h3><a class="toc-backref" href="#cast-pointer-arguments" role="doc-backlink">Cast pointer arguments</a></h3>
|
||
<p>Currently, most macros accepting pointers cast pointer arguments to
|
||
their expected types. For example, in Python 3.6, the <code class="docutils literal notranslate"><span class="pre">Py_TYPE()</span></code>
|
||
macro casts its argument to <code class="docutils literal notranslate"><span class="pre">PyObject*</span></code>:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">Py_TYPE()</span></code> macro accepts the <code class="docutils literal notranslate"><span class="pre">PyObject*</span></code> type, but also any
|
||
pointer types, such as <code class="docutils literal notranslate"><span class="pre">PyLongObject*</span></code> and <code class="docutils literal notranslate"><span class="pre">PyDictObject*</span></code>.</p>
|
||
<p>Functions are strongly typed, and can only accept one type of argument.</p>
|
||
<p>To avoid compiler errors and warnings in existing code, when a macro is
|
||
converted to a function and the macro casts at least one of its arguments
|
||
a new macro will be added to keep the cast. The new macro
|
||
and the function will have the same name.</p>
|
||
<p>Example with the <code class="docutils literal notranslate"><span class="pre">Py_TYPE()</span></code>
|
||
macro converted to a static inline function:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kr">inline</span><span class="w"> </span><span class="n">PyTypeObject</span><span class="o">*</span><span class="w"> </span><span class="nf">Py_TYPE</span><span class="p">(</span><span class="n">PyObject</span><span class="w"> </span><span class="o">*</span><span class="n">ob</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">ob</span><span class="o">-></span><span class="n">ob_type</span><span class="p">;</span>
|
||
<span class="p">}</span>
|
||
<span class="cp">#define Py_TYPE(ob) Py_TYPE((PyObject*)(ob))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The cast is kept for all pointer types, not only <code class="docutils literal notranslate"><span class="pre">PyObject*</span></code>.
|
||
This includes casts to <code class="docutils literal notranslate"><span class="pre">void*</span></code>: removing a cast to <code class="docutils literal notranslate"><span class="pre">void*</span></code> would emit
|
||
a new warning if the function is called with a <code class="docutils literal notranslate"><span class="pre">const</span> <span class="pre">void*</span></code> variable.
|
||
For example, the <code class="docutils literal notranslate"><span class="pre">PyUnicode_WRITE()</span></code> macro casts its <em>data</em> argument to
|
||
<code class="docutils literal notranslate"><span class="pre">void*</span></code>, and so it currently accepts <code class="docutils literal notranslate"><span class="pre">const</span> <span class="pre">void*</span></code> type, even though
|
||
it writes into <em>data</em>. This PEP will not change this.</p>
|
||
<section id="avoid-the-cast-in-the-limited-c-api-version-3-11">
|
||
<h4><a class="toc-backref" href="#avoid-the-cast-in-the-limited-c-api-version-3-11" role="doc-backlink">Avoid the cast in the limited C API version 3.11</a></h4>
|
||
<p>The casts will be excluded from the limited C API version 3.11 and newer.
|
||
When an API user opts into the new limited API, they must pass the expected
|
||
type or perform the cast.</p>
|
||
<p>As an example, <code class="docutils literal notranslate"><span class="pre">Py_TYPE()</span></code> will be defined like this:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kr">inline</span><span class="w"> </span><span class="n">PyTypeObject</span><span class="o">*</span><span class="w"> </span><span class="nf">Py_TYPE</span><span class="p">(</span><span class="n">PyObject</span><span class="w"> </span><span class="o">*</span><span class="n">ob</span><span class="p">)</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">ob</span><span class="o">-></span><span class="n">ob_type</span><span class="p">;</span>
|
||
<span class="p">}</span>
|
||
<span class="cp">#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000</span>
|
||
<span class="cp"># define Py_TYPE(ob) Py_TYPE((PyObject*)(ob))</span>
|
||
<span class="cp">#endif</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="return-type-is-not-changed">
|
||
<h3><a class="toc-backref" href="#return-type-is-not-changed" role="doc-backlink">Return type is not changed</a></h3>
|
||
<p>When a macro is converted to a function, its return type must not change
|
||
to prevent emitting new compiler warnings.</p>
|
||
<p>For example, Python 3.7 changed the return type of <code class="docutils literal notranslate"><span class="pre">PyUnicode_AsUTF8()</span></code>
|
||
from <code class="docutils literal notranslate"><span class="pre">char*</span></code> to <code class="docutils literal notranslate"><span class="pre">const</span> <span class="pre">char*</span></code> (<a class="reference external" href="https://github.com/python/cpython/commit/2a404b63d48d73bbaa007d89efb7a01048475acd">commit</a>).
|
||
The change emitted new compiler warnings when building C extensions
|
||
expecting <code class="docutils literal notranslate"><span class="pre">char*</span></code>. This PEP doesn’t change the return type to prevent
|
||
this issue.</p>
|
||
</section>
|
||
</section>
|
||
<section id="backwards-compatibility">
|
||
<h2><a class="toc-backref" href="#backwards-compatibility" role="doc-backlink">Backwards Compatibility</a></h2>
|
||
<p>The PEP is designed to avoid C API incompatible changes.</p>
|
||
<p>Only C extensions explicitly targeting the limited C API version 3.11
|
||
must now pass the expected types to functions: pointer arguments are no
|
||
longer cast to the expected types.</p>
|
||
<p>Function arguments of pointer types are still cast and return types are
|
||
not changed to prevent emitting new compiler warnings.</p>
|
||
<p>Macros which can be used as l-value in an assignment are not modified by
|
||
this PEP to avoid incompatible changes.</p>
|
||
</section>
|
||
<section id="examples-of-macro-pitfalls">
|
||
<h2><a class="toc-backref" href="#examples-of-macro-pitfalls" role="doc-backlink">Examples of Macro Pitfalls</a></h2>
|
||
<section id="duplication-of-side-effects">
|
||
<h3><a class="toc-backref" href="#duplication-of-side-effects" role="doc-backlink">Duplication of side effects</a></h3>
|
||
<p>Macros:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#define PySet_Check(ob) \</span>
|
||
<span class="cp"> (Py_IS_TYPE(ob, &PySet_Type) \</span>
|
||
<span class="cp"> || PyType_IsSubtype(Py_TYPE(ob), &PySet_Type))</span>
|
||
|
||
<span class="cp">#define Py_IS_NAN(X) ((X) != (X))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>If the <em>op</em> or the <em>X</em> argument has a side effect, the side effect is
|
||
duplicated: it executed twice by <code class="docutils literal notranslate"><span class="pre">PySet_Check()</span></code> and <code class="docutils literal notranslate"><span class="pre">Py_IS_NAN()</span></code>.</p>
|
||
<p>For example, the <code class="docutils literal notranslate"><span class="pre">pos++</span></code> argument in the
|
||
<code class="docutils literal notranslate"><span class="pre">PyUnicode_WRITE(kind,</span> <span class="pre">data,</span> <span class="pre">pos++,</span> <span class="pre">ch)</span></code> code has a side effect.
|
||
This code is safe because the <code class="docutils literal notranslate"><span class="pre">PyUnicode_WRITE()</span></code> macro only uses its
|
||
3rd argument once and so does not duplicate <code class="docutils literal notranslate"><span class="pre">pos++</span></code> side effect.</p>
|
||
</section>
|
||
<section id="misnesting">
|
||
<h3><a class="toc-backref" href="#misnesting" role="doc-backlink">Misnesting</a></h3>
|
||
<p>Example of the <a class="reference external" href="https://bugs.python.org/issue43181">bpo-43181: Python macros don’t shield arguments</a>. The <code class="docutils literal notranslate"><span class="pre">PyObject_TypeCheck()</span></code>
|
||
macro before it has been fixed:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#define PyObject_TypeCheck(ob, tp) \</span>
|
||
<span class="cp"> (Py_IS_TYPE(ob, tp) || PyType_IsSubtype(Py_TYPE(ob), (tp)))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>C++ usage example:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="n">PyObject_TypeCheck</span><span class="p">(</span><span class="n">ob</span><span class="p">,</span><span class="w"> </span><span class="n">U</span><span class="p">(</span><span class="n">f</span><span class="o"><</span><span class="n">a</span><span class="p">,</span><span class="n">b</span><span class="o">></span><span class="p">(</span><span class="n">c</span><span class="p">)))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The preprocessor first expands it:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="p">(</span><span class="n">Py_IS_TYPE</span><span class="p">(</span><span class="n">ob</span><span class="p">,</span><span class="w"> </span><span class="n">f</span><span class="o"><</span><span class="n">a</span><span class="p">,</span><span class="n">b</span><span class="o">></span><span class="p">(</span><span class="n">c</span><span class="p">))</span><span class="w"> </span><span class="o">||</span><span class="w"> </span><span class="p">...)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>C++ <code class="docutils literal notranslate"><span class="pre">"<"</span></code> and <code class="docutils literal notranslate"><span class="pre">">"</span></code> characters are not treated as brackets by the
|
||
preprocessor, so the <code class="docutils literal notranslate"><span class="pre">Py_IS_TYPE()</span></code> macro is invoked with 3 arguments:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">ob</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">f<a</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">b>(c)</span></code></li>
|
||
</ul>
|
||
<p>The compilation fails with an error on <code class="docutils literal notranslate"><span class="pre">Py_IS_TYPE()</span></code> which only takes
|
||
2 arguments.</p>
|
||
<p>The bug is that the <em>op</em> and <em>tp</em> arguments of <code class="docutils literal notranslate"><span class="pre">PyObject_TypeCheck()</span></code>
|
||
must be put between parentheses: replace <code class="docutils literal notranslate"><span class="pre">Py_IS_TYPE(ob,</span> <span class="pre">tp)</span></code> with
|
||
<code class="docutils literal notranslate"><span class="pre">Py_IS_TYPE((ob),</span> <span class="pre">(tp))</span></code>. In regular C code, these parentheses are
|
||
redundant, can be seen as a bug, and so are often forgotten when writing
|
||
macros.</p>
|
||
<p>To avoid Macro Pitfalls, the <code class="docutils literal notranslate"><span class="pre">PyObject_TypeCheck()</span></code> macro has been
|
||
converted to a static inline function:
|
||
<a class="reference external" href="https://github.com/python/cpython/commit/4bb2a1ebc569eee6f1b46ecef1965a26ae8cb76d">commit</a>.</p>
|
||
</section>
|
||
</section>
|
||
<section id="examples-of-hard-to-read-macros">
|
||
<h2><a class="toc-backref" href="#examples-of-hard-to-read-macros" role="doc-backlink">Examples of hard to read macros</a></h2>
|
||
<section id="pyobject-init">
|
||
<h3><a class="toc-backref" href="#pyobject-init" role="doc-backlink">PyObject_INIT()</a></h3>
|
||
<p>Example showing the usage of commas in a macro which has a return value.</p>
|
||
<p>Python 3.7 macro:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#define PyObject_INIT(op, typeobj) \</span>
|
||
<span class="cp"> ( Py_TYPE(op) = (typeobj), _Py_NewReference((PyObject *)(op)), (op) )</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Python 3.8 function (simplified code):</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kr">inline</span><span class="w"> </span><span class="n">PyObject</span><span class="o">*</span>
|
||
<span class="nf">_PyObject_INIT</span><span class="p">(</span><span class="n">PyObject</span><span class="w"> </span><span class="o">*</span><span class="n">op</span><span class="p">,</span><span class="w"> </span><span class="n">PyTypeObject</span><span class="w"> </span><span class="o">*</span><span class="n">typeobj</span><span class="p">)</span>
|
||
<span class="p">{</span>
|
||
<span class="w"> </span><span class="n">Py_TYPE</span><span class="p">(</span><span class="n">op</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">typeobj</span><span class="p">;</span>
|
||
<span class="w"> </span><span class="n">_Py_NewReference</span><span class="p">(</span><span class="n">op</span><span class="p">);</span>
|
||
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">op</span><span class="p">;</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="cp">#define PyObject_INIT(op, typeobj) \</span>
|
||
<span class="cp"> _PyObject_INIT(_PyObject_CAST(op), (typeobj))</span>
|
||
</pre></div>
|
||
</div>
|
||
<ul class="simple">
|
||
<li>The function doesn’t need the line continuation character <code class="docutils literal notranslate"><span class="pre">"\"</span></code>.</li>
|
||
<li>It has an explicit <code class="docutils literal notranslate"><span class="pre">"return</span> <span class="pre">op;"</span></code> rather than the surprising
|
||
<code class="docutils literal notranslate"><span class="pre">",</span> <span class="pre">(op)"</span></code> syntax at the end of the macro.</li>
|
||
<li>It uses short statements on multiple lines, rather than being written
|
||
as a single long line.</li>
|
||
<li>Inside the function, the <em>op</em> argument has the well defined type
|
||
<code class="docutils literal notranslate"><span class="pre">PyObject*</span></code> and so doesn’t need casts like <code class="docutils literal notranslate"><span class="pre">(PyObject</span> <span class="pre">*)(op)</span></code>.</li>
|
||
<li>Arguments don’t need to be put inside parentheses: use <code class="docutils literal notranslate"><span class="pre">typeobj</span></code>,
|
||
rather than <code class="docutils literal notranslate"><span class="pre">(typeobj)</span></code>.</li>
|
||
</ul>
|
||
</section>
|
||
<section id="py-newreference">
|
||
<h3><a class="toc-backref" href="#py-newreference" role="doc-backlink">_Py_NewReference()</a></h3>
|
||
<p>Example showing the usage of an <code class="docutils literal notranslate"><span class="pre">#ifdef</span></code> inside a macro.</p>
|
||
<p>Python 3.7 macro (simplified code):</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#ifdef COUNT_ALLOCS</span>
|
||
<span class="cp"># define _Py_INC_TPALLOCS(OP) inc_count(Py_TYPE(OP))</span>
|
||
<span class="cp"># define _Py_COUNT_ALLOCS_COMMA ,</span>
|
||
<span class="cp">#else</span>
|
||
<span class="cp"># define _Py_INC_TPALLOCS(OP)</span>
|
||
<span class="cp"># define _Py_COUNT_ALLOCS_COMMA</span>
|
||
<span class="cp">#endif </span><span class="cm">/* COUNT_ALLOCS */</span>
|
||
|
||
<span class="cp">#define _Py_NewReference(op) ( \</span>
|
||
<span class="cp"> _Py_INC_TPALLOCS(op) _Py_COUNT_ALLOCS_COMMA \</span>
|
||
<span class="cp"> Py_REFCNT(op) = 1)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Python 3.8 function (simplified code):</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kr">inline</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">_Py_NewReference</span><span class="p">(</span><span class="n">PyObject</span><span class="w"> </span><span class="o">*</span><span class="n">op</span><span class="p">)</span>
|
||
<span class="p">{</span>
|
||
<span class="w"> </span><span class="n">_Py_INC_TPALLOCS</span><span class="p">(</span><span class="n">op</span><span class="p">);</span>
|
||
<span class="w"> </span><span class="n">Py_REFCNT</span><span class="p">(</span><span class="n">op</span><span class="p">)</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="mi">1</span><span class="p">;</span>
|
||
<span class="p">}</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="pyunicode-read-char">
|
||
<h3><a class="toc-backref" href="#pyunicode-read-char" role="doc-backlink">PyUnicode_READ_CHAR()</a></h3>
|
||
<p>This macro reuses arguments, and possibly calls <code class="docutils literal notranslate"><span class="pre">PyUnicode_KIND</span></code> multiple
|
||
times:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#define PyUnicode_READ_CHAR(unicode, index) \</span>
|
||
<span class="cp">(assert(PyUnicode_Check(unicode)), \</span>
|
||
<span class="cp"> assert(PyUnicode_IS_READY(unicode)), \</span>
|
||
<span class="cp"> (Py_UCS4) \</span>
|
||
<span class="cp"> (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \</span>
|
||
<span class="cp"> ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \</span>
|
||
<span class="cp"> (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \</span>
|
||
<span class="cp"> ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \</span>
|
||
<span class="cp"> ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \</span>
|
||
<span class="cp"> ) \</span>
|
||
<span class="cp"> ))</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Possible implementation as a static inlined function:</p>
|
||
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="k">static</span><span class="w"> </span><span class="kr">inline</span><span class="w"> </span><span class="n">Py_UCS4</span>
|
||
<span class="nf">PyUnicode_READ_CHAR</span><span class="p">(</span><span class="n">PyObject</span><span class="w"> </span><span class="o">*</span><span class="n">unicode</span><span class="p">,</span><span class="w"> </span><span class="n">Py_ssize_t</span><span class="w"> </span><span class="n">index</span><span class="p">)</span>
|
||
<span class="p">{</span>
|
||
<span class="w"> </span><span class="n">assert</span><span class="p">(</span><span class="n">PyUnicode_Check</span><span class="p">(</span><span class="n">unicode</span><span class="p">));</span>
|
||
<span class="w"> </span><span class="n">assert</span><span class="p">(</span><span class="n">PyUnicode_IS_READY</span><span class="p">(</span><span class="n">unicode</span><span class="p">));</span>
|
||
|
||
<span class="w"> </span><span class="k">switch</span><span class="w"> </span><span class="p">(</span><span class="n">PyUnicode_KIND</span><span class="p">(</span><span class="n">unicode</span><span class="p">))</span><span class="w"> </span><span class="p">{</span>
|
||
<span class="w"> </span><span class="k">case</span><span class="w"> </span><span class="no">PyUnicode_1BYTE_KIND</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="p">(</span><span class="n">Py_UCS4</span><span class="p">)((</span><span class="k">const</span><span class="w"> </span><span class="n">Py_UCS1</span><span class="w"> </span><span class="o">*</span><span class="p">)(</span><span class="n">PyUnicode_DATA</span><span class="p">(</span><span class="n">unicode</span><span class="p">)))[</span><span class="n">index</span><span class="p">];</span>
|
||
<span class="w"> </span><span class="k">case</span><span class="w"> </span><span class="no">PyUnicode_2BYTE_KIND</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="p">(</span><span class="n">Py_UCS4</span><span class="p">)((</span><span class="k">const</span><span class="w"> </span><span class="n">Py_UCS2</span><span class="w"> </span><span class="o">*</span><span class="p">)(</span><span class="n">PyUnicode_DATA</span><span class="p">(</span><span class="n">unicode</span><span class="p">)))[</span><span class="n">index</span><span class="p">];</span>
|
||
<span class="w"> </span><span class="k">case</span><span class="w"> </span><span class="no">PyUnicode_4BYTE_KIND</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="k">default</span><span class="o">:</span>
|
||
<span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="p">(</span><span class="n">Py_UCS4</span><span class="p">)((</span><span class="k">const</span><span class="w"> </span><span class="n">Py_UCS4</span><span class="w"> </span><span class="o">*</span><span class="p">)(</span><span class="n">PyUnicode_DATA</span><span class="p">(</span><span class="n">unicode</span><span class="p">)))[</span><span class="n">index</span><span class="p">];</span>
|
||
<span class="w"> </span><span class="p">}</span>
|
||
<span class="p">}</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="macros-converted-to-functions-since-python-3-8">
|
||
<h2><a class="toc-backref" href="#macros-converted-to-functions-since-python-3-8" role="doc-backlink">Macros converted to functions since Python 3.8</a></h2>
|
||
<p>This is a list of macros already converted to functions between
|
||
Python 3.8 and Python 3.11.
|
||
Even though some converted macros (like <code class="docutils literal notranslate"><span class="pre">Py_INCREF()</span></code>) are very
|
||
commonly used by C extensions, these conversions did not significantly
|
||
impact Python performance and most of them didn’t break backward
|
||
compatibility.</p>
|
||
<section id="macros-converted-to-static-inline-functions">
|
||
<h3><a class="toc-backref" href="#macros-converted-to-static-inline-functions" role="doc-backlink">Macros converted to static inline functions</a></h3>
|
||
<p>Python 3.8:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_DECREF()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_INCREF()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_XDECREF()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_XINCREF()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_INIT()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_INIT_VAR()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">_PyObject_GC_UNTRACK()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">_Py_Dealloc()</span></code></li>
|
||
</ul>
|
||
</section>
|
||
<section id="macros-converted-to-regular-functions">
|
||
<h3><a class="toc-backref" href="#macros-converted-to-regular-functions" role="doc-backlink">Macros converted to regular functions</a></h3>
|
||
<p>Python 3.9:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyIndex_Check()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_CheckBuffer()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_GET_WEAKREFS_LISTPTR()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_IS_GC()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_NEW()</span></code>: alias to <code class="docutils literal notranslate"><span class="pre">PyObject_New()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_NEW_VAR()</span></code>: alias to <code class="docutils literal notranslate"><span class="pre">PyObjectVar_New()</span></code></li>
|
||
</ul>
|
||
<p>To avoid performance slowdown on Python built without LTO,
|
||
private static inline functions have been added to the internal C API:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">_PyIndex_Check()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">_PyObject_IS_GC()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">_PyType_HasFeature()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">_PyType_IS_GC()</span></code></li>
|
||
</ul>
|
||
</section>
|
||
<section id="static-inline-functions-converted-to-regular-functions">
|
||
<h3><a class="toc-backref" href="#static-inline-functions-converted-to-regular-functions" role="doc-backlink">Static inline functions converted to regular functions</a></h3>
|
||
<p>Python 3.11:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_CallOneArg()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_Vectorcall()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyVectorcall_Function()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">_PyObject_FastCall()</span></code></li>
|
||
</ul>
|
||
<p>To avoid performance slowdown on Python built without LTO, a
|
||
private static inline function has been added to the internal C API:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">_PyVectorcall_FunctionInline()</span></code></li>
|
||
</ul>
|
||
</section>
|
||
<section id="incompatible-changes">
|
||
<h3><a class="toc-backref" href="#incompatible-changes" role="doc-backlink">Incompatible changes</a></h3>
|
||
<p>While other converted macros didn’t break the backward compatibility,
|
||
there is an exception.</p>
|
||
<p>The 3 macros <code class="docutils literal notranslate"><span class="pre">Py_REFCNT()</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_TYPE()</span></code> and <code class="docutils literal notranslate"><span class="pre">Py_SIZE()</span></code> have been
|
||
converted to static inline functions in Python 3.10 and 3.11 to disallow
|
||
using them as l-value in assignment. It is an incompatible change made
|
||
on purpose: see <a class="reference external" href="https://bugs.python.org/issue39573">bpo-39573</a> for
|
||
the rationale.</p>
|
||
<p>This PEP does not propose converting macros which can be used as l-value
|
||
to avoid introducing new incompatible changes.</p>
|
||
</section>
|
||
</section>
|
||
<section id="performance-concerns-and-benchmarks">
|
||
<h2><a class="toc-backref" href="#performance-concerns-and-benchmarks" role="doc-backlink">Performance concerns and benchmarks</a></h2>
|
||
<p>There have been concerns that converting macros to functions can degrade
|
||
performance.</p>
|
||
<p>This section explains performance concerns and shows benchmark results
|
||
using <a class="reference external" href="https://github.com/python/cpython/pull/29728">PR 29728</a>, which
|
||
replaces the following static inline functions with macros:</p>
|
||
<ul class="simple">
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyObject_TypeCheck()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyType_Check()</span></code>, <code class="docutils literal notranslate"><span class="pre">PyType_CheckExact()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyType_HasFeature()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">PyVectorcall_NARGS()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_DECREF()</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_XDECREF()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_INCREF()</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_XINCREF()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_IS_TYPE()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_NewRef()</span></code></li>
|
||
<li><code class="docutils literal notranslate"><span class="pre">Py_REFCNT()</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_TYPE()</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_SIZE()</span></code></li>
|
||
</ul>
|
||
<p>The benchmarks were run on Fedora 35 (Linux) with GCC 11 on a laptop with 8
|
||
logical CPUs (4 physical CPU cores).</p>
|
||
<section id="static-inline-functions">
|
||
<h3><a class="toc-backref" href="#static-inline-functions" role="doc-backlink">Static inline functions</a></h3>
|
||
<p>First of all, converting macros to <em>static inline</em> functions has
|
||
negligible impact on performance: the measured differences are consistent
|
||
with noise due to unrelated factors.</p>
|
||
<p>Static inline functions are a new feature in the C99 standard. Modern C
|
||
compilers have efficient heuristics to decide if a function should be
|
||
inlined or not.</p>
|
||
<p>When a C compiler decides to not inline, there is likely a good reason.
|
||
For example, inlining would reuse a register which requires to
|
||
save/restore the register value on the stack and so increases the stack
|
||
memory usage, or be less efficient.</p>
|
||
<p>Benchmark of the <code class="docutils literal notranslate"><span class="pre">./python</span> <span class="pre">-m</span> <span class="pre">test</span> <span class="pre">-j5</span></code> command on Python built in
|
||
release mode with <code class="docutils literal notranslate"><span class="pre">gcc</span> <span class="pre">-O3</span></code>, LTO and PGO:</p>
|
||
<ul class="simple">
|
||
<li>Macros (PR 29728): 361 sec +- 1 sec</li>
|
||
<li>Static inline functions (reference): 361 sec +- 1 sec</li>
|
||
</ul>
|
||
<p>There is <strong>no significant performance difference</strong> between macros and
|
||
static inline functions when static inline functions <strong>are inlined</strong>.</p>
|
||
</section>
|
||
<section id="debug-build">
|
||
<h3><a class="toc-backref" href="#debug-build" role="doc-backlink">Debug build</a></h3>
|
||
<p>Performance in debug builds <em>can</em> suffer when macros are converted to
|
||
functions. This is compensated by better debuggability: debuggers can
|
||
retrieve function names, set breakpoints inside functions, etc.</p>
|
||
<p>On Windows, when Python is built in debug mode by Visual Studio, static
|
||
inline functions are not inlined.</p>
|
||
<p>On other platforms, <code class="docutils literal notranslate"><span class="pre">./configure</span> <span class="pre">--with-pydebug</span></code> uses the <code class="docutils literal notranslate"><span class="pre">-Og</span></code> compiler
|
||
option on compilers that support it (including GCC and LLVM Clang).
|
||
<code class="docutils literal notranslate"><span class="pre">-Og</span></code> means “optimize debugging experience”.
|
||
Otherwise, the <code class="docutils literal notranslate"><span class="pre">-O0</span></code> compiler option is used.
|
||
<code class="docutils literal notranslate"><span class="pre">-O0</span></code> means “disable most optimizations”.</p>
|
||
<p>With GCC 11, <code class="docutils literal notranslate"><span class="pre">gcc</span> <span class="pre">-Og</span></code> can inline static inline functions, whereas
|
||
<code class="docutils literal notranslate"><span class="pre">gcc</span> <span class="pre">-O0</span></code> does not inline static inline functions.</p>
|
||
<p>Benchmark of the <code class="docutils literal notranslate"><span class="pre">./python</span> <span class="pre">-m</span> <span class="pre">test</span> <span class="pre">-j10</span></code> command on Python built in
|
||
debug mode with <code class="docutils literal notranslate"><span class="pre">gcc</span> <span class="pre">-O0</span></code> (that is, compiler optimizations,
|
||
including inlining, are explicitly disabled):</p>
|
||
<ul class="simple">
|
||
<li>Macros (PR 29728): 345 sec ± 5 sec</li>
|
||
<li>Static inline functions (reference): 360 sec ± 6 sec</li>
|
||
</ul>
|
||
<p>Replacing macros with static inline functions makes Python
|
||
<strong>1.04x slower</strong> when the compiler <strong>does not inline</strong> static inline
|
||
functions.</p>
|
||
<p>Note that benchmarks should not be run on a Python debug build.
|
||
Moreover, using link-time optimization (LTO) and profile-guided optimization
|
||
(PGO) is recommended for best performance and reliable benchmarks.
|
||
PGO helps the compiler to decide if functions should be inlined or not.</p>
|
||
</section>
|
||
<section id="force-inlining">
|
||
<h3><a class="toc-backref" href="#force-inlining" role="doc-backlink">Force inlining</a></h3>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">Py_ALWAYS_INLINE</span></code> macro can be used to force inlining. This macro
|
||
uses <code class="docutils literal notranslate"><span class="pre">__attribute__((always_inline))</span></code> with GCC and Clang, and
|
||
<code class="docutils literal notranslate"><span class="pre">__forceinline</span></code> with MSC.</p>
|
||
<p>Previous attempts to use <code class="docutils literal notranslate"><span class="pre">Py_ALWAYS_INLINE</span></code> didn’t show any benefit, and were
|
||
abandoned. See for example <a class="reference external" href="https://bugs.python.org/issue45094">bpo-45094</a>
|
||
“Consider using <code class="docutils literal notranslate"><span class="pre">__forceinline</span></code> and <code class="docutils literal notranslate"><span class="pre">__attribute__((always_inline))</span></code> on
|
||
static inline functions (<code class="docutils literal notranslate"><span class="pre">Py_INCREF</span></code>, <code class="docutils literal notranslate"><span class="pre">Py_TYPE</span></code>) for debug build”.</p>
|
||
<p>When the <code class="docutils literal notranslate"><span class="pre">Py_INCREF()</span></code> macro was converted to a static inline
|
||
function in 2018 (<a class="reference external" href="https://github.com/python/cpython/commit/2aaf0c12041bcaadd7f2cc5a54450eefd7a6ff12">commit</a>),
|
||
it was decided not to force inlining. The machine code was analyzed with
|
||
multiple C compilers and compiler options, and <code class="docutils literal notranslate"><span class="pre">Py_INCREF()</span></code> was always
|
||
inlined without having to force inlining. The only case where it was not
|
||
inlined was the debug build. See discussion in <a class="reference external" href="https://bugs.python.org/issue35059">bpo-35059</a> “Convert <code class="docutils literal notranslate"><span class="pre">Py_INCREF()</span></code> and
|
||
<code class="docutils literal notranslate"><span class="pre">PyObject_INIT()</span></code> to inlined functions”.</p>
|
||
</section>
|
||
<section id="disabling-inlining">
|
||
<h3><a class="toc-backref" href="#disabling-inlining" role="doc-backlink">Disabling inlining</a></h3>
|
||
<p>On the other side, the <code class="docutils literal notranslate"><span class="pre">Py_NO_INLINE</span></code> macro can be used to disable
|
||
inlining. It can be used to reduce the stack memory usage, or to prevent
|
||
inlining on LTO+PGO builds, which generally inline code more aggressively:
|
||
see <a class="reference external" href="https://bugs.python.org/issue33720">bpo-33720</a>. The
|
||
<code class="docutils literal notranslate"><span class="pre">Py_NO_INLINE</span></code> macro uses <code class="docutils literal notranslate"><span class="pre">__attribute__</span> <span class="pre">((noinline))</span></code> with GCC and
|
||
Clang, and <code class="docutils literal notranslate"><span class="pre">__declspec(noinline)</span></code> with MSC.</p>
|
||
<p>This technique is available, though we currently don’t know a concrete
|
||
function for which it would be useful.
|
||
Note that with macros, it is not possible to disable inlining at all.</p>
|
||
</section>
|
||
</section>
|
||
<section id="rejected-ideas">
|
||
<h2><a class="toc-backref" href="#rejected-ideas" role="doc-backlink">Rejected Ideas</a></h2>
|
||
<section id="keep-macros-but-fix-some-macro-issues">
|
||
<h3><a class="toc-backref" href="#keep-macros-but-fix-some-macro-issues" role="doc-backlink">Keep macros, but fix some macro issues</a></h3>
|
||
<p>Macros are always “inlined” with any C compiler.</p>
|
||
<p>The duplication of side effects can be worked around in the caller of
|
||
the macro.</p>
|
||
<p>People using macros should be considered “consenting adults”. People who
|
||
feel unsafe with macros should simply not use them.</p>
|
||
<p>These ideas are rejected because macros <em>are</em> error prone, and it is too easy
|
||
to miss a macro pitfall when writing and reviewing macro code. Moreover, macros
|
||
are harder to read and maintain than functions.</p>
|
||
</section>
|
||
</section>
|
||
<section id="post-history">
|
||
<h2><a class="toc-backref" href="#post-history" role="doc-backlink">Post History</a></h2>
|
||
<p>python-dev mailing list threads:</p>
|
||
<ul class="simple">
|
||
<li><a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/VM6I3UHVMME6QRSUOYLK6N2OZHP454W6/">Version 2 of PEP 670 - Convert macros to functions in the Python C API</a>
|
||
(February 2022)</li>
|
||
<li><a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/message/IJ3IBVY3JDPROKX55YNDT6XZTVTTPGOP/">Steering Council reply to PEP 670 – Convert macros to
|
||
functions in the Python C API</a>
|
||
(February 2022)</li>
|
||
<li><a class="reference external" href="https://mail.python.org/archives/list/python-dev@python.org/thread/2GN646CGWGTO6ZHHU7JTA5XWDF4ULM77/">PEP 670: Convert macros to functions in the Python C API</a>
|
||
(October 2021)</li>
|
||
</ul>
|
||
</section>
|
||
<section id="references">
|
||
<h2><a class="toc-backref" href="#references" role="doc-backlink">References</a></h2>
|
||
<ul class="simple">
|
||
<li><a class="reference external" href="https://bugs.python.org/issue45490">bpo-45490</a>:
|
||
[C API] PEP 670: Convert macros to functions in the Python C API
|
||
(October 2021).</li>
|
||
<li><a class="reference external" href="https://discuss.python.org/t/what-to-do-with-unsafe-macros/7771">What to do with unsafe macros</a>
|
||
(March 2021).</li>
|
||
<li><a class="reference external" href="https://bugs.python.org/issue43502">bpo-43502</a>:
|
||
[C-API] Convert obvious unsafe macros to static inline functions
|
||
(March 2021).</li>
|
||
</ul>
|
||
</section>
|
||
<section id="version-history">
|
||
<h2><a class="toc-backref" href="#version-history" role="doc-backlink">Version History</a></h2>
|
||
<ul class="simple">
|
||
<li>Version 2:<ul>
|
||
<li>Stricter policy on not changing argument types and return type.</li>
|
||
<li>Better explain why pointer arguments require a cast to not emit new
|
||
compiler warnings.</li>
|
||
<li>Macros which can be used as l-values are no longer modified by the
|
||
PEP.</li>
|
||
<li>Macros having multiple return types are no longer modified by the
|
||
PEP.</li>
|
||
<li>Limited C API version 3.11 no longer casts pointer arguments.</li>
|
||
<li>No longer remove return values of macros “which should not have a
|
||
return value”.</li>
|
||
<li>Add “Macros converted to functions since Python 3.8” section.</li>
|
||
<li>Add “Benchmark comparing macros and static inline functions”
|
||
section.</li>
|
||
</ul>
|
||
</li>
|
||
<li>Version 1: First public version</li>
|
||
</ul>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document is placed in the public domain or under the
|
||
CC0-1.0-Universal license, whichever is more permissive.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0670.rst">https://github.com/python/peps/blob/main/peps/pep-0670.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0670.rst">2023-10-04 23:18:07 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#rationale">Rationale</a></li>
|
||
<li><a class="reference internal" href="#specification">Specification</a><ul>
|
||
<li><a class="reference internal" href="#convert-macros-to-static-inline-functions">Convert macros to static inline functions</a></li>
|
||
<li><a class="reference internal" href="#convert-static-inline-functions-to-regular-functions">Convert static inline functions to regular functions</a></li>
|
||
<li><a class="reference internal" href="#cast-pointer-arguments">Cast pointer arguments</a><ul>
|
||
<li><a class="reference internal" href="#avoid-the-cast-in-the-limited-c-api-version-3-11">Avoid the cast in the limited C API version 3.11</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#return-type-is-not-changed">Return type is not changed</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#backwards-compatibility">Backwards Compatibility</a></li>
|
||
<li><a class="reference internal" href="#examples-of-macro-pitfalls">Examples of Macro Pitfalls</a><ul>
|
||
<li><a class="reference internal" href="#duplication-of-side-effects">Duplication of side effects</a></li>
|
||
<li><a class="reference internal" href="#misnesting">Misnesting</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#examples-of-hard-to-read-macros">Examples of hard to read macros</a><ul>
|
||
<li><a class="reference internal" href="#pyobject-init">PyObject_INIT()</a></li>
|
||
<li><a class="reference internal" href="#py-newreference">_Py_NewReference()</a></li>
|
||
<li><a class="reference internal" href="#pyunicode-read-char">PyUnicode_READ_CHAR()</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#macros-converted-to-functions-since-python-3-8">Macros converted to functions since Python 3.8</a><ul>
|
||
<li><a class="reference internal" href="#macros-converted-to-static-inline-functions">Macros converted to static inline functions</a></li>
|
||
<li><a class="reference internal" href="#macros-converted-to-regular-functions">Macros converted to regular functions</a></li>
|
||
<li><a class="reference internal" href="#static-inline-functions-converted-to-regular-functions">Static inline functions converted to regular functions</a></li>
|
||
<li><a class="reference internal" href="#incompatible-changes">Incompatible changes</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#performance-concerns-and-benchmarks">Performance concerns and benchmarks</a><ul>
|
||
<li><a class="reference internal" href="#static-inline-functions">Static inline functions</a></li>
|
||
<li><a class="reference internal" href="#debug-build">Debug build</a></li>
|
||
<li><a class="reference internal" href="#force-inlining">Force inlining</a></li>
|
||
<li><a class="reference internal" href="#disabling-inlining">Disabling inlining</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#rejected-ideas">Rejected Ideas</a><ul>
|
||
<li><a class="reference internal" href="#keep-macros-but-fix-some-macro-issues">Keep macros, but fix some macro issues</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#post-history">Post History</a></li>
|
||
<li><a class="reference internal" href="#references">References</a></li>
|
||
<li><a class="reference internal" href="#version-history">Version History</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0670.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |