755 lines
50 KiB
HTML
755 lines
50 KiB
HTML
|
||
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<meta name="color-scheme" content="light dark">
|
||
<title>PEP 462 – Core development workflow automation for CPython | peps.python.org</title>
|
||
<link rel="shortcut icon" href="../_static/py.png">
|
||
<link rel="canonical" href="https://peps.python.org/pep-0462/">
|
||
<link rel="stylesheet" href="../_static/style.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/mq.css" type="text/css">
|
||
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" media="(prefers-color-scheme: light)" id="pyg-light">
|
||
<link rel="stylesheet" href="../_static/pygments_dark.css" type="text/css" media="(prefers-color-scheme: dark)" id="pyg-dark">
|
||
<link rel="alternate" type="application/rss+xml" title="Latest PEPs" href="https://peps.python.org/peps.rss">
|
||
<meta property="og:title" content='PEP 462 – Core development workflow automation for CPython | peps.python.org'>
|
||
<meta property="og:description" content="This PEP proposes investing in automation of several of the tedious, time-consuming activities that are currently required for the core development team to incorporate changes into CPython. This proposal is intended to allow core developers to make more...">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:url" content="https://peps.python.org/pep-0462/">
|
||
<meta property="og:site_name" content="Python Enhancement Proposals (PEPs)">
|
||
<meta property="og:image" content="https://peps.python.org/_static/og-image.png">
|
||
<meta property="og:image:alt" content="Python PEPs">
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="description" content="This PEP proposes investing in automation of several of the tedious, time-consuming activities that are currently required for the core development team to incorporate changes into CPython. This proposal is intended to allow core developers to make more...">
|
||
<meta name="theme-color" content="#3776ab">
|
||
</head>
|
||
<body>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Following system colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="9"></circle>
|
||
<path d="M12 3v18m0-12l4.65-4.65M12 14.3l7.37-7.37M12 19.6l8.85-8.85"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected dark colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z"></path>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24" pointer-events="all">
|
||
<title>Selected light colour scheme</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
|
||
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
<script>
|
||
|
||
document.documentElement.dataset.colour_scheme = localStorage.getItem("colour_scheme") || "auto"
|
||
</script>
|
||
<section id="pep-page-section">
|
||
<header>
|
||
<h1>Python Enhancement Proposals</h1>
|
||
<ul class="breadcrumbs">
|
||
<li><a href="https://www.python.org/" title="The Python Programming Language">Python</a> » </li>
|
||
<li><a href="../pep-0000/">PEP Index</a> » </li>
|
||
<li>PEP 462</li>
|
||
</ul>
|
||
<button id="colour-scheme-cycler" onClick="setColourScheme(nextColourScheme())">
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg aria-hidden="true" class="colour-scheme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
<span class="visually-hidden">Toggle light / dark / auto colour theme</span>
|
||
</button>
|
||
</header>
|
||
<article>
|
||
<section id="pep-content">
|
||
<h1 class="page-title">PEP 462 – Core development workflow automation for CPython</h1>
|
||
<dl class="rfc2822 field-list simple">
|
||
<dt class="field-odd">Author<span class="colon">:</span></dt>
|
||
<dd class="field-odd">Alyssa Coghlan <ncoghlan at gmail.com></dd>
|
||
<dt class="field-even">Status<span class="colon">:</span></dt>
|
||
<dd class="field-even"><abbr title="Removed from consideration by sponsor or authors">Withdrawn</abbr></dd>
|
||
<dt class="field-odd">Type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><abbr title="Normative PEP describing or proposing a change to a Python community process, workflow or governance">Process</abbr></dd>
|
||
<dt class="field-even">Requires<span class="colon">:</span></dt>
|
||
<dd class="field-even"><a class="reference external" href="../pep-0474/">474</a></dd>
|
||
<dt class="field-odd">Created<span class="colon">:</span></dt>
|
||
<dd class="field-odd">23-Jan-2014</dd>
|
||
<dt class="field-even">Post-History<span class="colon">:</span></dt>
|
||
<dd class="field-even">25-Jan-2014, 27-Jan-2014, 01-Feb-2015</dd>
|
||
</dl>
|
||
<hr class="docutils" />
|
||
<section id="contents">
|
||
<details><summary>Table of Contents</summary><ul class="simple">
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#pep-withdrawal">PEP Withdrawal</a></li>
|
||
<li><a class="reference internal" href="#rationale-for-changes-to-the-core-development-workflow">Rationale for changes to the core development workflow</a></li>
|
||
<li><a class="reference internal" href="#current-tools">Current Tools</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a></li>
|
||
<li><a class="reference internal" href="#deferred-proposals">Deferred Proposals</a></li>
|
||
<li><a class="reference internal" href="#suggested-variants">Suggested Variants</a></li>
|
||
<li><a class="reference internal" href="#perceived-benefits">Perceived Benefits</a></li>
|
||
<li><a class="reference internal" href="#technical-challenges">Technical Challenges</a><ul>
|
||
<li><a class="reference internal" href="#kallithea-vs-gerrit">Kallithea vs Gerrit</a></li>
|
||
<li><a class="reference internal" href="#mercurial-vs-gerrit-git">Mercurial vs Gerrit/git</a></li>
|
||
<li><a class="reference internal" href="#buildbot-vs-jenkins">Buildbot vs Jenkins</a></li>
|
||
<li><a class="reference internal" href="#handling-of-maintenance-branches">Handling of maintenance branches</a></li>
|
||
<li><a class="reference internal" href="#handling-of-security-branches">Handling of security branches</a></li>
|
||
<li><a class="reference internal" href="#handling-of-news-file-updates">Handling of NEWS file updates</a></li>
|
||
<li><a class="reference internal" href="#stability-of-stable-buildbot-slaves">Stability of “stable” Buildbot slaves</a></li>
|
||
<li><a class="reference internal" href="#intermittent-test-failures">Intermittent test failures</a></li>
|
||
<li><a class="reference internal" href="#custom-mercurial-client-workflow-support">Custom Mercurial client workflow support</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#social-challenges">Social Challenges</a></li>
|
||
<li><a class="reference internal" href="#practical-challenges">Practical Challenges</a></li>
|
||
<li><a class="reference internal" href="#open-questions">Open Questions</a></li>
|
||
<li><a class="reference internal" href="#next-steps">Next Steps</a></li>
|
||
<li><a class="reference internal" href="#acknowledgements">Acknowledgements</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
</details></section>
|
||
<section id="abstract">
|
||
<h2><a class="toc-backref" href="#abstract" role="doc-backlink">Abstract</a></h2>
|
||
<p>This PEP proposes investing in automation of several of the tedious,
|
||
time-consuming activities that are currently required for the core development
|
||
team to incorporate changes into CPython. This proposal is intended to
|
||
allow core developers to make more effective use of the time they have
|
||
available to contribute to CPython, which should also result in an improved
|
||
experience for other contributors that are reliant on the core team to get
|
||
their changes incorporated.</p>
|
||
</section>
|
||
<section id="pep-withdrawal">
|
||
<h2><a class="toc-backref" href="#pep-withdrawal" role="doc-backlink">PEP Withdrawal</a></h2>
|
||
<p>This PEP has been <a class="reference external" href="https://mail.python.org/pipermail/core-workflow/2015-October/000227.html">withdrawn by the author</a>
|
||
in favour of the GitLab based proposal in <a class="pep reference internal" href="../pep-0507/" title="PEP 507 – Migrate CPython to Git and GitLab">PEP 507</a>.</p>
|
||
<p>If anyone else would like to take over championing this PEP, contact the
|
||
<a class="reference external" href="https://mail.python.org/mailman/listinfo/core-workflow">core-workflow mailing list</a></p>
|
||
</section>
|
||
<section id="rationale-for-changes-to-the-core-development-workflow">
|
||
<h2><a class="toc-backref" href="#rationale-for-changes-to-the-core-development-workflow" role="doc-backlink">Rationale for changes to the core development workflow</a></h2>
|
||
<p>The current core developer workflow to merge a new feature into CPython
|
||
on a POSIX system “works” as follows:</p>
|
||
<ol class="arabic simple">
|
||
<li>If applying a change submitted to bugs.python.org by another user, first
|
||
check they have signed the PSF Contributor Licensing Agreement. If not,
|
||
request that they sign one before continuing with merging the change.</li>
|
||
<li>Apply the change locally to a current checkout of the main CPython
|
||
repository (the change will typically have been discussed and reviewed
|
||
as a patch on bugs.python.org first, but this step is not currently
|
||
considered mandatory for changes originating directly from core
|
||
developers).</li>
|
||
<li>Run the test suite locally, at least <code class="docutils literal notranslate"><span class="pre">make</span> <span class="pre">test</span></code> or
|
||
<code class="docutils literal notranslate"><span class="pre">./python</span> <span class="pre">-m</span> <span class="pre">test</span></code> (depending on system specs, this takes a few
|
||
minutes in the default configuration, but substantially longer if all
|
||
optional resources, like external network access, are enabled).</li>
|
||
<li>Run <code class="docutils literal notranslate"><span class="pre">make</span> <span class="pre">patchcheck</span></code> to fix any whitespace issues and as a reminder
|
||
of other changes that may be needed (such as updating Misc/ACKS or
|
||
adding an entry to Misc/NEWS)</li>
|
||
<li>Commit the change and push it to the main repository. If hg indicates
|
||
this would create a new head in the remote repository, run
|
||
<code class="docutils literal notranslate"><span class="pre">hg</span> <span class="pre">pull</span> <span class="pre">--rebase</span></code> (or an equivalent). Theoretically, you should
|
||
rerun the tests at this point, but it’s <em>very</em> tempting to skip that
|
||
step.</li>
|
||
<li>After pushing, monitor the <a class="reference external" href="http://buildbot.python.org/all/waterfall?category=3.x.stable">stable buildbots</a>
|
||
for any new failures introduced by your change. In particular, developers
|
||
on POSIX systems will often break the Windows buildbots, and vice-versa.
|
||
Less commonly, developers on Linux or Mac OS X may break other POSIX
|
||
systems.</li>
|
||
</ol>
|
||
<p>The steps required on Windows are similar, but the exact commands used
|
||
will be different.</p>
|
||
<p>Rather than being simpler, the workflow for a bug fix is <em>more</em> complicated
|
||
than that for a new feature! New features have the advantage of only being
|
||
applied to the <code class="docutils literal notranslate"><span class="pre">default</span></code> branch, while bug fixes also need to be considered
|
||
for inclusion in maintenance branches.</p>
|
||
<ul class="simple">
|
||
<li>If a bug fix is applicable to Python 2.7, then it is also separately
|
||
applied to the 2.7 branch, which is maintained as an independent head
|
||
in Mercurial</li>
|
||
<li>If a bug fix is applicable to the current 3.x maintenance release, then
|
||
it is first applied to the maintenance branch and then merged forward
|
||
to the default branch. Both branches are pushed to hg.python.org at the
|
||
same time.</li>
|
||
</ul>
|
||
<p>Documentation patches are simpler than functional patches, but not
|
||
hugely so - the main benefit is only needing to check the docs build
|
||
successfully rather than running the test suite.</p>
|
||
<p>I would estimate that even when everything goes smoothly, it would still
|
||
take me at least 20-30 minutes to commit a bug fix patch that applies
|
||
cleanly. Given that it should be possible to automate several of these
|
||
tasks, I do not believe our current practices are making effective use
|
||
of scarce core developer resources.</p>
|
||
<p>There are many, many frustrations involved with this current workflow, and
|
||
they lead directly to some undesirable development practices.</p>
|
||
<ul class="simple">
|
||
<li>Much of this overhead is incurred on a per-patch applied basis. This
|
||
encourages large commits, rather than small isolated changes. The time
|
||
required to commit a 500 line feature is essentially the same as that
|
||
needed to commit a 1 line bug fix - the additional time needed for the
|
||
larger change appears in any preceding review rather than as part of the
|
||
commit process.</li>
|
||
<li>The additional overhead of working on applying bug fixes creates an
|
||
additional incentive to work on new features instead, and new features
|
||
are already <em>inherently</em> more interesting to work on - they don’t need
|
||
workflow difficulties giving them a helping hand!</li>
|
||
<li>Getting a preceding review on bugs.python.org is <em>additional</em> work,
|
||
creating an incentive to commit changes directly, increasing the reliance
|
||
on post-review on the python-checkins mailing list.</li>
|
||
<li>Patches on the tracker that are complete, correct and ready to merge may
|
||
still languish for extended periods awaiting a core developer with the
|
||
time to devote to getting it merged.</li>
|
||
<li>The risk of push races (especially when pushing a merged bug fix) creates
|
||
a temptation to skip doing full local test runs (especially after a push
|
||
race has already been encountered once), increasing the chance of
|
||
breaking the buildbots.</li>
|
||
<li>The buildbots are sometimes red for extended periods, introducing errors
|
||
into local test runs, and also meaning that they sometimes fail to serve
|
||
as a reliable indicator of whether or not a patch has introduced cross
|
||
platform issues.</li>
|
||
<li>Post-conference development sprints are a nightmare, as they collapse
|
||
into a mire of push races. It’s tempting to just leave patches on the
|
||
tracker until after the sprint is over and then try to clean them up
|
||
afterwards.</li>
|
||
</ul>
|
||
<p>There are also many, many opportunities for core developers to make
|
||
mistakes that inconvenience others, both in managing the Mercurial branches
|
||
and in breaking the buildbots without being in a position to fix them
|
||
promptly. This both makes the existing core development team cautious in
|
||
granting new developers commit access, as well as making those new
|
||
developers cautious about actually making use of their increased level of
|
||
access.</p>
|
||
<p>There are also some incidental annoyances (like keeping the NEWS file up to
|
||
date) that will also be necessarily addressed as part of this proposal.</p>
|
||
<p>One of the most critical resources of a volunteer-driven open source project
|
||
is the emotional energy of its contributors. The current approach to change
|
||
incorporation doesn’t score well on that front for anyone:</p>
|
||
<ul class="simple">
|
||
<li>For core developers, the branch wrangling for bug fixes is delicate and
|
||
easy to get wrong. Conflicts on the NEWS file and push races when
|
||
attempting to upload changes add to the irritation of something most of
|
||
us aren’t being paid to spend time on (and for those that are, contributing
|
||
to CPython is likely to be only one of our responsibilities). The time we
|
||
spend actually getting a change merged is time we’re not spending coding
|
||
additional changes, writing or updating documentation or reviewing
|
||
contributions from others.</li>
|
||
<li>Red buildbots make life difficult for other developers (since a local
|
||
test failure may <em>not</em> be due to anything that developer did), release
|
||
managers (since they may need to enlist assistance cleaning up test
|
||
failures prior to a release) and for the developers themselves (since
|
||
it creates significant pressure to fix any failures we inadvertently
|
||
introduce right <em>now</em>, rather than at a more convenient time, as well
|
||
as potentially making <code class="docutils literal notranslate"><span class="pre">hg</span> <span class="pre">bisect</span></code> more difficult to use if
|
||
<code class="docutils literal notranslate"><span class="pre">hg</span> <span class="pre">annotate</span></code> isn’t sufficient to identify the source of a new failure).</li>
|
||
<li>For other contributors, a core developer spending time actually getting
|
||
changes merged is a developer that isn’t reviewing and discussing patches
|
||
on the issue tracker or otherwise helping others to contribute effectively.
|
||
It is especially frustrating for contributors that are accustomed to the
|
||
simplicity of a developer just being able to hit “Merge” on a pull
|
||
request that has already been automatically tested in the project’s CI
|
||
system (which is a common workflow on sites like GitHub and BitBucket), or
|
||
where the post-review part of the merge process is fully automated (as is
|
||
the case for OpenStack).</li>
|
||
</ul>
|
||
</section>
|
||
<section id="current-tools">
|
||
<h2><a class="toc-backref" href="#current-tools" role="doc-backlink">Current Tools</a></h2>
|
||
<p>The following tools are currently used to manage various parts of the
|
||
CPython core development workflow.</p>
|
||
<ul class="simple">
|
||
<li>Mercurial (hg.python.org) for version control</li>
|
||
<li>Roundup (bugs.python.org) for issue tracking</li>
|
||
<li>Rietveld (also hosted on bugs.python.org) for code review</li>
|
||
<li>Buildbot (buildbot.python.org) for automated testing</li>
|
||
</ul>
|
||
<p>This proposal suggests replacing the use of Rietveld for code review with
|
||
the more full-featured Kallithea-based forge.python.org service proposed in
|
||
<a class="pep reference internal" href="../pep-0474/" title="PEP 474 – Creating forge.python.org">PEP 474</a>. Guido has indicated that the original Rietveld implementation was
|
||
primarily intended as a public demonstration application for Google App
|
||
Engine, and switching to Kallithea will address some of the issues with
|
||
identifying intended target branches that arise when working with patch files
|
||
on Roundup and the associated reviews in the integrated Rietveld instance.</p>
|
||
<p>It also suggests the addition of new tools in order to automate
|
||
additional parts of the workflow, as well as a critical review of the
|
||
remaining tools to see which, if any, may be candidates for replacement.</p>
|
||
</section>
|
||
<section id="proposal">
|
||
<h2><a class="toc-backref" href="#proposal" role="doc-backlink">Proposal</a></h2>
|
||
<p>The essence of this proposal is that CPython aim to adopt a “core reviewer”
|
||
development model, similar to that used by the OpenStack project.</p>
|
||
<p>The workflow problems experienced by the CPython core development team are
|
||
not unique. The OpenStack infrastructure team have come up with a well
|
||
designed automated workflow that is designed to ensure:</p>
|
||
<ul class="simple">
|
||
<li>once a patch has been reviewed, further developer involvement is needed
|
||
only if the automated tests fail prior to merging</li>
|
||
<li>patches never get merged without being tested relative to the current
|
||
state of the branch</li>
|
||
<li>the main development branch always stays green. Patches that do not pass
|
||
the automated tests do not get merged</li>
|
||
</ul>
|
||
<p>If a core developer wants to tweak a patch prior to merging, they download
|
||
it from the review tool, modify and <em>upload it back to the review tool</em>
|
||
rather than pushing it directly to the source code repository.</p>
|
||
<p>The core of this workflow is implemented using a tool called <a class="reference external" href="http://ci.openstack.org/zuul/">Zuul</a>, a
|
||
Python web service created specifically for the OpenStack project, but
|
||
deliberately designed with a plugin based trigger and action system to make
|
||
it easier to adapt to alternate code review systems, issue trackers and
|
||
CI systems. James Blair of the OpenStack infrastructure team provided
|
||
an <a class="reference external" href="https://www.youtube.com/watch?v=sLD9LHc1QFM">excellent overview of Zuul</a> at linux.conf.au 2014.</p>
|
||
<p>While Zuul handles several workflows for OpenStack, the specific one of
|
||
interest for this PEP is the “merge gating” workflow.</p>
|
||
<p>For this workflow, Zuul is configured to monitor the Gerrit code review
|
||
system for patches which have been marked as “Approved”. Once it sees
|
||
such a patch, Zuul takes it, and combines it into a queue of “candidate
|
||
merges”. It then creates a pipeline of test runs that execute in parallel in
|
||
Jenkins (in order to allow more than 24 commits a day when a full test run
|
||
takes the better part of an hour), and are merged as they pass (and as all
|
||
the candidate merges ahead of them in the queue pass). If a patch fails the
|
||
tests, Zuul takes it out of the queue, cancels any test runs after that patch in
|
||
the queue, and rebuilds the queue without the failing patch.</p>
|
||
<p>If a developer looks at a test which failed on merge and determines that it
|
||
was due to an intermittent failure, they can then resubmit the patch for
|
||
another attempt at merging.</p>
|
||
<p>To adapt this process to CPython, it should be feasible to have Zuul monitor
|
||
Kallithea for approved pull requests (which may require a feature addition in
|
||
Kallithea), submit them to Buildbot for testing on the stable buildbots, and
|
||
then merge the changes appropriately in Mercurial. This idea poses a few
|
||
technical challenges, which have their own section below.</p>
|
||
<p>For CPython, I don’t believe we will need to take advantage of Zuul’s
|
||
ability to execute tests in parallel (certainly not in the initial
|
||
iteration - if we get to a point where serial testing of patches by the
|
||
merge gating system is our primary bottleneck rather than having the
|
||
people we need in order to be able to review and approve patches, then
|
||
that will be a very good day).</p>
|
||
<p>However, the merge queue itself is a very powerful concept that should
|
||
directly address several of the issues described in the Rationale above.</p>
|
||
</section>
|
||
<section id="deferred-proposals">
|
||
<h2><a class="toc-backref" href="#deferred-proposals" role="doc-backlink">Deferred Proposals</a></h2>
|
||
<p>The OpenStack team also use Zuul to coordinate several other activities:</p>
|
||
<ul class="simple">
|
||
<li>Running preliminary “check” tests against patches posted to Gerrit.</li>
|
||
<li>Creation of updated release artefacts and republishing documentation when
|
||
changes are merged</li>
|
||
<li>The <a class="reference external" href="http://status.openstack.org/elastic-recheck/">Elastic recheck</a> feature that uses ElasticSearch in conjunction with
|
||
a spam filter to monitor test output and suggest the specific intermittent
|
||
failure that may have caused a test to fail, rather than requiring users
|
||
to search logs manually</li>
|
||
</ul>
|
||
<p>While these are possibilities worth exploring in the future (and one of the
|
||
possible benefits I see to seeking closer coordination with the OpenStack
|
||
Infrastructure team), I don’t see them as offering quite the same kind of
|
||
fundamental workflow improvement that merge gating appears to provide.</p>
|
||
<p>However, if we find we are having too many problems with intermittent test
|
||
failures in the gate, then introducing the “Elastic recheck” feature may
|
||
need to be considered as part of the initial deployment.</p>
|
||
</section>
|
||
<section id="suggested-variants">
|
||
<h2><a class="toc-backref" href="#suggested-variants" role="doc-backlink">Suggested Variants</a></h2>
|
||
<p>Terry Reedy has suggested doing an initial filter which specifically looks
|
||
for approved documentation-only patches (~700 of the 4000+ open CPython
|
||
issues are pure documentation updates). This approach would avoid several
|
||
of the issues related to flaky tests and cross-platform testing, while
|
||
still allowing the rest of the automation flows to be worked out (such as
|
||
how to push a patch into the merge queue).</p>
|
||
<p>The key downside to this approach is that Zuul wouldn’t have complete
|
||
control of the merge process as it usually expects, so there would
|
||
potentially be additional coordination needed around that.</p>
|
||
<p>It may be worth keeping this approach as a fallback option if the initial
|
||
deployment proves to have more trouble with test reliability than is
|
||
anticipated.</p>
|
||
<p>It would also be possible to tweak the merge gating criteria such that it
|
||
doesn’t run the test suite if it detects that the patch hasn’t modified any
|
||
files outside the “Docs” tree, and instead only checks that the documentation
|
||
builds without errors.</p>
|
||
<p>As yet another alternative, it may be reasonable to move some parts of the
|
||
documentation (such as the tutorial and the HOWTO guides) out of the main
|
||
source repository and manage them using the simpler pull request based model
|
||
described in <a class="pep reference internal" href="../pep-0474/" title="PEP 474 – Creating forge.python.org">PEP 474</a>.</p>
|
||
</section>
|
||
<section id="perceived-benefits">
|
||
<h2><a class="toc-backref" href="#perceived-benefits" role="doc-backlink">Perceived Benefits</a></h2>
|
||
<p>The benefits of this proposal accrue most directly to the core development
|
||
team. First and foremost, it means that once we mark a patch as “Approved”
|
||
in the updated code review system, <em>we’re usually done</em>. The extra 20-30
|
||
minutes (or more) of actually applying the patch, running the tests and
|
||
merging it into Mercurial would all be orchestrated by Zuul. Push races
|
||
would also be a thing of the past - if lots of core developers are
|
||
approving patches at a sprint, then that just means the queue gets
|
||
deeper in Zuul, rather than developers getting frustrated trying to
|
||
merge changes and failing. Test failures would still happen, but they
|
||
would result in the affected patch being removed from the merge queue,
|
||
rather than breaking the code in the main repository.</p>
|
||
<p>With the bulk of the time investment moved to the review process, this
|
||
also encourages “development for reviewability” - smaller, easier to review
|
||
patches, since the overhead of running the tests multiple times will be
|
||
incurred by Zuul rather than by the core developers.</p>
|
||
<p>However, removing this time sink from the core development team should also
|
||
improve the experience of CPython development for other contributors, as it
|
||
eliminates several of the opportunities for patches to get “dropped on the
|
||
floor”, as well as increasing the time core developers are likely to have
|
||
available for reviewing contributed patches.</p>
|
||
<p>Another example of benefits to other contributors is that when a sprint
|
||
aimed primarily at new contributors is running with just a single core
|
||
developer present (such as the sprints at PyCon AU for the last
|
||
few years), the merge queue would allow that developer to focus more of
|
||
their time on reviewing patches and helping the other contributors at the
|
||
sprint, since accepting a patch for inclusion would now be a single click
|
||
in the Kallithea UI, rather than the relatively time-consuming process that
|
||
it is currently. Even when multiple core developers are present, it is
|
||
better to enable them to spend their time and effort on interacting with
|
||
the other sprint participants than it is on things that are sufficiently
|
||
mechanical that a computer can (and should) handle them.</p>
|
||
<p>With most of the ways to make a mistake when committing a change
|
||
automated out of existence, there are also substantially fewer new things to
|
||
learn when a contributor is nominated to become a core developer. This
|
||
should have a dual benefit, both in making the existing core developers more
|
||
comfortable with granting that additional level of responsibility, and in
|
||
making new contributors more comfortable with exercising it.</p>
|
||
<p>Finally, a more stable default branch in CPython makes it easier for
|
||
other Python projects to conduct continuous integration directly against the
|
||
main repo, rather than having to wait until we get into the release
|
||
candidate phase of a new release. At the moment, setting up such a system
|
||
isn’t particularly attractive, as it would need to include an additional
|
||
mechanism to wait until CPython’s own Buildbot fleet indicated that the
|
||
build was in a usable state. With the proposed merge gating system, the
|
||
trunk always remains usable.</p>
|
||
</section>
|
||
<section id="technical-challenges">
|
||
<h2><a class="toc-backref" href="#technical-challenges" role="doc-backlink">Technical Challenges</a></h2>
|
||
<p>Adapting Zuul from the OpenStack infrastructure to the CPython
|
||
infrastructure will at least require the development of additional
|
||
Zuul trigger and action plugins, and may require additional development
|
||
in some of our existing tools.</p>
|
||
<section id="kallithea-vs-gerrit">
|
||
<h3><a class="toc-backref" href="#kallithea-vs-gerrit" role="doc-backlink">Kallithea vs Gerrit</a></h3>
|
||
<p>Kallithea does not currently include a voting/approval feature that is
|
||
equivalent to Gerrit’s. For CPython, we wouldn’t need anything as
|
||
sophisticated as Gerrit’s voting system - a simple core-developer-only
|
||
“Approved” marker to trigger action from Zuul should suffice. The
|
||
core-developer-or-not flag is available in Roundup, as is the flag
|
||
indicating whether or not the uploader of a patch has signed a PSF
|
||
Contributor Licensing Agreement, which may require further development to
|
||
link contributor accounts between the Kallithea instance and Roundup.</p>
|
||
<p>Some of the existing Zuul triggers work by monitoring for particular comments
|
||
(in particular, recheck/reverify comments to ask Zuul to try merging a
|
||
change again if it was previously rejected due to an unrelated intermittent
|
||
failure). We will likely also want similar explicit triggers for Kallithea.</p>
|
||
<p>The current Zuul plugins for Gerrit work by monitoring the Gerrit activity
|
||
stream for particular events. If Kallithea has no equivalent, we will need
|
||
to add something suitable for the events we would like to trigger on.</p>
|
||
<p>There would also be development effort needed to create a Zuul plugin
|
||
that monitors Kallithea activity rather than Gerrit.</p>
|
||
</section>
|
||
<section id="mercurial-vs-gerrit-git">
|
||
<h3><a class="toc-backref" href="#mercurial-vs-gerrit-git" role="doc-backlink">Mercurial vs Gerrit/git</a></h3>
|
||
<p>Gerrit uses git as the actual storage mechanism for patches, and
|
||
automatically handles merging of approved patches. By contrast, Kallithea
|
||
use the RhodeCode created <a class="reference external" href="https://pythonhosted.org/vcs/">vcs</a> library as
|
||
an abstraction layer over specific DVCS implementations (with Mercurial and
|
||
git backends currently available).</p>
|
||
<p>Zuul is also directly integrated with git for patch manipulation - as far
|
||
as I am aware, this part of the design currently isn’t pluggable. However,
|
||
at PyCon US 2014, the Mercurial core developers at the sprints expressed
|
||
some interest in collaborating with the core development team and the Zuul
|
||
developers on enabling the use of Zuul with Mercurial in addition to git.
|
||
As Zuul is itself a Python application, migrating it to use the same DVCS
|
||
abstraction library as RhodeCode and Kallithea may be a viable path towards
|
||
achieving that.</p>
|
||
</section>
|
||
<section id="buildbot-vs-jenkins">
|
||
<h3><a class="toc-backref" href="#buildbot-vs-jenkins" role="doc-backlink">Buildbot vs Jenkins</a></h3>
|
||
<p>Zuul’s interaction with the CI system is also pluggable, using Gearman
|
||
as the <a class="reference external" href="http://ci.openstack.org/zuul/launchers.html">preferred interface</a>.
|
||
Accordingly, adapting the CI jobs to run in Buildbot rather than Jenkins
|
||
should just be a matter of writing a Gearman client that can process the
|
||
requests from Zuul and pass them on to the Buildbot master. Zuul uses the
|
||
pure Python <a class="reference external" href="https://pypi.python.org/pypi/gear">gear client library</a> to
|
||
communicate with Gearman, and this library should also be useful to handle
|
||
the Buildbot side of things.</p>
|
||
<p>Note that, in the initial iteration, I am proposing that we <em>do not</em>
|
||
attempt to pipeline test execution. This means Zuul would be running in
|
||
a very simple mode where only the patch at the head of the merge queue
|
||
is being tested on the Buildbot fleet, rather than potentially testing
|
||
several patches in parallel. I am picturing something equivalent to
|
||
requesting a forced build from the Buildbot master, and then waiting for
|
||
the result to come back before moving on to the second patch in the queue.</p>
|
||
<p>If we ultimately decide that this is not sufficient, and we need to start
|
||
using the CI pipelining features of Zuul, then we may need to look at moving
|
||
the test execution to dynamically provisioned cloud images, rather than
|
||
relying on volunteer maintained statically provisioned systems as we do
|
||
currently. The OpenStack CI infrastructure team are exploring the idea of
|
||
replacing their current use of Jenkins masters with a simpler pure Python
|
||
test runner, so if we find that we can’t get Buildbot to effectively
|
||
support the pipelined testing model, we’d likely participate in that
|
||
effort rather than setting up a Jenkins instance for CPython.</p>
|
||
<p>In this case, the main technical risk would be a matter of ensuring we
|
||
support testing on platforms other than Linux (as our stable buildbots
|
||
currently cover Windows, Mac OS X, FreeBSD and OpenIndiana in addition to a
|
||
couple of different Linux variants).</p>
|
||
<p>In such a scenario, the Buildbot fleet would still have a place in doing
|
||
“check” runs against the master repository (either periodically or for
|
||
every commit), even if it did not play a part in the merge gating process.
|
||
More unusual configurations (such as building without threads, or without
|
||
SSL/TLS support) would likely still be handled that way rather than being
|
||
included in the gate criteria (at least initially, anyway).</p>
|
||
</section>
|
||
<section id="handling-of-maintenance-branches">
|
||
<h3><a class="toc-backref" href="#handling-of-maintenance-branches" role="doc-backlink">Handling of maintenance branches</a></h3>
|
||
<p>The OpenStack project largely leaves the question of maintenance branches
|
||
to downstream vendors, rather than handling it directly. This means there
|
||
are questions to be answered regarding how we adapt Zuul to handle our
|
||
maintenance branches.</p>
|
||
<p>Python 2.7 can be handled easily enough by treating it as a separate patch
|
||
queue. This would be handled natively in Kallithea by submitting separate
|
||
pull requests in order to update the Python 2.7 maintenance branch.</p>
|
||
<p>The Python 3.x maintenance branches are potentially more complicated. My
|
||
current recommendation is to simply stop using Mercurial merges to manage
|
||
them, and instead treat them as independent heads, similar to the Python
|
||
2.7 branch. Separate pull requests would need to be submitted for the active
|
||
Python 3 maintenance branch and the default development branch. The
|
||
downside of this approach is that it increases the risk that a fix is merged
|
||
only to the maintenance branch without also being submitted to the default
|
||
branch, so we may want to design some additional tooling that ensures that
|
||
every maintenance branch pull request either has a corresponding default
|
||
branch pull request prior to being merged, or else has an explicit disclaimer
|
||
indicating that it is only applicable to that branch and doesn’t need to be
|
||
ported forward to later branches.</p>
|
||
<p>Such an approach has the benefit of adjusting relatively cleanly to the
|
||
intermittent periods where we have two active Python 3 maintenance branches.</p>
|
||
<p>This issue does suggest some potential user interface ideas for Kallithea,
|
||
where it may be desirable to be able to clone a pull request in order to be
|
||
able to apply it to a second branch.</p>
|
||
</section>
|
||
<section id="handling-of-security-branches">
|
||
<h3><a class="toc-backref" href="#handling-of-security-branches" role="doc-backlink">Handling of security branches</a></h3>
|
||
<p>For simplicity’s sake, I would suggest leaving the handling of
|
||
security-fix only branches alone: the release managers for those branches
|
||
would continue to backport specific changes manually. The only change is
|
||
that they would be able to use the Kallithea pull request workflow to do the
|
||
backports if they would like others to review the updates prior to merging
|
||
them.</p>
|
||
</section>
|
||
<section id="handling-of-news-file-updates">
|
||
<h3><a class="toc-backref" href="#handling-of-news-file-updates" role="doc-backlink">Handling of NEWS file updates</a></h3>
|
||
<p>Our current approach to handling NEWS file updates regularly results in
|
||
spurious conflicts when merging bug fixes forward from an active maintenance
|
||
branch to a later branch.</p>
|
||
<p><a class="reference external" href="https://github.com/python/cpython/issues/63167">Issue #18967*</a> discusses some
|
||
possible improvements in that area, which would be beneficial regardless
|
||
of whether or not we adopt Zuul as a workflow automation tool.</p>
|
||
</section>
|
||
<section id="stability-of-stable-buildbot-slaves">
|
||
<h3><a class="toc-backref" href="#stability-of-stable-buildbot-slaves" role="doc-backlink">Stability of “stable” Buildbot slaves</a></h3>
|
||
<p>Instability of the nominally stable buildbots has a substantially larger
|
||
impact under this proposal. We would need to ensure we’re genuinely happy
|
||
with each of those systems gating merges to the development branches, or
|
||
else move then to “unstable” status.</p>
|
||
</section>
|
||
<section id="intermittent-test-failures">
|
||
<h3><a class="toc-backref" href="#intermittent-test-failures" role="doc-backlink">Intermittent test failures</a></h3>
|
||
<p>Some tests, especially timing tests, exhibit intermittent failures on the
|
||
existing Buildbot fleet. In particular, test systems running as VMs may
|
||
sometimes exhibit timing failures when the VM host is under higher than
|
||
normal load.</p>
|
||
<p>The OpenStack CI infrastructure includes a number of additional features to
|
||
help deal with intermittent failures, the most basic of which is simply
|
||
allowing developers to request that merging a patch be tried again when the
|
||
original failure appears to be due to a known intermittent failure (whether
|
||
that intermittent failure is in OpenStack itself or just in a flaky test).</p>
|
||
<p>The more sophisticated <a class="reference external" href="http://status.openstack.org/elastic-recheck/">Elastic recheck</a> feature may be worth considering,
|
||
especially since the output of the CPython test suite is substantially
|
||
simpler than that from OpenStack’s more complex multi-service testing, and
|
||
hence likely even more amenable to automated analysis.</p>
|
||
</section>
|
||
<section id="custom-mercurial-client-workflow-support">
|
||
<h3><a class="toc-backref" href="#custom-mercurial-client-workflow-support" role="doc-backlink">Custom Mercurial client workflow support</a></h3>
|
||
<p>One useful part of the OpenStack workflow is the “git review” plugin,
|
||
which makes it relatively easy to push a branch from a local git clone up
|
||
to Gerrit for review.</p>
|
||
<p><a class="pep reference internal" href="../pep-0474/" title="PEP 474 – Creating forge.python.org">PEP 474</a> mentions a draft <a class="reference external" href="https://bitbucket.org/ncoghlan/cpydev/src/default/cpyhg.py?at=default">custom Mercurial
|
||
extension</a>
|
||
that automates some aspects of the existing CPython core development workflow.</p>
|
||
<p>As part of this proposal, that custom extension would be extended to work
|
||
with the new Kallithea based review workflow in addition to the legacy
|
||
Roundup/Rietveld based review workflow.</p>
|
||
</section>
|
||
</section>
|
||
<section id="social-challenges">
|
||
<h2><a class="toc-backref" href="#social-challenges" role="doc-backlink">Social Challenges</a></h2>
|
||
<p>The primary social challenge here is getting the core development team to
|
||
change their practices. However, the tedious-but-necessary steps that are
|
||
automated by the proposal should create a strong incentive for the
|
||
existing developers to go along with the idea.</p>
|
||
<p>I believe three specific features may be needed to assure existing
|
||
developers that there are no downsides to the automation of this workflow:</p>
|
||
<ul class="simple">
|
||
<li>Only requiring approval from a single core developer to incorporate a
|
||
patch. This could be revisited in the future, but we should preserve the
|
||
status quo for the initial rollout.</li>
|
||
<li>Explicitly stating that core developers remain free to approve their own
|
||
patches, except during the release candidate phase of a release. This
|
||
could be revisited in the future, but we should preserve the status quo
|
||
for the initial rollout.</li>
|
||
<li>Ensuring that at least release managers have a “merge it now” capability
|
||
that allows them to force a particular patch to the head of the merge
|
||
queue. Using a separate clone for release preparation may be sufficient
|
||
for this purpose. Longer term, automatic merge gating may also allow for
|
||
more automated preparation of release artefacts as well.</li>
|
||
</ul>
|
||
</section>
|
||
<section id="practical-challenges">
|
||
<h2><a class="toc-backref" href="#practical-challenges" role="doc-backlink">Practical Challenges</a></h2>
|
||
<p>The PSF runs its own directly and indirectly sponsored workflow
|
||
infrastructure primarily due to past experience with unacceptably poor
|
||
performance and inflexibility of infrastructure provided for free to the
|
||
general public. CPython development was originally hosted on SourceForge,
|
||
with source control moved to self hosting when SF was both slow to offer
|
||
Subversion support and suffering from CVS performance issues (see <a class="pep reference internal" href="../pep-0347/" title="PEP 347 – Migrating the Python CVS to Subversion">PEP 347</a>),
|
||
while issue tracking later moved to the open source Roundup issue tracker
|
||
on dedicated sponsored hosting (from Upfront Systems), due to a combination
|
||
of both SF performance issues and general usability issues with the SF
|
||
tracker at the time (the outcome and process for the new tracker selection
|
||
were captured on the <a class="reference external" href="https://wiki.python.org/moin/CallForTrackers">python.org wiki</a> rather than in a PEP).</p>
|
||
<p>Accordingly, proposals that involve setting ourselves up for “SourceForge
|
||
usability and reliability issues, round two” will face significant
|
||
opposition from at least some members of the CPython core development team
|
||
(including the author of this PEP). This proposal respects that history by
|
||
recommending only tools that are available for self-hosting as sponsored
|
||
or PSF funded infrastructure, and are also open source Python projects that
|
||
can be customised to meet the needs of the CPython core development team.</p>
|
||
<p>However, for this proposal to be a success (if it is accepted), we need to
|
||
understand how we are going to carry out the necessary configuration,
|
||
customisation, integration and deployment work.</p>
|
||
<p>The last attempt at adding a new piece to the CPython support infrastructure
|
||
(speed.python.org) has unfortunately foundered due to the lack of time to
|
||
drive the project from the core developers and PSF board members involved,
|
||
and the difficulties of trying to bring someone else up to speed to lead
|
||
the activity (the hardware donated to that project by HP is currently in
|
||
use to support PyPy instead, but the situation highlights some
|
||
of the challenges of relying on volunteer labour with many other higher
|
||
priority demands on their time to steer projects to completion).</p>
|
||
<p>Even ultimately successful past projects, such as the source control
|
||
migrations from CVS to Subversion and from Subversion to Mercurial, the
|
||
issue tracker migration from SourceForge to Roundup, the code review
|
||
integration between Roundup and Rietveld and the introduction of the
|
||
Buildbot continuous integration fleet, have taken an extended period of
|
||
time as volunteers worked their way through the many technical and social
|
||
challenges involved.</p>
|
||
<p>Fortunately, as several aspects of this proposal and <a class="pep reference internal" href="../pep-0474/" title="PEP 474 – Creating forge.python.org">PEP 474</a> align with
|
||
various workflow improvements under consideration for Red Hat’s
|
||
<a class="reference external" href="https://beaker-project.org">Beaker</a> open source hardware integration
|
||
testing system and other work-related projects, I have arranged to be able
|
||
to devote ~1 day a week to working on CPython infrastructure projects.</p>
|
||
<p>Together with Rackspace’s existing contributions to maintaining the
|
||
pypi.python.org infrastructure, I personally believe this arrangement is
|
||
indicative of a more general recognition amongst CPython redistributors and
|
||
major users of the merit in helping to sustain upstream infrastructure
|
||
through direct contributions of developer time, rather than expecting
|
||
volunteer contributors to maintain that infrastructure entirely in their
|
||
spare time or funding it indirectly through the PSF (with the additional
|
||
management overhead that would entail). I consider this a positive trend, and
|
||
one that I will continue to encourage as best I can.</p>
|
||
</section>
|
||
<section id="open-questions">
|
||
<h2><a class="toc-backref" href="#open-questions" role="doc-backlink">Open Questions</a></h2>
|
||
<p>Pretty much everything in the PEP. Do we want to adopt merge gating and
|
||
Zuul? How do we want to address the various technical challenges?
|
||
Are the Kallithea and Zuul development communities open to the kind
|
||
of collaboration that would be needed to make this effort a success?</p>
|
||
<p>While I’ve arranged to spend some of my own work time on this, do we want to
|
||
approach the OpenStack Foundation for additional assistance, since
|
||
we’re a key dependency of OpenStack itself, Zuul is a creation of the
|
||
OpenStack infrastructure team, and the available development resources for
|
||
OpenStack currently dwarf those for CPython?</p>
|
||
<p>Are other interested folks working for Python redistributors and major users
|
||
also in a position to make a business case to their superiors for investing
|
||
developer time in supporting this effort?</p>
|
||
</section>
|
||
<section id="next-steps">
|
||
<h2><a class="toc-backref" href="#next-steps" role="doc-backlink">Next Steps</a></h2>
|
||
<p>If pursued, this will be a follow-on project to the Kallithea-based
|
||
forge.python.org proposal in <a class="pep reference internal" href="../pep-0474/" title="PEP 474 – Creating forge.python.org">PEP 474</a>. Refer to that PEP for more details
|
||
on the discussion, review and proof-of-concept pilot process currently
|
||
under way.</p>
|
||
</section>
|
||
<section id="acknowledgements">
|
||
<h2><a class="toc-backref" href="#acknowledgements" role="doc-backlink">Acknowledgements</a></h2>
|
||
<p>Thanks to Jesse Noller, Alex Gaynor and James Blair for providing valuable
|
||
feedback on a preliminary draft of this proposal, and to James and Monty
|
||
Taylor for additional technical feedback following publication of the
|
||
initial draft.</p>
|
||
<p>Thanks to Bradley Kuhn, Mads Kiellerich and other Kallithea developers for
|
||
the discussions around <a class="pep reference internal" href="../pep-0474/" title="PEP 474 – Creating forge.python.org">PEP 474</a> that led to a significant revision of this
|
||
proposal to be based on using Kallithea for the review component rather than
|
||
the existing Rietveld installation.</p>
|
||
</section>
|
||
<section id="copyright">
|
||
<h2><a class="toc-backref" href="#copyright" role="doc-backlink">Copyright</a></h2>
|
||
<p>This document has been placed in the public domain.</p>
|
||
</section>
|
||
</section>
|
||
<hr class="docutils" />
|
||
<p>Source: <a class="reference external" href="https://github.com/python/peps/blob/main/peps/pep-0462.rst">https://github.com/python/peps/blob/main/peps/pep-0462.rst</a></p>
|
||
<p>Last modified: <a class="reference external" href="https://github.com/python/peps/commits/main/peps/pep-0462.rst">2023-10-11 12:05:51 GMT</a></p>
|
||
|
||
</article>
|
||
<nav id="pep-sidebar">
|
||
<h2>Contents</h2>
|
||
<ul>
|
||
<li><a class="reference internal" href="#abstract">Abstract</a></li>
|
||
<li><a class="reference internal" href="#pep-withdrawal">PEP Withdrawal</a></li>
|
||
<li><a class="reference internal" href="#rationale-for-changes-to-the-core-development-workflow">Rationale for changes to the core development workflow</a></li>
|
||
<li><a class="reference internal" href="#current-tools">Current Tools</a></li>
|
||
<li><a class="reference internal" href="#proposal">Proposal</a></li>
|
||
<li><a class="reference internal" href="#deferred-proposals">Deferred Proposals</a></li>
|
||
<li><a class="reference internal" href="#suggested-variants">Suggested Variants</a></li>
|
||
<li><a class="reference internal" href="#perceived-benefits">Perceived Benefits</a></li>
|
||
<li><a class="reference internal" href="#technical-challenges">Technical Challenges</a><ul>
|
||
<li><a class="reference internal" href="#kallithea-vs-gerrit">Kallithea vs Gerrit</a></li>
|
||
<li><a class="reference internal" href="#mercurial-vs-gerrit-git">Mercurial vs Gerrit/git</a></li>
|
||
<li><a class="reference internal" href="#buildbot-vs-jenkins">Buildbot vs Jenkins</a></li>
|
||
<li><a class="reference internal" href="#handling-of-maintenance-branches">Handling of maintenance branches</a></li>
|
||
<li><a class="reference internal" href="#handling-of-security-branches">Handling of security branches</a></li>
|
||
<li><a class="reference internal" href="#handling-of-news-file-updates">Handling of NEWS file updates</a></li>
|
||
<li><a class="reference internal" href="#stability-of-stable-buildbot-slaves">Stability of “stable” Buildbot slaves</a></li>
|
||
<li><a class="reference internal" href="#intermittent-test-failures">Intermittent test failures</a></li>
|
||
<li><a class="reference internal" href="#custom-mercurial-client-workflow-support">Custom Mercurial client workflow support</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#social-challenges">Social Challenges</a></li>
|
||
<li><a class="reference internal" href="#practical-challenges">Practical Challenges</a></li>
|
||
<li><a class="reference internal" href="#open-questions">Open Questions</a></li>
|
||
<li><a class="reference internal" href="#next-steps">Next Steps</a></li>
|
||
<li><a class="reference internal" href="#acknowledgements">Acknowledgements</a></li>
|
||
<li><a class="reference internal" href="#copyright">Copyright</a></li>
|
||
</ul>
|
||
|
||
<br>
|
||
<a id="source" href="https://github.com/python/peps/blob/main/peps/pep-0462.rst">Page Source (GitHub)</a>
|
||
</nav>
|
||
</section>
|
||
<script src="../_static/colour_scheme.js"></script>
|
||
<script src="../_static/wrap_tables.js"></script>
|
||
<script src="../_static/sticky_banner.js"></script>
|
||
</body>
|
||
</html> |