diff --git a/.codespell/exclude-file.txt b/.codespell/exclude-file.txt new file mode 100644 index 000000000..b3b96e5b4 --- /dev/null +++ b/.codespell/exclude-file.txt @@ -0,0 +1,17 @@ + of scoped seem allright. I still think there is not enough need + + da de dum, hmm, hmm, dum de dum. + + output=`dmesg | grep hda` + p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) + + Error-de_DE=Wenn ist das NunstĂŒck git und Slotermeyer? + Ja! Beiherhund das Oder die Virtualenvironment gersput! + +`__ + + class ClassE[T: [str, int]]: ... # Type checker error: illegal expression form + class ClassE[T: t1]: ... # Type checker error: literal tuple expression required + +explicitly declared using ``in``, ``out`` and ``inout`` keywords. +| | | | | | | inout | diff --git a/.codespell/ignore-words.txt b/.codespell/ignore-words.txt new file mode 100644 index 000000000..20a642ae9 --- /dev/null +++ b/.codespell/ignore-words.txt @@ -0,0 +1,21 @@ +adaptee +ancilliary +ans +arithmetics +asend +ba +clos +complies +crate +dedented +extraversion +falsy +fo +iif +nd +ned +recuse +reenable +referencable +therefor +warmup diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 000000000..c3cd40b94 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,5 @@ +[codespell] +skip = ./.git +ignore-words = .codespell/ignore-words.txt +exclude-file = .codespell/exclude-file.txt +uri-ignore-words-list = daa,ist,searchin,theses diff --git a/.gitattributes b/.gitattributes index f3bffe3c0..6107f6104 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,7 @@ *.png binary *.pptx binary *.odp binary + +# Instruct linguist not to ignore the PEPs +# https://github.com/github-linguist/linguist/blob/master/docs/overrides.md +peps/*.rst text linguist-detectable diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 307c20211..15fc5c5bd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,613 +5,691 @@ # The PEP editors are the fallback "owners" for everything in this repository. * @python/pep-editors -pep-0001.txt @warsaw @ncoghlan -pep-0001-process_flow.png @warsaw @ncoghlan -pep-0001/ @warsaw @ncoghlan -# pep-0002.txt -pep-0003.txt @jeremyhylton -pep-0004.txt @brettcannon -# pep-0005.txt -# pep-0006.txt -pep-0007.txt @gvanrossum @warsaw -pep-0008.txt @gvanrossum @warsaw @ncoghlan -pep-0009.txt @warsaw -pep-0010.txt @warsaw -pep-0011.txt @brettcannon -pep-0012.rst @brettcannon @warsaw -# pep-0013.rst is owned by the entire core team. +# PEP infrastructure +.github/workflows/ @AA-Turner @CAM-Gerlach +Makefile @AA-Turner +requirements.txt @AA-Turner +infra/ @ewdurbin + +pep_sphinx_extensions/ @AA-Turner +build.py @AA-Turner +peps/conf.py @AA-Turner +peps/contents.rst @AA-Turner + +# Linting infrastructure +.codespell/ @CAM-Gerlach @hugovk +.codespellrc @CAM-Gerlach @hugovk +.pre-commit-config.yaml @CAM-Gerlach @hugovk +.ruff.toml @AA-Turner @CAM-Gerlach @hugovk +check-peps.py @AA-Turner @CAM-Gerlach @hugovk + +# Git infrastructure +.gitattributes @CAM-Gerlach +.gitignore @CAM-Gerlach + +peps/pep-0001.rst @warsaw @ncoghlan +peps/pep-0001-process_flow.png @warsaw @ncoghlan +peps/pep-0001/ @warsaw @ncoghlan +# peps/pep-0002.rst +peps/pep-0003.rst @jeremyhylton +peps/pep-0004.rst @brettcannon +# peps/pep-0005.rst +# peps/pep-0006.rst +peps/pep-0007.rst @gvanrossum @warsaw +peps/pep-0008.rst @gvanrossum @warsaw @ncoghlan +peps/pep-0009.rst @warsaw +peps/pep-0010.rst @warsaw +peps/pep-0011.rst @brettcannon +peps/pep-0012.rst @brettcannon @warsaw +peps/pep-0012/ @brettcannon +# peps/pep-0013.rst is owned by the entire core team. # ... -pep-0020.txt @tim-one +peps/pep-0020.rst @tim-one # ... -pep-0042.txt @jeremyhylton +peps/pep-0042.rst @jeremyhylton # ... -pep-0100.txt @malemburg -pep-0101.txt @warsaw @gvanrossum -pep-0102.txt @warsaw @gvanrossum -# pep-0103.txt +peps/pep-0100.rst @malemburg +peps/pep-0101.rst @Yhg1s @pablogsal @ambv @ned-deily +peps/pep-0102.rst @warsaw @gvanrossum +# peps/pep-0103.rst # ... -pep-0160.txt @freddrake +peps/pep-0160.rst @freddrake # ... -pep-0200.txt @jeremyhylton -pep-0201.txt @warsaw -pep-0202.txt @warsaw -pep-0203.txt @Yhg1s -pep-0204.txt @Yhg1s -pep-0205.txt @freddrake -# pep-0206.txt -pep-0207.txt @gvanrossum -pep-0208.txt @nascheme @malemburg -# pep-0209.txt -# pep-0210.txt -# pep-0211.txt -# pep-0212.txt -# pep-0213.txt -pep-0214.txt @warsaw -# pep-0215.txt -# pep-0216.txt -# pep-0217.txt -pep-0218.txt @rhettinger -# pep-0219.txt -# pep-0220.txt -pep-0221.txt @Yhg1s -# pep-0222.txt -pep-0223.txt @tim-one -pep-0224.txt @malemburg -# pep-0225.txt -pep-0226.txt @jeremyhylton -pep-0227.txt @jeremyhylton -pep-0228.txt @gvanrossum -# pep-0229.txt -pep-0230.txt @gvanrossum -pep-0231.txt @warsaw -pep-0232.txt @warsaw -# pep-0233.txt -pep-0234.txt @gvanrossum -pep-0235.txt @tim-one -pep-0236.txt @tim-one -pep-0237.txt @gvanrossum -pep-0238.txt @gvanrossum -# pep-0239.txt -# pep-0240.txt -# pep-0241.txt -# pep-0242.txt -# pep-0243.txt -# pep-0244.txt -# pep-0245.txt -pep-0246.txt @aleaxit -# pep-0247.txt -pep-0248.txt @malemburg -pep-0249.txt @malemburg -pep-0250.txt @pfmoore -pep-0251.txt @warsaw @gvanrossum -pep-0252.txt @gvanrossum -pep-0253.txt @gvanrossum -pep-0254.txt @gvanrossum -pep-0255.txt @nascheme @tim-one -# pep-0256.txt -pep-0257.txt @gvanrossum -# pep-0258.txt -pep-0259.txt @gvanrossum -pep-0260.txt @gvanrossum -# pep-0261.txt -# pep-0262.txt -pep-0263.txt @malemburg -# pep-0264.txt -# pep-0265.txt -# pep-0266.txt -pep-0267.txt @jeremyhylton -# pep-0268.txt -# pep-0269.txt -# pep-0270.txt -# pep-0271.txt -# pep-0272.txt -# pep-0273.txt -pep-0274.txt @warsaw -pep-0275.txt @malemburg -# pep-0276.txt -# pep-0277.txt -pep-0278.txt @jackjansen -pep-0279.txt @rhettinger -pep-0280.txt @gvanrossum -# pep-0281.txt -pep-0282.txt @vsajip -pep-0283.txt @gvanrossum -# pep-0284.txt -pep-0285.txt @gvanrossum -# pep-0286.txt -# pep-0287.txt -pep-0288.txt @rhettinger -pep-0289.txt @rhettinger -pep-0290.txt @rhettinger -# pep-0291.txt -pep-0292.txt @warsaw -pep-0293.txt @doerwalter -# pep-0294.txt -# pep-0295.txt -# pep-0296.txt -pep-0297.txt @malemburg -pep-0298.txt @theller -# pep-0299.txt -# pep-0301.txt -pep-0302.txt @pfmoore -# pep-0303.txt -# pep-0304.txt -# pep-0305.txt -pep-0306.txt @jackdied @ncoghlan @benjaminp -pep-0307.txt @gvanrossum @tim-one -pep-0308.txt @gvanrossum @rhettinger -# pep-0309.txt -pep-0310.txt @pfmoore -pep-0311.txt @mhammond -pep-0312.txt @aleaxit -# pep-0313.txt -# pep-0314.txt -pep-0315.txt @rhettinger -# pep-0316.txt -# pep-0317.txt -# pep-0318.txt -# pep-0319.txt -pep-0320.txt @warsaw @rhettinger -# pep-0321.txt -pep-0322.txt @rhettinger -pep-0323.txt @aleaxit -# pep-0324.txt -# pep-0325.txt -pep-0326.txt @terryjreedy -pep-0327.txt @facundobatista -# pep-0328.txt -pep-0329.txt @rhettinger -# pep-0330.txt -# pep-0331.txt -# pep-0332.txt -pep-0333.txt @pjeby -# pep-0334.txt -# pep-0335.txt -# pep-0336.txt -# pep-0337.txt -pep-0338.txt @ncoghlan -pep-0339.txt @brettcannon -pep-0340.txt @gvanrossum -pep-0341.txt @birkenfeld -pep-0342.txt @gvanrossum @pjeby -pep-0343.txt @gvanrossum @ncoghlan -# pep-0344.txt -# pep-0345.txt -pep-0346.txt @ncoghlan -# pep-0347.txt -pep-0348.txt @brettcannon -pep-0349.txt @nascheme -# pep-0350.txt -pep-0351.txt @warsaw -pep-0352.txt @brettcannon @gvanrossum -# pep-0353.txt -# pep-0354.txt -# pep-0355.txt -pep-0356.txt @gvanrossum -# pep-0357.txt -pep-0358.txt @nascheme @gvanrossum -# pep-0359.txt -pep-0360.txt @brettcannon -pep-0361.txt @warsaw -pep-0362.txt @brettcannon @1st1 @larryhastings -# pep-0363.txt -pep-0364.txt @warsaw -pep-0365.txt @pjeby -pep-0366.txt @ncoghlan -# pep-0367.txt -# pep-0368.txt -pep-0369.txt @tiran -pep-0370.txt @tiran -# pep-0371.txt -pep-0372.txt @mitsuhiko @rhettinger -pep-0373.txt @benjaminp -pep-0374.txt @brettcannon @avassalotti @warsaw -pep-0375.txt @benjaminp -# pep-0376.txt -pep-0377.txt @ncoghlan -pep-0378.txt @rhettinger -# pep-0379.txt -# pep-0380.txt -# pep-0381.txt -# pep-0382.txt -# pep-0383.txt -# pep-0384.txt -pep-0385.txt @pitrou @birkenfeld -# pep-0386.txt -pep-0387.txt @benjaminp -# pep-0389.txt -# pep-0390.txt -pep-0391.txt @vsajip -pep-0392.txt @birkenfeld -# pep-0393.txt -pep-0394.txt @ncoghlan @warsaw @encukou @willingc -pep-0395.txt @ncoghlan -pep-0396.txt @warsaw -pep-0397.txt @mhammond -pep-0398.txt @birkenfeld -pep-0399.txt @brettcannon -pep-0400.txt @vstinner -pep-0401.txt @warsaw @brettcannon -pep-0402.txt @pjeby -pep-0403.txt @ncoghlan -pep-0404.txt @warsaw -# pep-0405.txt -pep-0406.txt @ncoghlan -pep-0407.txt @pitrou @birkenfeld @warsaw -pep-0408.txt @ncoghlan @eliben -pep-0409.txt @ethanfurman -pep-0410.txt @vstinner -pep-0411.txt @ncoghlan @eliben -pep-0412.txt @markshannon -pep-0413.txt @ncoghlan -pep-0414.txt @mitsuhiko @ncoghlan -pep-0415.txt @benjaminp -pep-0416.txt @vstinner -pep-0417.txt @voidspace -pep-0418.txt @vstinner -pep-0418/ @vstinner -# pep-0419.txt -pep-0420.txt @ericvsmith -pep-0421.txt @ericsnowcurrently -pep-0422.txt @ncoghlan -# pep-0423.txt -pep-0424.txt @alex -# pep-0425.txt -pep-0426.txt @ncoghlan @dstufft -pep-0426/ @ncoghlan @dstufft -# pep-0427.txt -pep-0428.txt @pitrou -pep-0429.txt @larryhastings -pep-0430.txt @ncoghlan -# pep-0431.txt -pep-0432.txt @ncoghlan @vstinner @ericsnowcurrently -pep-0433.txt @vstinner -pep-0433/ @vstinner -pep-0434.txt @terryjreedy -pep-0435.txt @warsaw @eliben @ethanfurman -pep-0436.txt @larryhastings -# pep-0437.txt -# pep-0438.txt -# pep-0439.txt -pep-0440.txt @ncoghlan @dstufft -pep-0441.txt @pfmoore -pep-0442.txt @pitrou -pep-0443.txt @ambv -pep-0444.txt @mitsuhiko -pep-0445.txt @vstinner -pep-0446.txt @vstinner -pep-0446/ @vstinner -pep-0447.txt @ronaldoussoren -# pep-0448.txt -pep-0449.txt @dstufft -pep-0450.txt @stevendaprano -pep-0451.txt @ericsnowcurrently -pep-0452.txt @tiran -pep-0453.txt @dstufft @ncoghlan -pep-0454.txt @vstinner -pep-0455.txt @pitrou -pep-0456.txt @tiran -pep-0457.txt @larryhastings -# pep-0458.txt, pep-0458-1.png -pep-0459.txt @ncoghlan -pep-0460.txt @pitrou -pep-0461.txt @ethanfurman -pep-0462.txt @ncoghlan -# pep-0463.txt -pep-0464.txt @dstufft -pep-0465.txt @njsmith -pep-0466.txt @ncoghlan -pep-0467.txt @ncoghlan @ethanfurman -pep-0468.txt @ericsnowcurrently -pep-0469.txt @ncoghlan -pep-0470.txt @dstufft -# pep-0471.txt -# pep-0472.txt -# pep-0473.txt -pep-0474.txt @ncoghlan -pep-0475.txt @vstinner -pep-0476.txt @alex -pep-0477.txt @dstufft @ncoghlan -pep-0478.txt @larryhastings -pep-0479.txt @gvanrossum -# pep-0480.txt, pep-0480-1.png -pep-0481.txt @dstufft -pep-0482.txt @ambv -pep-0483.txt @gvanrossum @ilevkivskyi -pep-0484.txt @gvanrossum @ambv -# pep-0485.txt -pep-0486.txt @pfmoore -# pep-0487.txt -pep-0488.txt @brettcannon -pep-0489.txt @encukou @scoder @ncoghlan -pep-0490.txt @vstinner -# pep-0491.txt -pep-0492.txt @1st1 -pep-0493.txt @ncoghlan @malemburg -pep-0494.txt @ned-deily -pep-0495.txt @abalkin @tim-one -pep-0495-gap.png @abalkin @tim-one -pep-0495-gap.svg @abalkin @tim-one -pep-0495-fold.svg @abalkin @tim-one -pep-0495-fold-2.png @abalkin @tim-one -pep-0495-daylightsavings.png @abalkin @tim-one -# pep-0496.txt -# pep-0497.txt -pep-0498.txt @ericvsmith -# pep-0499.txt -pep-0500.txt @abalkin @tim-one -pep-0501.txt @ncoghlan -# pep-0502.txt -pep-0503.txt @dstufft -pep-0504.txt @ncoghlan -pep-0505.rst @zooba -pep-0505/ @zooba -pep-0506.txt @stevendaprano -pep-0507.txt @warsaw -pep-0508.txt @rbtcollins -pep-0509.txt @vstinner -pep-0510.txt @vstinner -pep-0511.txt @vstinner -pep-0512.txt @brettcannon -pep-0513.txt @njsmith -pep-0514.txt @zooba -pep-0515.txt @birkenfeld @serhiy-storchaka -pep-0516.txt @rbtcollins @njsmith -pep-0517.txt @njsmith -pep-0518.txt @brettcannon @njsmith @dstufft -pep-0519.txt @brettcannon -pep-0520.txt @ericsnowcurrently -pep-0521.txt @njsmith -pep-0522.txt @ncoghlan @njsmith -pep-0523.txt @brettcannon @DinoV -pep-0524.txt @vstinner -pep-0525.txt @1st1 -pep-0525-1.png @1st1 -pep-0526.txt @ilevkivskyi @lisroach @gvanrossum -pep-0527.txt @dstufft -pep-0528.txt @zooba -pep-0529.txt @zooba -pep-0530.txt @1st1 -pep-0531.txt @ncoghlan -pep-0532.txt @ncoghlan -pep-0532/ @ncoghlan -pep-0533.txt @njsmith -pep-0534.txt @encukou @ncoghlan -pep-0535.txt @ncoghlan -# pep-0536.txt -pep-0537.txt @ned-deily -pep-0538.txt @ncoghlan -# pep-0539.txt -pep-0540.txt @vstinner -pep-0541.txt @ambv -# pep-0542.txt -pep-0543.rst @tiran -pep-0544.txt @ilevkivskyi @ambv -pep-0545.txt @JulienPalard @methane @vstinner -pep-0546.txt @vstinner -pep-0547.rst @encukou -pep-0548.rst @bitdancer -pep-0549.rst @larryhastings -pep-0550.rst @1st1 -pep-0550-lookup_hamt.png @1st1 -pep-0550-hamt_vs_dict.png @1st1 -pep-0550-hamt_vs_dict-v2.png @1st1 -pep-0551.rst @zooba -pep-0552.rst @benjaminp -pep-0553.rst @warsaw -pep-0554.rst @ericsnowcurrently -# pep-0555.rst -pep-0556.rst @pitrou -pep-0557.rst @ericvsmith -pep-0558.rst @ncoghlan -pep-0559.rst @warsaw -pep-0560.rst @ilevkivskyi -# pep-0561.rst -pep-0562.rst @ilevkivskyi -pep-0563.rst @ambv -pep-0564.rst @vstinner -pep-0565.rst @ncoghlan -# pep-0566.rst -pep-0567.rst @1st1 -pep-0568.rst @njsmith -pep-0569.rst @ambv -pep-0570.rst @larryhastings @pablogsal -# pep-0571.rst -pep-0572.rst @tim-one @gvanrossum -pep-0573.rst @encukou @ncoghlan @ericsnowcurrently -pep-0574.rst @pitrou -# pep-0575.rst -pep-0576.rst @markshannon -pep-0577.rst @ncoghlan -pep-0578.rst @zooba -# pep-0579.rst -# pep-0580.rst -pep-0581.rst @Mariatta -pep-0582.rst @kushaldas @zooba @dstufft @ncoghlan -# pep-0583.rst -pep-0584.rst @stevendaprano @brandtbucher -pep-0585.rst @ambv -pep-0586.rst @ilevkivskyi -pep-0587.rst @vstinner @ncoghlan -pep-0588.rst @Mariatta -pep-0589.rst @gvanrossum -pep-0590.rst @markshannon -pep-0591.rst @ilevkivskyi -pep-0592.rst @dstufft -pep-0593.rst @ilevkivskyi -pep-0594.rst @tiran -pep-0595.rst @ezio-melotti @berkerpeksag -pep-0596.rst @ambv -pep-0597.rst @methane -pep-0598.rst @ncoghlan -pep-0599.rst @pfmoore -pep-0600.rst @njsmith -pep-0601.txt @isidentical -pep-0602.rst @ambv -pep-0602-example-release-calendar.png @ambv -pep-0602-example-release-calendar.pptx @ambv -pep-0602-overlapping-support-matrix.png @ambv -pep-0602-overlapping-support-matrix.pptx @ambv -pep-0603.rst @1st1 -pep-0603-lookup_hamt.png @1st1 -pep-0603-hamt_vs_dict.png @1st1 -# pep-0604.rst -pep-0605.rst @zooba @ncoghlan -pep-0605-example-release-calendar.png @zooba @ncoghlan -pep-0605-overlapping-support-matrix.png @zooba @ncoghlan -/pep-0605/ @zooba @ncoghlan -pep-0606.rst @vstinner -pep-0607.rst @ambv @zooba @ncoghlan -pep-0608.rst @vstinner -pep-0609.rst @pganssle -pep-0610.rst @cjerdonek -pep-0611.rst @markshannon -pep-0612.rst @gvanrossum -pep-0613.rst @gvanrossum -pep-0614.rst @brandtbucher -pep-0615.rst @pganssle -pep-0616.rst @ericvsmith -pep-0617.rst @gvanrossum @pablogsal @lysnikolaou -pep-0618.rst @brandtbucher -pep-0619.rst @pablogsal -pep-0620.rst @vstinner -pep-0621.rst @brettcannon @pganssle -pep-0622.rst @brandtbucher @ilevkivskyi @gvanrossum -pep-0623.rst @methane -pep-0624.rst @methane -pep-0625.rst @pfmoore -pep-0626.rst @markshannon -pep-0627.rst @encukou -pep-0628.txt @ncoghlan -pep-0629.rst @dstufft -pep-0630.rst @encukou -pep-0631.rst @pganssle -pep-0632.rst @zooba -pep-0633.rst @brettcannon -pep-0634.rst @brandtbucher @gvanrossum -pep-0635.rst @brandtbucher @gvanrossum -pep-0636.rst @brandtbucher @gvanrossum -pep-0637.rst @stevendaprano -pep-0638.rst @markshannon -pep-0639.rst @pfmoore -pep-0640.rst @Yhg1s -pep-0641.rst @zooba @warsaw @brettcannon -pep-0642.rst @ncoghlan -pep-0643.rst @pfmoore -pep-0644.rst @tiran -pep-0645.rst @gvanrossum -pep-0646.rst @gvanrossum -pep-0647.rst @gvanrossum -pep-0648.rst @pablogsal -pep-0649.rst @larryhastings -pep-0650.rst @brettcannon -pep-0651.rst @markshannon -pep-0652.rst @encukou -pep-0653.rst @markshannon -pep-0654.rst @1st1 @gvanrossum @iritkatriel -pep-0655.rst @gvanrossum -pep-0656.rst @brettcannon -pep-0657.rst @pablogsal @isidentical @ammaraskar -pep-0658.rst @brettcannon -pep-0659.rst @markshannon -pep-0660.rst @pfmoore -pep-0661.rst @taleinat -pep-0662.rst @brettcannon -pep-0662/pep-0662-editables.json @brettcannon -pep-0663.txt @ethanfurman -pep-0664.rst @pablogsal -pep-0665.rst @brettcannon -# pep-0666.txt -pep-0667.rst @markshannon -pep-0668.rst @dstufft -pep-0670.rst @vstinner @erlend-aasland -pep-0671.rst @rosuav -pep-0672.rst @encukou -pep-0673.rst @jelle.zijlstra +peps/pep-0200.rst @jeremyhylton +peps/pep-0201.rst @warsaw +peps/pep-0202.rst @warsaw +peps/pep-0203.rst @Yhg1s +peps/pep-0204.rst @Yhg1s +peps/pep-0205.rst @freddrake +# peps/pep-0206.rst +peps/pep-0207.rst @gvanrossum +peps/pep-0208.rst @nascheme @malemburg +# peps/pep-0209.rst +# peps/pep-0210.rst +# peps/pep-0211.rst +# peps/pep-0212.rst +# peps/pep-0213.rst +peps/pep-0214.rst @warsaw +# peps/pep-0215.rst +# peps/pep-0216.rst +# peps/pep-0217.rst +peps/pep-0218.rst @rhettinger +# peps/pep-0219.rst +# peps/pep-0220.rst +peps/pep-0221.rst @Yhg1s +# peps/pep-0222.rst +peps/pep-0223.rst @tim-one +peps/pep-0224.rst @malemburg +# peps/pep-0225.rst +peps/pep-0226.rst @jeremyhylton +peps/pep-0227.rst @jeremyhylton +peps/pep-0228.rst @gvanrossum +# peps/pep-0229.rst +peps/pep-0230.rst @gvanrossum +peps/pep-0231.rst @warsaw +peps/pep-0232.rst @warsaw +# peps/pep-0233.rst +peps/pep-0234.rst @gvanrossum +peps/pep-0235.rst @tim-one +peps/pep-0236.rst @tim-one +peps/pep-0237.rst @gvanrossum +peps/pep-0238.rst @gvanrossum +# peps/pep-0239.rst +# peps/pep-0240.rst +# peps/pep-0241.rst +# peps/pep-0242.rst +# peps/pep-0243.rst +# peps/pep-0244.rst +# peps/pep-0245.rst +peps/pep-0246.rst @aleaxit +# peps/pep-0247.rst +peps/pep-0248.rst @malemburg +peps/pep-0249.rst @malemburg +peps/pep-0250.rst @pfmoore +peps/pep-0251.rst @warsaw @gvanrossum +peps/pep-0252.rst @gvanrossum +peps/pep-0253.rst @gvanrossum +peps/pep-0254.rst @gvanrossum +peps/pep-0255.rst @nascheme @tim-one +# peps/pep-0256.rst +peps/pep-0257.rst @gvanrossum +# peps/pep-0258.rst +peps/pep-0259.rst @gvanrossum +peps/pep-0260.rst @gvanrossum +# peps/pep-0261.rst +# peps/pep-0262.rst +peps/pep-0263.rst @malemburg +# peps/pep-0264.rst +# peps/pep-0265.rst +# peps/pep-0266.rst +peps/pep-0267.rst @jeremyhylton +# peps/pep-0268.rst +# peps/pep-0269.rst +# peps/pep-0270.rst +# peps/pep-0271.rst +# peps/pep-0272.rst +# peps/pep-0273.rst +peps/pep-0274.rst @warsaw +peps/pep-0275.rst @malemburg +# peps/pep-0276.rst +# peps/pep-0277.rst +peps/pep-0278.rst @jackjansen +peps/pep-0279.rst @rhettinger +peps/pep-0280.rst @gvanrossum +# peps/pep-0281.rst +peps/pep-0282.rst @vsajip +peps/pep-0283.rst @gvanrossum +# peps/pep-0284.rst +peps/pep-0285.rst @gvanrossum +# peps/pep-0286.rst +# peps/pep-0287.rst +peps/pep-0288.rst @rhettinger +peps/pep-0289.rst @rhettinger +peps/pep-0290.rst @rhettinger +# peps/pep-0291.rst +peps/pep-0292.rst @warsaw +peps/pep-0293.rst @doerwalter +# peps/pep-0294.rst +# peps/pep-0295.rst +# peps/pep-0296.rst +peps/pep-0297.rst @malemburg +peps/pep-0298.rst @theller +# peps/pep-0299.rst +# peps/pep-0301.rst +peps/pep-0302.rst @pfmoore +# peps/pep-0303.rst +# peps/pep-0304.rst +# peps/pep-0305.rst +peps/pep-0306.rst @jackdied @ncoghlan @benjaminp +peps/pep-0307.rst @gvanrossum @tim-one +peps/pep-0308.rst @gvanrossum @rhettinger +# peps/pep-0309.rst +peps/pep-0310.rst @pfmoore +peps/pep-0311.rst @mhammond +peps/pep-0312.rst @aleaxit +# peps/pep-0313.rst +# peps/pep-0314.rst +peps/pep-0315.rst @rhettinger +# peps/pep-0316.rst +# peps/pep-0317.rst +# peps/pep-0318.rst +# peps/pep-0319.rst +peps/pep-0320.rst @warsaw @rhettinger +# peps/pep-0321.rst +peps/pep-0322.rst @rhettinger +peps/pep-0323.rst @aleaxit +# peps/pep-0324.rst +# peps/pep-0325.rst +peps/pep-0326.rst @terryjreedy +peps/pep-0327.rst @facundobatista +# peps/pep-0328.rst +peps/pep-0329.rst @rhettinger +# peps/pep-0330.rst +# peps/pep-0331.rst +# peps/pep-0332.rst +# peps/pep-0333.rst +# peps/pep-0334.rst +# peps/pep-0335.rst +# peps/pep-0336.rst +# peps/pep-0337.rst +peps/pep-0338.rst @ncoghlan +peps/pep-0339.rst @brettcannon +peps/pep-0340.rst @gvanrossum +peps/pep-0341.rst @birkenfeld +peps/pep-0342.rst @gvanrossum +peps/pep-0343.rst @gvanrossum @ncoghlan +# peps/pep-0344.rst +# peps/pep-0345.rst +peps/pep-0346.rst @ncoghlan +# peps/pep-0347.rst +peps/pep-0348.rst @brettcannon +peps/pep-0349.rst @nascheme +# peps/pep-0350.rst +peps/pep-0351.rst @warsaw +peps/pep-0352.rst @brettcannon @gvanrossum +# peps/pep-0353.rst +# peps/pep-0354.rst +# peps/pep-0355.rst +peps/pep-0356.rst @gvanrossum +# peps/pep-0357.rst +peps/pep-0358.rst @nascheme @gvanrossum +# peps/pep-0359.rst +peps/pep-0360.rst @brettcannon +peps/pep-0361.rst @warsaw +peps/pep-0362.rst @brettcannon @1st1 @larryhastings +# peps/pep-0363.rst +peps/pep-0364.rst @warsaw +# peps/pep-0365.rst +peps/pep-0366.rst @ncoghlan +# peps/pep-0367.rst +# peps/pep-0368.rst +peps/pep-0369.rst @tiran +peps/pep-0370.rst @tiran +# peps/pep-0371.rst +peps/pep-0372.rst @mitsuhiko @rhettinger +peps/pep-0373.rst @benjaminp +peps/pep-0374.rst @brettcannon @avassalotti @warsaw +peps/pep-0375.rst @benjaminp +# peps/pep-0376.rst +peps/pep-0377.rst @ncoghlan +peps/pep-0378.rst @rhettinger +# peps/pep-0379.rst +# peps/pep-0380.rst +# peps/pep-0381.rst +# peps/pep-0382.rst +# peps/pep-0383.rst +# peps/pep-0384.rst +peps/pep-0385.rst @pitrou @birkenfeld +# peps/pep-0386.rst +peps/pep-0387.rst @benjaminp @vstinner +# peps/pep-0389.rst +# peps/pep-0390.rst +peps/pep-0391.rst @vsajip +peps/pep-0392.rst @birkenfeld +# peps/pep-0393.rst +peps/pep-0394.rst @ncoghlan @warsaw @encukou @willingc +peps/pep-0395.rst @ncoghlan +peps/pep-0396.rst @warsaw +peps/pep-0397.rst @mhammond +peps/pep-0398.rst @birkenfeld +peps/pep-0399.rst @brettcannon +peps/pep-0400.rst @vstinner +peps/pep-0401.rst @warsaw @brettcannon +# peps/pep-0402.rst +peps/pep-0403.rst @ncoghlan +peps/pep-0404.rst @warsaw +# peps/pep-0405.rst +peps/pep-0406.rst @ncoghlan +peps/pep-0407.rst @pitrou @birkenfeld @warsaw +peps/pep-0408.rst @ncoghlan @eliben +peps/pep-0409.rst @ethanfurman +peps/pep-0410.rst @vstinner +peps/pep-0411.rst @ncoghlan @eliben +peps/pep-0412.rst @markshannon +peps/pep-0413.rst @ncoghlan +peps/pep-0414.rst @mitsuhiko @ncoghlan +peps/pep-0415.rst @benjaminp +peps/pep-0416.rst @vstinner +peps/pep-0417.rst @voidspace +peps/pep-0418.rst @vstinner +peps/pep-0418/ @vstinner +# peps/pep-0419.rst +peps/pep-0420.rst @ericvsmith +peps/pep-0421.rst @ericsnowcurrently +peps/pep-0422.rst @ncoghlan +# peps/pep-0423.rst +peps/pep-0424.rst @alex +# peps/pep-0425.rst +peps/pep-0426.rst @ncoghlan @dstufft +peps/pep-0426/ @ncoghlan @dstufft +# peps/pep-0427.rst +peps/pep-0428.rst @pitrou +peps/pep-0429.rst @larryhastings +peps/pep-0430.rst @ncoghlan +# peps/pep-0431.rst +peps/pep-0432.rst @ncoghlan @vstinner @ericsnowcurrently +peps/pep-0433.rst @vstinner +peps/pep-0433/ @vstinner +peps/pep-0434.rst @terryjreedy +peps/pep-0435.rst @warsaw @eliben @ethanfurman +peps/pep-0436.rst @larryhastings +# peps/pep-0437.rst +# peps/pep-0438.rst +# peps/pep-0439.rst +peps/pep-0440.rst @ncoghlan @dstufft +peps/pep-0441.rst @pfmoore +peps/pep-0442.rst @pitrou +peps/pep-0443.rst @ambv +peps/pep-0444.rst @mitsuhiko +peps/pep-0445.rst @vstinner +peps/pep-0446.rst @vstinner +peps/pep-0446/ @vstinner +peps/pep-0447.rst @ronaldoussoren +# peps/pep-0448.rst +peps/pep-0449.rst @dstufft +# peps/pep-0450.rst @stevendaprano +peps/pep-0451.rst @ericsnowcurrently +peps/pep-0452.rst @tiran +peps/pep-0453.rst @dstufft @ncoghlan +peps/pep-0454.rst @vstinner +peps/pep-0455.rst @pitrou +peps/pep-0456.rst @tiran +peps/pep-0457.rst @larryhastings +# peps/pep-0458.rst, peps/pep-0458-1.png +peps/pep-0459.rst @ncoghlan +peps/pep-0460.rst @pitrou +peps/pep-0461.rst @ethanfurman +peps/pep-0462.rst @ncoghlan +# peps/pep-0463.rst +peps/pep-0464.rst @dstufft +peps/pep-0465.rst @njsmith +peps/pep-0465/ @njsmith +peps/pep-0466.rst @ncoghlan +peps/pep-0467.rst @ncoghlan @ethanfurman +peps/pep-0468.rst @ericsnowcurrently +peps/pep-0469.rst @ncoghlan +peps/pep-0470.rst @dstufft +# peps/pep-0471.rst +# peps/pep-0472.rst +# peps/pep-0473.rst +peps/pep-0474.rst @ncoghlan +peps/pep-0475.rst @vstinner +peps/pep-0476.rst @alex +peps/pep-0477.rst @dstufft @ncoghlan +peps/pep-0478.rst @larryhastings +peps/pep-0479.rst @gvanrossum +# peps/pep-0480.rst, peps/pep-0480-1.png +peps/pep-0481.rst @dstufft +peps/pep-0482.rst @ambv +peps/pep-0483.rst @gvanrossum @ilevkivskyi +peps/pep-0484.rst @gvanrossum @ambv +# peps/pep-0485.rst +peps/pep-0486.rst @pfmoore +# peps/pep-0487.rst +peps/pep-0488.rst @brettcannon +peps/pep-0489.rst @encukou @scoder @ncoghlan +peps/pep-0490.rst @vstinner +# peps/pep-0491.rst +peps/pep-0492.rst @1st1 +peps/pep-0493.rst @ncoghlan @malemburg +peps/pep-0494.rst @ned-deily +peps/pep-0495.rst @abalkin @tim-one +peps/pep-0495-gap.png @abalkin @tim-one +peps/pep-0495-gap.svg @abalkin @tim-one +peps/pep-0495-fold.svg @abalkin @tim-one +peps/pep-0495-fold-2.png @abalkin @tim-one +peps/pep-0495-daylightsavings.png @abalkin @tim-one +# peps/pep-0496.rst +# peps/pep-0497.rst +peps/pep-0498.rst @ericvsmith +# peps/pep-0499.rst +peps/pep-0500.rst @abalkin @tim-one +peps/pep-0501.rst @ncoghlan +# peps/pep-0502.rst +peps/pep-0503.rst @dstufft +peps/pep-0504.rst @ncoghlan +peps/pep-0505.rst @zooba +peps/pep-0505/ @zooba +# peps/pep-0506.rst @stevendaprano +peps/pep-0507.rst @warsaw +peps/pep-0508.rst @rbtcollins +peps/pep-0509.rst @vstinner +peps/pep-0510.rst @vstinner +peps/pep-0511.rst @vstinner +peps/pep-0512.rst @brettcannon +peps/pep-0513.rst @njsmith +peps/pep-0514.rst @zooba +peps/pep-0515.rst @birkenfeld @serhiy-storchaka +peps/pep-0516.rst @rbtcollins @njsmith +peps/pep-0517.rst @njsmith +peps/pep-0518.rst @brettcannon @njsmith @dstufft +peps/pep-0519.rst @brettcannon +peps/pep-0520.rst @ericsnowcurrently +peps/pep-0521.rst @njsmith +peps/pep-0522.rst @ncoghlan @njsmith +peps/pep-0523.rst @brettcannon @DinoV +peps/pep-0524.rst @vstinner +peps/pep-0525.rst @1st1 +peps/pep-0525-1.png @1st1 +peps/pep-0526.rst @ilevkivskyi @lisroach @gvanrossum +peps/pep-0527.rst @dstufft +peps/pep-0528.rst @zooba +peps/pep-0529.rst @zooba +peps/pep-0530.rst @1st1 +peps/pep-0531.rst @ncoghlan +peps/pep-0532.rst @ncoghlan +peps/pep-0532/ @ncoghlan +peps/pep-0533.rst @njsmith +peps/pep-0534.rst @encukou @ncoghlan +peps/pep-0535.rst @ncoghlan +# peps/pep-0536.rst +peps/pep-0537.rst @ned-deily +peps/pep-0538.rst @ncoghlan +# peps/pep-0539.rst +peps/pep-0540.rst @vstinner +peps/pep-0541.rst @ambv +# peps/pep-0542.rst +peps/pep-0543.rst @tiran +peps/pep-0544.rst @ilevkivskyi @ambv +peps/pep-0545.rst @JulienPalard @methane @vstinner +peps/pep-0546.rst @vstinner +peps/pep-0547.rst @encukou +peps/pep-0548.rst @bitdancer +peps/pep-0549.rst @larryhastings +peps/pep-0550.rst @1st1 +peps/pep-0550-lookup_hamt.png @1st1 +peps/pep-0550-hamt_vs_dict.png @1st1 +peps/pep-0550-hamt_vs_dict-v2.png @1st1 +peps/pep-0551.rst @zooba +peps/pep-0552.rst @benjaminp +peps/pep-0553.rst @warsaw +peps/pep-0554.rst @ericsnowcurrently +# peps/pep-0555.rst +peps/pep-0556.rst @pitrou +peps/pep-0557.rst @ericvsmith +peps/pep-0558.rst @ncoghlan +peps/pep-0559.rst @warsaw +peps/pep-0560.rst @ilevkivskyi +# peps/pep-0561.rst +peps/pep-0562.rst @ilevkivskyi +peps/pep-0563.rst @ambv +peps/pep-0564.rst @vstinner +peps/pep-0565.rst @ncoghlan +# peps/pep-0566.rst +peps/pep-0567.rst @1st1 +peps/pep-0568.rst @njsmith +peps/pep-0569.rst @ambv +peps/pep-0570.rst @larryhastings @pablogsal +# peps/pep-0571.rst +peps/pep-0572.rst @tim-one @gvanrossum +peps/pep-0573.rst @encukou @ncoghlan @ericsnowcurrently +peps/pep-0574.rst @pitrou +# peps/pep-0575.rst +peps/pep-0576.rst @markshannon +peps/pep-0577.rst @ncoghlan +peps/pep-0578.rst @zooba +# peps/pep-0579.rst +# peps/pep-0580.rst +peps/pep-0581.rst @Mariatta +peps/pep-0582.rst @kushaldas @zooba @dstufft @ncoghlan +# peps/pep-0583.rst +peps/pep-0584.rst @brandtbucher # @stevendaprano +peps/pep-0585.rst @ambv +peps/pep-0586.rst @ilevkivskyi +peps/pep-0587.rst @vstinner @ncoghlan +peps/pep-0588.rst @Mariatta +peps/pep-0589.rst @gvanrossum +peps/pep-0590.rst @markshannon +peps/pep-0591.rst @ilevkivskyi +peps/pep-0592.rst @dstufft +peps/pep-0593.rst @ilevkivskyi +peps/pep-0594.rst @tiran @brettcannon +peps/pep-0595.rst @ezio-melotti @berkerpeksag +peps/pep-0596.rst @ambv +peps/pep-0597.rst @methane +peps/pep-0598.rst @ncoghlan +peps/pep-0599.rst @pfmoore +peps/pep-0600.rst @njsmith +peps/pep-0601.rst @isidentical +peps/pep-0602.rst @ambv +peps/pep-0602-example-release-calendar.png @ambv +peps/pep-0602-example-release-calendar.pptx @ambv +peps/pep-0602-overlapping-support-matrix.png @ambv +peps/pep-0602-overlapping-support-matrix.pptx @ambv +peps/pep-0603.rst @1st1 +peps/pep-0603-lookup_hamt.png @1st1 +peps/pep-0603-hamt_vs_dict.png @1st1 +# peps/pep-0604.rst +peps/pep-0605.rst @zooba @ncoghlan +peps/pep-0605-example-release-calendar.png @zooba @ncoghlan +peps/pep-0605-overlapping-support-matrix.png @zooba @ncoghlan +peps/pep-0605/ @zooba @ncoghlan +peps/pep-0606.rst @vstinner +peps/pep-0607.rst @ambv @zooba @ncoghlan +peps/pep-0608.rst @vstinner +peps/pep-0609.rst @pganssle +peps/pep-0610.rst @cjerdonek +peps/pep-0611.rst @markshannon +peps/pep-0612.rst @gvanrossum +peps/pep-0613.rst @gvanrossum +peps/pep-0614.rst @brandtbucher +peps/pep-0615.rst @pganssle +peps/pep-0616.rst @ericvsmith +peps/pep-0617.rst @gvanrossum @pablogsal @lysnikolaou +peps/pep-0618.rst @brandtbucher +peps/pep-0619.rst @pablogsal +peps/pep-0620.rst @vstinner +peps/pep-0621.rst @brettcannon @pganssle +peps/pep-0622.rst @brandtbucher @ilevkivskyi @gvanrossum +peps/pep-0623.rst @methane +peps/pep-0624.rst @methane +peps/pep-0625.rst @pfmoore +peps/pep-0626.rst @markshannon +peps/pep-0627.rst @encukou +peps/pep-0628.rst @ncoghlan +peps/pep-0629.rst @dstufft +peps/pep-0630.rst @encukou +peps/pep-0631.rst @pganssle +peps/pep-0632.rst @zooba +peps/pep-0633.rst @brettcannon +peps/pep-0634.rst @brandtbucher @gvanrossum +peps/pep-0635.rst @brandtbucher @gvanrossum +peps/pep-0636.rst @brandtbucher @gvanrossum +# peps/pep-0637.rst @stevendaprano +peps/pep-0638.rst @markshannon +peps/pep-0639.rst @CAM-Gerlach +peps/pep-0640.rst @Yhg1s +peps/pep-0641.rst @zooba @warsaw @brettcannon +peps/pep-0642.rst @ncoghlan +peps/pep-0643.rst @pfmoore +peps/pep-0644.rst @tiran +peps/pep-0645.rst @gvanrossum +peps/pep-0646.rst @gvanrossum +peps/pep-0647.rst @gvanrossum +peps/pep-0648.rst @pablogsal +peps/pep-0649.rst @larryhastings +peps/pep-0650.rst @brettcannon +peps/pep-0651.rst @markshannon +peps/pep-0652.rst @encukou +peps/pep-0653.rst @markshannon +peps/pep-0654.rst @1st1 @gvanrossum @iritkatriel +peps/pep-0655.rst @gvanrossum +peps/pep-0656.rst @brettcannon +peps/pep-0657.rst @pablogsal @isidentical @ammaraskar +peps/pep-0658.rst @brettcannon +peps/pep-0659.rst @markshannon +peps/pep-0660.rst @pfmoore +peps/pep-0661.rst @taleinat +peps/pep-0662.rst @brettcannon +peps/pep-0662/ @brettcannon +peps/pep-0663.rst @ethanfurman +peps/pep-0664.rst @pablogsal +peps/pep-0665.rst @brettcannon +# peps/pep-0666.rst +peps/pep-0667.rst @markshannon +peps/pep-0668.rst @dstufft +peps/pep-0669.rst @markshannon +peps/pep-0670.rst @vstinner @erlend-aasland +peps/pep-0671.rst @rosuav +peps/pep-0672.rst @encukou +peps/pep-0673.rst @jellezijlstra +peps/pep-0674.rst @vstinner +peps/pep-0675.rst @jellezijlstra +peps/pep-0676.rst @AA-Turner @Mariatta +peps/pep-0677.rst @gvanrossum +peps/pep-0678.rst @iritkatriel +peps/pep-0679.rst @pablogsal +peps/pep-0680.rst @encukou +peps/pep-0681.rst @jellezijlstra +peps/pep-0682.rst @mdickinson +peps/pep-0683.rst @ericsnowcurrently +peps/pep-0684.rst @ericsnowcurrently +# peps/pep-0684.rst +peps/pep-0685.rst @brettcannon +peps/pep-0686.rst @methane +peps/pep-0687.rst @encukou @erlend-aasland +peps/pep-0688.rst @jellezijlstra +peps/pep-0689.rst @encukou +peps/pep-0690.rst @warsaw +peps/pep-0691.rst @dstufft +peps/pep-0692.rst @jellezijlstra +peps/pep-0693.rst @Yhg1s +peps/pep-0694.rst @dstufft +peps/pep-0695.rst @gvanrossum +peps/pep-0696.rst @jellezijlstra +peps/pep-0697.rst @encukou +peps/pep-0698.rst @jellezijlstra +peps/pep-0699.rst @Fidget-Spinner +peps/pep-0700.rst @pfmoore +peps/pep-0701.rst @pablogsal @isidentical @lysnikolaou +peps/pep-0702.rst @jellezijlstra +peps/pep-0703.rst @ambv +peps/pep-0704.rst @brettcannon @pradyunsg +peps/pep-0705.rst @pablogsal +peps/pep-0706.rst @encukou +peps/pep-0707.rst @iritkatriel +peps/pep-0708.rst @dstufft +peps/pep-0709.rst @carljm +peps/pep-0710.rst @dstufft +peps/pep-0711.rst @njsmith +peps/pep-0712.rst @ericvsmith +peps/pep-0713.rst @ambv +peps/pep-0714.rst @dstufft +peps/pep-0715.rst @dstufft +peps/pep-0718.rst @gvanrossum +peps/pep-0719.rst @Yhg1s +peps/pep-0720.rst @FFY00 +peps/pep-0721.rst @encukou +peps/pep-0722.rst @pfmoore +peps/pep-0723.rst @AA-Turner +peps/pep-0724.rst @jellezijlstra +peps/pep-0725.rst @pradyunsg +peps/pep-0726.rst @AA-Turner +peps/pep-0727.rst @JelleZijlstra # ... -# pep-0754.txt +# peps/pep-0754.rst # ... -pep-0801.rst @warsaw +peps/pep-0801.rst @warsaw # ... -pep-3000.txt @gvanrossum -pep-3001.txt @birkenfeld -# pep-3002.txt -pep-3003.txt @brettcannon @gvanrossum +peps/pep-3000.rst @gvanrossum +peps/pep-3001.rst @birkenfeld +# peps/pep-3002.rst +peps/pep-3003.rst @brettcannon @gvanrossum # ... -pep-3099.txt @birkenfeld -pep-3100.txt @brettcannon -# pep-3101.txt -# pep-3102.txt -pep-3103.txt @gvanrossum -# pep-3104.txt -pep-3105.txt @birkenfeld -pep-3106.txt @gvanrossum -# pep-3107.txt -pep-3108.txt @brettcannon -# pep-3109.txt -# pep-3110.txt -# pep-3111.txt -# pep-3112.txt -pep-3113.txt @brettcannon -# pep-3114.txt -# pep-3115.txt -pep-3116.txt @gvanrossum -pep-3117.txt @birkenfeld -# pep-3118.txt -pep-3119.txt @gvanrossum -# pep-3120.txt -# pep-3121.txt -pep-3122.txt @brettcannon -# pep-3123.txt -pep-3124.txt @pjeby -# pep-3125.txt -pep-3126.txt @rhettinger -# pep-3127.txt -# pep-3128.txt -# pep-3129.txt -# pep-3130.txt -# pep-3131.txt -pep-3132.txt @birkenfeld -# pep-3133.txt -# pep-3134.txt -# pep-3135.txt -# pep-3136.txt -pep-3137.txt @gvanrossum -# pep-3138.txt -pep-3139.txt @benjaminp -# pep-3140.txt -# pep-3141.txt -# pep-3142.txt -# pep-3143.txt -# pep-3144.txt -# pep-3145.txt -# pep-3146.txt -pep-3147.txt @warsaw -pep-3147-1.dia @warsaw -pep-3147-1.png @warsaw -pep-3148.txt @brianquinlan -pep-3149.txt @warsaw -pep-3150.txt @ncoghlan -pep-3151.txt @pitrou -# pep-3152.txt -# pep-3153.txt -pep-3154.txt @pitrou -pep-3155.txt @pitrou -pep-3156.txt @gvanrossum +peps/pep-3099.rst @birkenfeld +peps/pep-3100.rst @brettcannon +# peps/pep-3101.rst +# peps/pep-3102.rst +peps/pep-3103.rst @gvanrossum +# peps/pep-3104.rst +peps/pep-3105.rst @birkenfeld +peps/pep-3106.rst @gvanrossum +# peps/pep-3107.rst +peps/pep-3108.rst @brettcannon +# peps/pep-3109.rst +# peps/pep-3110.rst +# peps/pep-3111.rst +# peps/pep-3112.rst +peps/pep-3113.rst @brettcannon +# peps/pep-3114.rst +# peps/pep-3115.rst +peps/pep-3116.rst @gvanrossum +peps/pep-3117.rst @birkenfeld +# peps/pep-3118.rst +peps/pep-3119.rst @gvanrossum +# peps/pep-3120.rst +# peps/pep-3121.rst +peps/pep-3122.rst @brettcannon +# peps/pep-3123.rst +# peps/pep-3124.rst +# peps/pep-3125.rst +peps/pep-3126.rst @rhettinger +# peps/pep-3127.rst +# peps/pep-3128.rst +# peps/pep-3129.rst +# peps/pep-3130.rst +# peps/pep-3131.rst +peps/pep-3132.rst @birkenfeld +# peps/pep-3133.rst +# peps/pep-3134.rst +# peps/pep-3135.rst +# peps/pep-3136.rst +peps/pep-3137.rst @gvanrossum +# peps/pep-3138.rst +peps/pep-3139.rst @benjaminp +# peps/pep-3140.rst +# peps/pep-3141.rst +# peps/pep-3142.rst +# peps/pep-3143.rst +# peps/pep-3144.rst +# peps/pep-3145.rst +# peps/pep-3146.rst +peps/pep-3147.rst @warsaw +peps/pep-3147-1.dia @warsaw +peps/pep-3147-1.png @warsaw +peps/pep-3148.rst @brianquinlan +peps/pep-3149.rst @warsaw +peps/pep-3150.rst @ncoghlan +peps/pep-3151.rst @pitrou +# peps/pep-3152.rst +# peps/pep-3153.rst +peps/pep-3154.rst @pitrou +peps/pep-3155.rst @pitrou +peps/pep-3156.rst @gvanrossum # ... -pep-3333.txt @pjeby +# peps/pep-3333.rst # ... -pep-8000.rst @warsaw -pep-8001.rst @brettcannon @tiran @dstufft @ericsnowcurrently @gpshead @ambv @Mariatta @njsmith @pablogsal @rhettinger @taleinat @tim-one @zware -pep-8002.rst @warsaw @ambv @pitrou @dhellmann @willingc -pep-8010.rst @warsaw -pep-8011.rst @Mariatta @warsaw -pep-8012.rst @ambv -pep-8013.rst @zooba -pep-8014.rst @jackjansen -pep-8015.rst @vstinner -pep-8016.rst @njsmith @dstufft +peps/pep-8000.rst @warsaw +peps/pep-8001.rst @brettcannon @tiran @dstufft @ericsnowcurrently @gpshead @ambv @Mariatta @njsmith @pablogsal @rhettinger @taleinat @tim-one @zware +peps/pep-8002.rst @warsaw @ambv @pitrou @dhellmann @willingc +peps/pep-8010.rst @warsaw +peps/pep-8011.rst @Mariatta @warsaw +peps/pep-8012.rst @ambv +peps/pep-8013.rst @zooba +peps/pep-8014.rst @jackjansen +peps/pep-8015.rst @vstinner +peps/pep-8016.rst @njsmith @dstufft # ... -pep-8100.rst @njsmith -# pep-8101.rst -# pep-8102.rst +peps/pep-8100.rst @njsmith +# peps/pep-8101.rst +# peps/pep-8102.rst diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 3c4934ac8..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,11 +0,0 @@ - diff --git a/.github/PULL_REQUEST_TEMPLATE/Add a new PEP.md b/.github/PULL_REQUEST_TEMPLATE/Add a new PEP.md new file mode 100644 index 000000000..48aaa3072 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/Add a new PEP.md @@ -0,0 +1,42 @@ + + +## Basic requirements (all PEP Types) + +* [ ] Read and followed [PEP 1](https://peps.python.org/1) & [PEP 12](https://peps.python.org/12) +* [ ] File created from the [latest PEP template](https://github.com/python/peps/blob/main/peps/pep-0012/pep-NNNN.rst?plain=1) +* [ ] PEP has next available number, & set in filename (``pep-NNNN.rst``), PR title (``PEP 123: ``) and ``PEP`` header +* [ ] Title clearly, accurately and concisely describes the content in 79 characters or less +* [ ] Core dev/PEP editor listed as ``Author`` or ``Sponsor``, and formally confirmed their approval +* [ ] ``Author``, ``Status`` (``Draft``), ``Type`` and ``Created`` headers filled out correctly +* [ ] ``PEP-Delegate``, ``Topic``, ``Requires`` and ``Replaces`` headers completed if appropriate +* [ ] Required sections included + * [ ] Abstract (first section) + * [ ] Copyright (last section; exact wording from template required) +* [ ] Code is well-formatted (PEP 7/PEP 8) and is in [code blocks, with the right lexer names](https://peps.python.org/pep-0012/#literal-blocks) if non-Python +* [ ] PEP builds with no warnings, pre-commit checks pass and content displays as intended in the rendered HTML +* [ ] Authors/sponsor added to ``.github/CODEOWNERS`` for the PEP + + +## Standards Track requirements + +* [ ] PEP topic [discussed in a suitable venue](https://peps.python.org/pep-0001/#start-with-an-idea-for-python) with general agreement that a PEP is appropriate +* [ ] [Suggested sections](https://peps.python.org/pep-0012/#suggested-sections) included (unless not applicable) + * [ ] Motivation + * [ ] Rationale + * [ ] Specification + * [ ] Backwards Compatibility + * [ ] Security Implications + * [ ] How to Teach This + * [ ] Reference Implementation + * [ ] Rejected Ideas + * [ ] Open Issues +* [ ] ``Python-Version`` set to valid (pre-beta) future Python version, if relevant +* [ ] Any project stated in the PEP as supporting/endorsing/benefiting from the PEP formally confirmed such +* [ ] Right before or after initial merging, [PEP discussion thread](https://peps.python.org/pep-0001/#discussing-a-pep) created and linked to in ``Discussions-To`` and ``Post-History`` diff --git a/.github/PULL_REQUEST_TEMPLATE/Change an existing PEP.md b/.github/PULL_REQUEST_TEMPLATE/Change an existing PEP.md new file mode 100644 index 000000000..186b10438 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/Change an existing PEP.md @@ -0,0 +1,10 @@ +<!-- +**Please** read our Contributing Guidelines (CONTRIBUTING.rst) +to make sure this repo is the right place for your proposed change. Thanks! +--> + +* Change is either: + * [ ] To a Draft PEP + * [ ] To an Accepted or Final PEP, with Steering Council approval + * [ ] To fix an editorial issue (markup, typo, link, header, etc) +* [ ] PR title prefixed with PEP number (e.g. ``PEP 123: Summary of changes``) diff --git a/.github/PULL_REQUEST_TEMPLATE/Mark a PEP Accepted or Rejected.md b/.github/PULL_REQUEST_TEMPLATE/Mark a PEP Accepted or Rejected.md new file mode 100644 index 000000000..1aca40d5a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/Mark a PEP Accepted or Rejected.md @@ -0,0 +1,13 @@ +<!-- +You can help complete the following checklist yourself if you like +by ticking any boxes you're sure about, like this: [x] + +If you're unsure about anything, just leave it blank and we'll take a look. +--> + +* [ ] SC/PEP Delegate has formally accepted/rejected the PEP and posted to the ``Discussions-To`` thread +* [ ] Pull request title in appropriate format (``PEP 123: Mark as Accepted``) +* [ ] ``Status`` changed to ``Accepted``/``Rejected`` +* [ ] ``Resolution`` link points directly to SC/PEP Delegate official acceptance/rejected post +* [ ] Acceptance/rejection notice added, if the SC/PEP delegate had major conditions or comments +* [ ] ``Discussions-To``, ``Post-History`` and ``Python-Version`` up to date diff --git a/.github/PULL_REQUEST_TEMPLATE/Mark a PEP Final.md b/.github/PULL_REQUEST_TEMPLATE/Mark a PEP Final.md new file mode 100644 index 000000000..25463c773 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/Mark a PEP Final.md @@ -0,0 +1,12 @@ +<!-- +You can help complete the following checklist yourself if you like +by ticking any boxes you're sure about, like this: [x] +If you're unsure about something, just leave it blank and we'll take a look. +--> + +* [ ] Final implementation has been merged (including tests and docs) +* [ ] PEP matches the final implementation +* [ ] Any substantial changes since the accepted version approved by the SC/PEP delegate +* [ ] Pull request title in appropriate format (``PEP 123: Mark Final``) +* [ ] ``Status`` changed to ``Final`` (and ``Python-Version`` is correct) +* [ ] Canonical docs/spec linked with a ``canonical-doc`` directive (or ``canonical-pypa-spec``, for packaging PEPs) diff --git a/.github/PULL_REQUEST_TEMPLATE/Other - infra or meta change.md b/.github/PULL_REQUEST_TEMPLATE/Other - infra or meta change.md new file mode 100644 index 000000000..34839dab6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/Other - infra or meta change.md @@ -0,0 +1,5 @@ +<!-- +This template is for an infra or meta change not belonging to another category. +**Please** read our Contributing Guidelines (CONTRIBUTING.rst) +to make sure this repo is the right place for your proposed change. Thanks! +--> diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index ee5a3d4f3..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install -U pip - python -m pip install -U docutils - - - name: Build - run: | - make rss - make -j$(nproc) - - - name: Deploy - if: > - ( - github.repository == 'python/peps' && - github.ref == 'refs/heads/master' - ) - run: | - bash deploy.bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/deploy-gh-pages.yaml b/.github/workflows/deploy-gh-pages.yaml deleted file mode 100644 index 4cfe2a6b6..000000000 --- a/.github/workflows/deploy-gh-pages.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: Deploy to GitHub Pages - -on: - push: - branches: [master] - -jobs: - deploy-to-pages: - runs-on: ubuntu-latest - - steps: - - name: đŸ›Žïž Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 # fetch all history so that last modified date-times are accurate - - - name: 🐍 Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - - name: 🧳 Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: đŸ‘·â€ Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: 🔧 Build PEPs - run: make pages -j$(nproc) - - # remove the .doctrees folder when building for deployment as it takes two thirds of disk space - - name: đŸ”„ Clean up files - run: rm -r build/.doctrees/ - - - name: 🚀 Deploy to GitHub pages - uses: JamesIves/github-pages-deploy-action@4.1.1 - with: - branch: gh-pages # The branch to deploy to. - folder: build # Synchronise with build.py -> build_directory - single-commit: true # Delete existing files diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 000000000..fc9f44a2b --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,23 @@ +name: Read the Docs PR preview + +on: + pull_request_target: + types: + - opened + +permissions: + contents: read + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "pep-previews" + single-version: "true" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 00faf27fb..fc096a0cd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,11 +1,52 @@ -name: Lint +name: Lint PEPs -on: [push, pull_request] +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + RUFF_FORMAT: github jobs: pre-commit: + name: Run pre-commit runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 + - uses: actions/checkout@v4 + - name: Set up Python 3 + uses: actions/setup-python@v4 + with: + python-version: "3.x" + cache: pip + + - name: Run pre-commit hooks + uses: pre-commit/action@v3.0.0 + + - name: Check spelling + uses: pre-commit/action@v3.0.0 + with: + extra_args: --all-files --hook-stage manual codespell || true + + check-peps: + name: Run check-peps + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3 + uses: actions/setup-python@v4 + with: + python-version: "3" + + - name: Run check-peps + run: python check-peps.py --detailed diff --git a/.github/workflows/render.yml b/.github/workflows/render.yml new file mode 100644 index 000000000..98dcbf730 --- /dev/null +++ b/.github/workflows/render.yml @@ -0,0 +1,68 @@ +name: Render PEPs + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + render-peps: + name: Render PEPs + runs-on: ubuntu-latest + permissions: + contents: write + strategy: + fail-fast: false + matrix: + python-version: + - "3.x" + - "3.12-dev" + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch all history so that last modified date-times are accurate + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Update pip + run: | + python -m pip install --upgrade pip + + - name: Render PEPs + run: make dirhtml JOBS=$(nproc) + + # remove the .doctrees folder when building for deployment as it takes two thirds of disk space + - name: Clean up files + run: rm -r build/.doctrees/ + + - name: Deploy to GitHub pages + # This allows CI to build branches for testing + if: (github.ref == 'refs/heads/main') && (matrix.python-version == '3.x') + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: build # Synchronise with Makefile -> OUTPUT_DIR + single-commit: true # Delete existing files + + - name: Purge CDN cache + if: github.ref == 'refs/heads/main' + run: | + curl -H "Accept: application/json" -H "Fastly-Key: $FASTLY_TOKEN" -X POST "https://api.fastly.com/service/$FASTLY_SERVICE_ID/purge_all" + env: + FASTLY_TOKEN: ${{ secrets.FASTLY_TOKEN }} + FASTLY_SERVICE_ID: ${{ secrets.FASTLY_SERVICE_ID }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..e71847944 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,64 @@ +name: Test Sphinx Extensions + +on: + push: + paths: + - ".github/workflows/test.yml" + - "pep_sphinx_extensions/**" + - "tox.ini" + pull_request: + paths: + - ".github/workflows/test.yml" + - "pep_sphinx_extensions/**" + - "tox.ini" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: + - "3.9" + - "3.10" + - "3.11" + - "3.12-dev" + os: + - "windows-latest" + - "macos-latest" + - "ubuntu-latest" + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install -U wheel + python -m pip install -U tox + + - name: Run tests + run: | + tox -e py -- -v --cov-report term + + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + flags: ${{ matrix.os }} + name: ${{ matrix.os }} Python ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index b9c892157..6beae9d8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,24 @@ -pep-0000.txt +# PEPs pep-0000.rst -pep-????.html peps.rss +topic +/build + +# Bytecode __pycache__ -*.pyc -*.pyo +*.py[co] + +# Editors *~ -*env +.idea .vscode *.swp -/build -/package + +# Tests +coverage.xml +.coverage +.tox + +# Virtual environments +*env /venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d339a4fd4..b1c1602fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,78 +1,231 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 - hooks: - - id: mixed-line-ending - name: Normalize mixed line endings - args: [--fix=lf] +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +minimum_pre_commit_version: '2.8.2' + +default_language_version: + python: python3 + +default_stages: [commit] + + +repos: + # General file checks and fixers + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: mixed-line-ending + name: "Normalize mixed line endings" + args: [--fix=lf] + - id: file-contents-sorter + name: "Sort codespell ignore list" + files: '.codespell/ignore-words.txt' + + - id: check-case-conflict + name: "Check for case conflicts" + - id: check-merge-conflict + name: "Check for merge conflict markers" + - id: check-executables-have-shebangs + name: "Check that executables have shebangs" + - id: check-shebang-scripts-are-executable + name: "Check that shebangs are executable" + + - id: check-vcs-permalinks + name: "Check that VCS links are permalinks" + + # - id: check-ast + # name: "Check Python AST" + - id: check-json + name: "Check JSON" + - id: check-toml + name: "Check TOML" + - id: check-yaml + name: "Check YAML" + + - repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + name: "Format with Black" + args: + - '--target-version=py39' + - '--target-version=py310' + files: 'pep_sphinx_extensions/tests/.*' + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.287 + hooks: + - id: ruff + name: "Lint with Ruff" + args: + - '--exit-non-zero-on-fix' + files: '^pep_sphinx_extensions/tests/' + + - repo: https://github.com/tox-dev/tox-ini-fmt + rev: 1.3.1 + hooks: + - id: tox-ini-fmt + name: "Format tox.ini" + + - repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v0.6.8 + hooks: + - id: sphinx-lint + name: "Sphinx lint" + args: ["--disable=trailing-whitespace"] + + # RST checks - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.8.0 + rev: v1.10.0 hooks: - id: rst-backticks - - id: rst-inline-touching-normal - files: '^pep-\d+\.txt|\.rst$' - types: [text] - - id: rst-directive-colons - files: '^pep-\d+\.txt|\.rst$' - types: [text] + name: "Check RST: No single backticks" + - id: rst-inline-touching-normal + name: "Check RST: No backticks touching text" + + - id: rst-directive-colons + name: "Check RST: 2 colons after directives" + + # Manual codespell check + - repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell + name: "Check for common misspellings in text files" + stages: [manual] + + # Local checks for PEP headers and more - repo: local hooks: - - id: check-required-fields - name: "Check all PEPs have required fields" +# # Hook to run "check-peps.py" +# - id: "check-peps" +# name: "Check PEPs for metadata and content enforcement" +# entry: "python check-peps.py" +# language: "system" +# files: "^pep-\d{4}\.(rst|txt)$" +# require_serial: true + + - id: check-required-headers + name: "PEPs must have all required headers" language: pygrep entry: '(?-m:^PEP:(?=[\s\S]*\nTitle:)(?=[\s\S]*\nAuthor:)(?=[\s\S]*\nStatus:)(?=[\s\S]*\nType:)(?=[\s\S]*\nContent-Type:)(?=[\s\S]*\nCreated:))' args: ['--negate', '--multiline'] - files: '^pep-\d+\.(rst|txt)$' - types: [text] + files: '^peps/pep-\d+\.rst$' + + - id: check-header-order + name: "PEP header order must follow PEP 12" + language: pygrep + entry: '^PEP:[^\n]+\nTitle:[^\n]+\n(Version:[^\n]+\n)?(Last-Modified:[^\n]+\n)?Author:[^\n]+\n( +\S[^\n]+\n)*(Sponsor:[^\n]+\n)?((PEP|BDFL)-Delegate:[^\n]*\n)?(Discussions-To:[^\n]*\n)?Status:[^\n]+\nType:[^\n]+\n(Topic:[^\n]+\n)?Content-Type:[^\n]+\n(Requires:[^\n]+\n)?Created:[^\n]+\n(Python-Version:[^\n]*\n)?(Post-History:[^\n]*\n( +\S[^\n]*\n)*)?(Replaces:[^\n]+\n)?(Superseded-By:[^\n]+\n)?(Resolution:[^\n]*\n)?\n' + args: ['--negate', '--multiline'] + files: '^peps/pep-\d+\.rst$' + - id: validate-pep-number - name: "Validate PEP number field" + name: "'PEP' header must be a number 1-9999" language: pygrep entry: '(?-m:^PEP:(?:(?! +(0|[1-9][0-9]{0,3})\n)))' args: ['--multiline'] - files: '^pep-\d+\.(rst|txt)$' - types: [text] + files: '^peps/pep-\d+\.rst$' + + - id: validate-title + name: "'Title' must be 1-79 characters" + language: pygrep + entry: '(?<=\n)Title:(?:(?! +\S.{1,78}\n(?=[A-Z])))' + args: ['--multiline'] + files: '^peps/pep-\d+\.rst$' + exclude: '^peps/pep-(0499)\.rst$' + + - id: validate-author + name: "'Author' must be list of 'Name <email@example.com>, ...'" + language: pygrep + entry: '(?<=\n)Author:(?:(?!((( +|\n {1,8})[^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+( <[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+>)?)(,|(?=\n[^ ])))+\n(?=[A-Z])))' + args: ["--multiline"] + files: '^peps/pep-\d+\.rst$' + + - id: validate-sponsor + name: "'Sponsor' must have format 'Name <email@example.com>'" + language: pygrep + entry: '^Sponsor:(?: (?! *[^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+( <[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+>)?$))' + files: '^peps/pep-\d+\.rst$' + + - id: validate-delegate + name: "'Delegate' must have format 'Name <email@example.com>'" + language: pygrep + entry: '^(PEP|BDFL)-Delegate: (?:(?! *[^!#$%&()*+,/:;<=>?@\[\\\]\^_`{|}~]+( <[\w!#$%&''*+\-/=?^_{|}~.]+(@| at )[\w\-.]+\.[A-Za-z0-9]+>)?$))' + files: '^peps/pep-\d+\.rst$' + exclude: '^peps/pep-(0451)\.rst$' + + - id: validate-discussions-to + name: "'Discussions-To' must be a thread URL" + language: pygrep + entry: '^Discussions-To: (?:(?!([\w\-]+@(python\.org|googlegroups\.com))|https://((discuss\.python\.org/t/([\w\-]+/)?\d+/?)|(mail\.python\.org/pipermail/[\w\-]+/\d{4}-[A-Za-z]+/[A-Za-z0-9]+\.html)|(mail\.python\.org/archives/list/[\w\-]+@python\.org/thread/[A-Za-z0-9]+/?))$))' + files: '^peps/pep-\d+\.rst$' + - id: validate-status - name: "Validate PEP Status field" + name: "'Status' must be a valid PEP status" language: pygrep entry: '^Status:(?:(?! +(Draft|Withdrawn|Rejected|Accepted|Final|Active|Provisional|Deferred|Superseded|April Fool!)$))' - files: '^pep-\d+\.(rst|txt)$' - types: [text] + files: '^peps/pep-\d+\.rst$' + - id: validate-type - name: "Validate PEP Type field" + name: "'Type' must be a valid PEP type" language: pygrep entry: '^Type:(?:(?! +(Standards Track|Informational|Process)$))' - files: '^pep-\d+\.(rst|txt)$' - types: [text] + files: '^peps/pep-\d+\.rst$' + + - id: validate-topic + name: "'Topic' must be for a valid sub-index" + language: pygrep + entry: '^Topic:(?:(?! +(Governance|Packaging|Typing|Release)(, (Governance|Packaging|Typing|Release))*$))' + files: '^peps/pep-\d+\.rst$' + - id: validate-content-type - name: "Validate PEP Content-Type field" + name: "'Content-Type' must be 'text/x-rst'" language: pygrep - entry: '^Content-Type:(?:(?! +text\/x-rst$))' - files: '^pep-\d+\.(rst|txt)$' - types: [text] + entry: '^Content-Type:(?:(?! +text/x-rst$))' + files: '^peps/pep-\d+\.rst$' + - id: validate-pep-references - name: "Validate PEP reference fields" + name: "`Requires`/`Replaces`/`Superseded-By` must be 'NNN' PEP IDs" language: pygrep - entry: '^(Requires|Replaces|Superseded-By):(?:(?! +( ?(0|[1-9][0-9]{0,3}),?)+$))' - files: '^pep-\d+\.(rst|txt)$' - types: [text] + entry: '^(Requires|Replaces|Superseded-By):(?:(?! *( (0|[1-9][0-9]{0,3})(,|$))+$))' + files: '^peps/pep-\d+\.rst$' + - id: validate-created - name: "Validate created dates" + name: "'Created' must be a 'DD-mmm-YYYY' date" language: pygrep - entry: '^Created:(?:(?! +([0-2][0-9]|(3[01]))-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(199[0-9]|20[0-9][0-9])( \([^()]+\))?$))' - files: '^pep-\d+\.(rst|txt)$' - types: [text] + entry: '^Created:(?:(?! +([0-2][0-9]|(3[01]))-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(199[0-9]|20[0-9][0-9])$))' + files: '^peps/pep-\d+\.rst$' + - id: validate-python-version - name: "Validate PEP Python-Version field" + name: "'Python-Version' must be a 'X.Y[.Z]` version" language: pygrep - entry: '^Python-Version:(?:(?! +( ?[1-9]\.([0-9][0-9]?|x)(\.[1-9][0-9]?)?\??,?)+( \([^()]+\))?$))' - files: '^pep-\d+\.(rst|txt)$' - types: [text] + entry: '^Python-Version:(?:(?! *( [1-9]\.([0-9][0-9]?|x)(\.[1-9][0-9]?)?(,|$))+$))' + files: '^peps/pep-\d+\.rst$' + + - id: validate-post-history + name: "'Post-History' must be '`DD-mmm-YYYY <Thread URL>`__, ...'" + language: pygrep + entry: '(?<=\n)Post-History:(?:(?! ?\n|((( +|\n {1,14})(([0-2][0-9]|(3[01]))-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(199[0-9]|20[0-9][0-9])|`([0-2][0-9]|(3[01]))-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(199[0-9]|20[0-9][0-9]) <https://((discuss\.python\.org/t/([\w\-]+/)?\d+(?:/\d+/|/?))|(mail\.python\.org/pipermail/[\w\-]+/\d{4}-[A-Za-z]+/[A-Za-z0-9]+\.html)|(mail\.python\.org/archives/list/[\w\-]+@python\.org/thread/[A-Za-z0-9]+/?(#[A-Za-z0-9]+)?))>`__)(,|(?=\n[^ ])))+\n(?=[A-Z\n]))))' + args: [--multiline] + files: '^peps/pep-\d+\.rst$' + - id: validate-resolution - name: "Validate PEP Resolution field" + name: "'Resolution' must be a direct thread/message URL" language: pygrep - entry: '(?<!\n\n)^Resolution: (?:(?!https:\/\/\S*\n))' + entry: '(?<!\n\n)(?<=\n)Resolution: (?:(?!https://((discuss\.python\.org/t/([\w\-]+/)?\d+(/\d+)?/?)|(mail\.python\.org/pipermail/[\w\-]+/\d{4}-[A-Za-z]+/[A-Za-z0-9]+\.html)|(mail\.python\.org/archives/list/[\w\-]+@python\.org/(message|thread)/[A-Za-z0-9]+/?(#[A-Za-z0-9]+)?))\n))' args: ['--multiline'] - files: '^pep-\d+\.(rst|txt)$' - types: [text] + files: '^peps/pep-\d+\.rst$' + + - id: check-direct-pep-links + name: "Check that PEPs aren't linked directly" + language: pygrep + entry: '(dev/peps|peps\.python\.org)/pep-\d+' + files: '^peps/pep-\d+\.rst$' + exclude: '^peps/pep-(0009|0287|0676|0684|8001)\.rst$' + + - id: check-direct-rfc-links + name: "Check that RFCs aren't linked directly" + language: pygrep + entry: '(rfc-editor\.org|ietf\.org)/[\.\-_\?\&\#\w/]*[Rr][Ff][Cc][\-_]?\d+' + types: ['rst'] diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 000000000..5ab2337f9 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,15 @@ +ignore = [ + "E501", # Line too long +] + +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "PT", # flake8-pytest-style + "W", # pycodestyle warnings +] + +show-source = true + +target-version = "py39" diff --git a/AUTHOR_OVERRIDES.csv b/AUTHOR_OVERRIDES.csv deleted file mode 100644 index c97178f0c..000000000 --- a/AUTHOR_OVERRIDES.csv +++ /dev/null @@ -1,11 +0,0 @@ -Overridden Name,Surname First,Name Reference -The Python core team and community,The Python core team and community,python-dev -Ernest W. Durbin III,"Durbin, Ernest W., III",Durbin -Greg Ewing,"Ewing, Gregory",Ewing -Guido van Rossum,"van Rossum, Guido (GvR)",GvR -Inada Naoki,"Inada, Naoki",Inada -Jim Jewett,"Jewett, Jim J.",Jewett -Just van Rossum,"van Rossum, Just (JvR)",JvR -Martin v. Löwis,"von Löwis, Martin",von Löwis -Nathaniel Smith,"Smith, Nathaniel J.",Smith -P.J. Eby,"Eby, Phillip J.",Eby diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 14c6e4fc2..000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,13 +0,0 @@ -# Code of Conduct - - -Please note that all interactions on -[Python Software Foundation](https://www.python.org/psf-landing/)-supported -infrastructure is -[covered](https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties) -by the [PSF Code of Conduct](https://www.python.org/psf/codeofconduct/), -which includes all infrastructure used in the development of Python itself -(e.g. mailing lists, issue trackers, GitHub, etc.). - -In general this means everyone is expected to be open, considerate, and -respectful of others no matter what their position is within the project. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8f01b5802..bf58bd2c4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,47 +1,71 @@ Contributing Guidelines ======================= -To learn more about the purpose of PEPs and how to go about writing a PEP, please -start reading at PEP 1 (`pep-0001.txt <./pep-0001.txt>`_ in this repo). Note that -PEP 0, the index PEP, is now automatically generated, and not committed to the repo. +To learn more about the purpose of PEPs and how to go about writing one, please +start reading at `PEP 1 <https://peps.python.org/pep-0001/>`_. +Also, make sure to check the `README <./README.rst>`_ for information +on how to render the PEPs in this repository. +Thanks again for your contributions, and we look forward to reviewing them! + Before writing a new PEP ------------------------ -Has this idea been proposed on `python-ideas <https://mail.python.org/mailman/listinfo/python-ideas>`_ -and received general acceptance as being an idea worth pursuing? (if not then -please start a discussion there before submitting a pull request). - -More details about it in `PEP 1 <https://www.python.org/dev/peps/pep-0001/#start-with-an-idea-for-python>`_. - -Do you have an implementation of your idea? (this is important for when you -propose this PEP to `python-dev <https://mail.python.org/mailman/listinfo/python-dev>`_ -as code maintenance is a critical aspect of all PEP proposals prior to a -final decision; in special circumstances an implementation can be deferred) +Prior to submitting a pull request here with your draft PEP, see `PEP 1 +<https://peps.python.org/pep-0001/#start-with-an-idea-for-python>`_ +for some important steps to consider, including proposing and discussing it +first in an appropriate venue, drafting a PEP and gathering feedback, and +developing at least a prototype reference implementation of your idea. -Commit messages ---------------- +Contributing changes to existing PEPs +------------------------------------- -When committing to a PEP, please always include the PEP number in the subject -title. For example, ``PEP NNN: <summary of changes>``. +In general, most non-Draft/Active PEPs are considered to be historical +documents rather than living specifications or documentation. Major changes to +their core content usually require a new PEP, while smaller modifications may +or may not be appropriate, depending on the PEP's status. See `PEP Maintenance +<https://peps.python.org/pep-0001/#pep-maintenance>`_ +and `Changing Existing PEPs +<https://peps.python.org/pep-0001/#changing-existing-peps>`_ in PEP 1 for more. + +Copyediting and proofreading Draft and Active PEPs is welcome (subject to +review by the PEP author), and can be done via pull request to this repo. +Substantive content changes should first be proposed on PEP discussion threads. +We do advise against PRs that simply mass-correct minor typos on older PEPs +which don't significantly impair meaning and understanding. + +If you're still unsure, we encourage you to reach out first before opening a +PR here. For example, you could contact the PEP author(s), propose your idea in +a discussion venue appropriate to the PEP (such as `Typing-SIG +<https://mail.python.org/archives/list/typing-sig@python.org/>`__ for static +typing, or `Packaging Discourse <https://discuss.python.org/c/packaging/>`__ +for packaging), or `open an issue <https://github.com/python/peps/issues>`__. -Sign the CLA ------------- +Commit messages and PR titles +----------------------------- -Before you hit "Create pull request", please take a moment to ensure that this -project can legally accept your contribution by verifying you have signed the -PSF Contributor Agreement: +When adding or modifying a PEP, please include the PEP number in the commit +summary and pull request title. For example, ``PEP NNN: <summary of changes>``. +Likewise, prefix rendering infrastructure changes with ``Infra:``, linting +alterations with ``Lint:`` and other non-PEP meta changes, such as updates to +the Readme/Contributing Guide, issue/PR template, etc., with ``Meta:``. - https://www.python.org/psf/contrib/contrib-form/ -If you haven't signed the CLA before, please follow the steps outlined in the -CPython devguide to do so: +Sign the Contributor License Agreement +-------------------------------------- - https://devguide.python.org/pullrequest/#licensing +All contributors need to sign the +`PSF Contributor Agreement <https://www.python.org/psf/contrib/contrib-form/>`_. +to ensure we legally accept your work. -Thanks again to your contribution and we look forward to looking at it! +You don't need to do anything beforehand; +go ahead and create your pull request, +and our bot will ping you to sign the CLA if needed. +`See the CPython devguide +<https://devguide.python.org/pullrequest/#licensing>`__ +for more information. Code of Conduct @@ -49,5 +73,82 @@ Code of Conduct All interactions for this project are covered by the `PSF Code of Conduct <https://www.python.org/psf/codeofconduct/>`_. Everyone is -expected to be open, considerate, and respectful of others no matter their +expected to be open, considerate, and respectful of others, no matter their position within the project. + + +Run pre-commit linting locally +------------------------------ + +You can run this repo's basic linting suite locally, +either on-demand, or automatically against modified files +whenever you commit your changes. + +They are also run in CI, so you don't have to run them locally, though doing +so will help you catch and potentially fix common mistakes before pushing +your changes and opening a pull request. + +This repository uses the `pre-commit <https://pre-commit.com/>`_ tool to +install, configure and update a suite of hooks that check for +common problems and issues, and fix many of them automatically. + +If your system has ``make`` installed, you can run the pre-commit checkers +on the full repo by running ``make lint``. This will +install pre-commit in the current virtual environment if it isn't already, +so make sure you've activated the environment you want it to use +before running this command. + +Otherwise, you can install pre-commit with + +.. code-block:: bash + + python -m pip install pre-commit + +(or your choice of installer), and then run the hooks on all the files +in the repo with + +.. code-block:: bash + + pre-commit run --all-files + +or only on any files that have been modified but not yet committed with + +.. code-block:: bash + + pre-commit run + +If you would like pre-commit to run automatically against any modified files +every time you commit, install the hooks with + +.. code-block:: bash + + pre-commit install + +Then, whenever you ``git commit``, pre-commit will run and report any issues +it finds or changes it makes, and abort the commit to allow you to check, +and if necessary correct them before committing again. + + +Check and fix PEP spelling +-------------------------- + +To check for common spelling mistakes in your PEP and automatically suggest +corrections, you can run the codespell tool through pre-commit as well. + +Like the linters, on a system with ``make`` available, it can be installed +(in the currently-activated environment) and run on all files in the +repository with a single command, ``make spellcheck``. + +For finer control or on other systems, after installing pre-commit as in +the previous section, you can run it against only the files +you've modified and not yet committed with + +.. code-block:: bash + + pre-commit run --hook-stage manual codespell + +or against all files with + +.. code-block:: bash + + pre-commit run --all-files --hook-stage manual codespell diff --git a/Makefile b/Makefile index 0f201b0c0..8f973be2c 100644 --- a/Makefile +++ b/Makefile @@ -1,81 +1,82 @@ -# Builds PEP files to HTML using docutils or sphinx -# Also contains testing targets +# Builds PEP files to HTML using sphinx -PEP2HTML=pep2html.py +# You can set these variables from the command line. +PYTHON = python3 +VENVDIR = .venv +SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build +BUILDER = html +JOBS = 8 +SOURCES = +# synchronise with render.yml -> deploy step +OUTPUT_DIR = build +SPHINXERRORHANDLING = -W --keep-going -w sphinx-warnings.txt -PYTHON=python3 +ALLSPHINXOPTS = -b $(BUILDER) -j $(JOBS) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) peps $(OUTPUT_DIR) $(SOURCES) -VENV_DIR=venv +## html to render PEPs to "pep-NNNN.html" files +.PHONY: html +html: venv + $(SPHINXBUILD) $(ALLSPHINXOPTS) -.SUFFIXES: .txt .html .rst +## htmlview to open the index page built by the html target in your browser +.PHONY: htmlview +htmlview: html + $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/index.html'))" -.txt.html: - @$(PYTHON) $(PEP2HTML) $< +## dirhtml to render PEPs to "index.html" files within "pep-NNNN" directories +.PHONY: dirhtml +dirhtml: BUILDER = dirhtml +dirhtml: venv + $(SPHINXBUILD) $(ALLSPHINXOPTS) -.rst.html: - @$(PYTHON) $(PEP2HTML) $< +## check-links to check validity of links within PEP sources +.PHONY: check-links +check-links: BUILDER = linkcheck +check-links: venv + $(SPHINXBUILD) $(ALLSPHINXOPTS) -TARGETS= $(patsubst %.rst,%.html,$(wildcard pep-????.rst)) $(patsubst %.txt,%.html,$(wildcard pep-????.txt)) pep-0000.html +## clean to remove the venv and build files +.PHONY: clean +clean: clean-venv + -rm -rf build topic -all: pep-0000.rst $(TARGETS) - -$(TARGETS): pep2html.py - -pep-0000.rst: $(wildcard pep-????.txt) $(wildcard pep-????.rst) $(wildcard pep0/*.py) genpepindex.py - $(PYTHON) genpepindex.py . - -rss: - $(PYTHON) pep2rss.py . - -install: - echo "Installing is not necessary anymore. It will be done in post-commit." - -clean: - -rm pep-0000.rst - -rm *.html - -rm -rf build - -update: - git pull https://github.com/python/peps.git +## clean-venv to remove the venv +.PHONY: clean-venv +clean-venv: + rm -rf $(VENVDIR) +## venv to create a venv with necessary tools +.PHONY: venv venv: - $(PYTHON) -m venv $(VENV_DIR) - ./$(VENV_DIR)/bin/python -m pip install -r requirements.txt + @if [ -d $(VENVDIR) ] ; then \ + echo "venv already exists."; \ + echo "To recreate it, remove it first with \`make clean-venv'."; \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install -U pip wheel; \ + $(VENVDIR)/bin/python3 -m pip install -r requirements.txt; \ + echo "The venv has been created in the $(VENVDIR) directory"; \ + fi -package: all rss - mkdir -p build/peps - cp pep-*.txt build/peps/ - cp pep-*.rst build/peps/ - cp *.html build/peps/ - cp *.png build/peps/ - cp *.rss build/peps/ - tar -C build -czf build/peps.tar.gz peps +## lint to lint all the files +.PHONY: lint +lint: venv + $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files -lint: - pre-commit --version > /dev/null || python3 -m pip install pre-commit - pre-commit run --all-files +## test to test the Sphinx extensions for PEPs +.PHONY: test +test: venv + $(VENVDIR)/bin/python3 -bb -X dev -W error -m pytest -# New Sphinx targets: +## spellcheck to check spelling +.PHONY: spellcheck +spellcheck: venv + $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files --hook-stage manual codespell -SPHINX_JOBS=8 -SPHINX_BUILD=$(PYTHON) build.py -j $(SPHINX_JOBS) - -# TODO replace `rss:` with this when merged & tested -pep_rss: - $(PYTHON) pep_rss_gen.py - -pages: pep_rss - $(SPHINX_BUILD) --index-file - -sphinx: - $(SPHINX_BUILD) - -# for building Sphinx without a web-server -sphinx-local: - $(SPHINX_BUILD) --build-files - -fail-warning: - $(SPHINX_BUILD) --fail-on-warning - -check-links: - $(SPHINX_BUILD) --check-links +.PHONY: help +help : Makefile + @echo "Please use \`make <target>' where <target> is one of" + @sed -n 's/^##//p' $< diff --git a/PyRSS2Gen.py b/PyRSS2Gen.py deleted file mode 100644 index 65c1f0983..000000000 --- a/PyRSS2Gen.py +++ /dev/null @@ -1,456 +0,0 @@ -"""PyRSS2Gen - A Python library for generating RSS 2.0 feeds.""" - -__name__ = "PyRSS2Gen" -__version__ = (1, 1, 0) -__author__ = "Andrew Dalke <dalke@dalkescientific.com>" - -_generator_name = __name__ + "-" + ".".join(map(str, __version__)) - -import datetime - -import sys - -if sys.version_info[0] == 3: - # Python 3 - basestring = str - from io import StringIO -else: - # Python 2 - try: - from cStringIO import StringIO - except ImportError: - # Very old (or memory constrained) systems might - # have left out the compiled C version. Fall back - # to the pure Python one. Haven't seen this sort - # of system since the early 2000s. - from StringIO import StringIO - -# Could make this the base class; will need to add 'publish' -class WriteXmlMixin: - def write_xml(self, outfile, encoding = "iso-8859-1"): - from xml.sax import saxutils - handler = saxutils.XMLGenerator(outfile, encoding) - handler.startDocument() - self.publish(handler) - handler.endDocument() - - def to_xml(self, encoding = "iso-8859-1"): - f = StringIO() - self.write_xml(f, encoding) - return f.getvalue() - - -def _element(handler, name, obj, d = {}): - if isinstance(obj, basestring) or obj is None: - # special-case handling to make the API easier - # to use for the common case. - handler.startElement(name, d) - if obj is not None: - handler.characters(obj) - handler.endElement(name) - else: - # It better know how to emit the correct XML. - obj.publish(handler) - -def _opt_element(handler, name, obj): - if obj is None: - return - _element(handler, name, obj) - - -def _format_date(dt): - """convert a datetime into an RFC 822 formatted date - - Input date must be in GMT. - """ - # Looks like: - # Sat, 07 Sep 2002 00:00:01 GMT - # Can't use strftime because that's locale dependent - # - # Isn't there a standard way to do this for Python? The - # rfc822 and email.Utils modules assume a timestamp. The - # following is based on the rfc822 module. - return "%s, %02d %s %04d %02d:%02d:%02d GMT" % ( - ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()], - dt.day, - ["Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][dt.month-1], - dt.year, dt.hour, dt.minute, dt.second) - - -## -# A couple simple wrapper objects for the fields which -# take a simple value other than a string. -class IntElement: - """implements the 'publish' API for integers - - Takes the tag name and the integer value to publish. - - (Could be used for anything which uses str() to be published - to text for XML.) - """ - element_attrs = {} - def __init__(self, name, val): - self.name = name - self.val = val - def publish(self, handler): - handler.startElement(self.name, self.element_attrs) - handler.characters(str(self.val)) - handler.endElement(self.name) - -class DateElement: - """implements the 'publish' API for a datetime.datetime - - Takes the tag name and the datetime to publish. - - Converts the datetime to RFC 2822 timestamp (4-digit year). - """ - def __init__(self, name, dt): - self.name = name - self.dt = dt - def publish(self, handler): - _element(handler, self.name, _format_date(self.dt)) -#### - -class Category: - """Publish a category element""" - def __init__(self, category, domain = None): - self.category = category - self.domain = domain - def publish(self, handler): - d = {} - if self.domain is not None: - d["domain"] = self.domain - _element(handler, "category", self.category, d) - -class Cloud: - """Publish a cloud""" - def __init__(self, domain, port, path, - registerProcedure, protocol): - self.domain = domain - self.port = port - self.path = path - self.registerProcedure = registerProcedure - self.protocol = protocol - def publish(self, handler): - _element(handler, "cloud", None, { - "domain": self.domain, - "port": str(self.port), - "path": self.path, - "registerProcedure": self.registerProcedure, - "protocol": self.protocol}) - -class Image: - """Publish a channel Image""" - element_attrs = {} - def __init__(self, url, title, link, - width = None, height = None, description = None): - self.url = url - self.title = title - self.link = link - self.width = width - self.height = height - self.description = description - - def publish(self, handler): - handler.startElement("image", self.element_attrs) - - _element(handler, "url", self.url) - _element(handler, "title", self.title) - _element(handler, "link", self.link) - - width = self.width - if isinstance(width, int): - width = IntElement("width", width) - _opt_element(handler, "width", width) - - height = self.height - if isinstance(height, int): - height = IntElement("height", height) - _opt_element(handler, "height", height) - - _opt_element(handler, "description", self.description) - - handler.endElement("image") - -class Guid: - """Publish a guid - - Defaults to being a permalink, which is the assumption if it's - omitted. Hence strings are always permalinks. - """ - def __init__(self, guid, isPermaLink = 1): - self.guid = guid - self.isPermaLink = isPermaLink - def publish(self, handler): - d = {} - if self.isPermaLink: - d["isPermaLink"] = "true" - else: - d["isPermaLink"] = "false" - _element(handler, "guid", self.guid, d) - -class TextInput: - """Publish a textInput - - Apparently this is rarely used. - """ - element_attrs = {} - def __init__(self, title, description, name, link): - self.title = title - self.description = description - self.name = name - self.link = link - - def publish(self, handler): - handler.startElement("textInput", self.element_attrs) - _element(handler, "title", self.title) - _element(handler, "description", self.description) - _element(handler, "name", self.name) - _element(handler, "link", self.link) - handler.endElement("textInput") - - -class Enclosure: - """Publish an enclosure""" - def __init__(self, url, length, type): - self.url = url - self.length = length - self.type = type - def publish(self, handler): - _element(handler, "enclosure", None, - {"url": self.url, - "length": str(self.length), - "type": self.type, - }) - -class Source: - """Publish the item's original source, used by aggregators""" - def __init__(self, name, url): - self.name = name - self.url = url - def publish(self, handler): - _element(handler, "source", self.name, {"url": self.url}) - -class SkipHours: - """Publish the skipHours - - This takes a list of hours, as integers. - """ - element_attrs = {} - def __init__(self, hours): - self.hours = hours - def publish(self, handler): - if self.hours: - handler.startElement("skipHours", self.element_attrs) - for hour in self.hours: - _element(handler, "hour", str(hour)) - handler.endElement("skipHours") - -class SkipDays: - """Publish the skipDays - - This takes a list of days as strings. - """ - element_attrs = {} - def __init__(self, days): - self.days = days - def publish(self, handler): - if self.days: - handler.startElement("skipDays", self.element_attrs) - for day in self.days: - _element(handler, "day", day) - handler.endElement("skipDays") - -class RSS2(WriteXmlMixin): - """The main RSS class. - - Stores the channel attributes, with the "category" elements under - ".categories" and the RSS items under ".items". - """ - - rss_attrs = {"version": "2.0"} - element_attrs = {} - def __init__(self, - title, - link, - description, - - language = None, - copyright = None, - managingEditor = None, - webMaster = None, - pubDate = None, # a datetime, *in* *GMT* - lastBuildDate = None, # a datetime - - categories = None, # list of strings or Category - generator = _generator_name, - docs = "http://blogs.law.harvard.edu/tech/rss", - cloud = None, # a Cloud - ttl = None, # integer number of minutes - - image = None, # an Image - rating = None, # a string; I don't know how it's used - textInput = None, # a TextInput - skipHours = None, # a SkipHours with a list of integers - skipDays = None, # a SkipDays with a list of strings - - items = None, # list of RSSItems - ): - self.title = title - self.link = link - self.description = description - self.language = language - self.copyright = copyright - self.managingEditor = managingEditor - - self.webMaster = webMaster - self.pubDate = pubDate - self.lastBuildDate = lastBuildDate - - if categories is None: - categories = [] - self.categories = categories - self.generator = generator - self.docs = docs - self.cloud = cloud - self.ttl = ttl - self.image = image - self.rating = rating - self.textInput = textInput - self.skipHours = skipHours - self.skipDays = skipDays - - if items is None: - items = [] - self.items = items - - def publish(self, handler): - handler.startElement("rss", self.rss_attrs) - handler.startElement("channel", self.element_attrs) - _element(handler, "title", self.title) - _element(handler, "link", self.link) - _element(handler, "description", self.description) - - self.publish_extensions(handler) - - _opt_element(handler, "language", self.language) - _opt_element(handler, "copyright", self.copyright) - _opt_element(handler, "managingEditor", self.managingEditor) - _opt_element(handler, "webMaster", self.webMaster) - - pubDate = self.pubDate - if isinstance(pubDate, datetime.datetime): - pubDate = DateElement("pubDate", pubDate) - _opt_element(handler, "pubDate", pubDate) - - lastBuildDate = self.lastBuildDate - if isinstance(lastBuildDate, datetime.datetime): - lastBuildDate = DateElement("lastBuildDate", lastBuildDate) - _opt_element(handler, "lastBuildDate", lastBuildDate) - - for category in self.categories: - if isinstance(category, basestring): - category = Category(category) - category.publish(handler) - - _opt_element(handler, "generator", self.generator) - _opt_element(handler, "docs", self.docs) - - if self.cloud is not None: - self.cloud.publish(handler) - - ttl = self.ttl - if isinstance(self.ttl, int): - ttl = IntElement("ttl", ttl) - _opt_element(handler, "ttl", ttl) - - if self.image is not None: - self.image.publish(handler) - - _opt_element(handler, "rating", self.rating) - if self.textInput is not None: - self.textInput.publish(handler) - if self.skipHours is not None: - self.skipHours.publish(handler) - if self.skipDays is not None: - self.skipDays.publish(handler) - - for item in self.items: - item.publish(handler) - - handler.endElement("channel") - handler.endElement("rss") - - def publish_extensions(self, handler): - # Derived classes can hook into this to insert - # output after the three required fields. - pass - - - -class RSSItem(WriteXmlMixin): - """Publish an RSS Item""" - element_attrs = {} - def __init__(self, - title = None, # string - link = None, # url as string - description = None, # string - author = None, # email address as string - categories = None, # list of string or Category - comments = None, # url as string - enclosure = None, # an Enclosure - guid = None, # a unique string - pubDate = None, # a datetime - source = None, # a Source - ): - - if title is None and description is None: - raise TypeError( - "must define at least one of 'title' or 'description'") - self.title = title - self.link = link - self.description = description - self.author = author - if categories is None: - categories = [] - self.categories = categories - self.comments = comments - self.enclosure = enclosure - self.guid = guid - self.pubDate = pubDate - self.source = source - # It sure does get tedious typing these names three times... - - def publish(self, handler): - handler.startElement("item", self.element_attrs) - _opt_element(handler, "title", self.title) - _opt_element(handler, "link", self.link) - self.publish_extensions(handler) - _opt_element(handler, "description", self.description) - _opt_element(handler, "author", self.author) - - for category in self.categories: - if isinstance(category, basestring): - category = Category(category) - category.publish(handler) - - _opt_element(handler, "comments", self.comments) - if self.enclosure is not None: - self.enclosure.publish(handler) - _opt_element(handler, "guid", self.guid) - - pubDate = self.pubDate - if isinstance(pubDate, datetime.datetime): - pubDate = DateElement("pubDate", pubDate) - _opt_element(handler, "pubDate", pubDate) - - if self.source is not None: - self.source.publish(handler) - - handler.endElement("item") - - def publish_extensions(self, handler): - # Derived classes can hook into this to insert - # output after the title and link elements - pass diff --git a/README.rst b/README.rst index 2bd303563..fa1d17d0a 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,23 @@ Python Enhancement Proposals ============================ -.. image:: https://travis-ci.org/python/peps.svg?branch=master - :target: https://travis-ci.org/python/peps +.. image:: https://github.com/python/peps/actions/workflows/render.yml/badge.svg + :target: https://github.com/python/peps/actions The PEPs in this repo are published automatically on the web at -https://www.python.org/dev/peps/. To learn more about the purpose of -PEPs and how to go about writing a PEP, please start reading at PEP 1 -(``pep-0001.txt`` in this repo). Note that PEP 0, the index PEP, is -now automatically generated, and not committed to the repo. +https://peps.python.org/. To learn more about the purpose of PEPs and how to go +about writing one, please start reading at :pep:`1`. Note that the PEP Index +(:pep:`0`) is automatically generated based on the metadata headers in other PEPs. + + +Canonical links +=============== + +The canonical form of PEP links are zero-padded, such as +``https://peps.python.org/pep-0008/`` + +Shortcut redirects are also available. +For example, ``https://peps.python.org/8`` redirects to the canonical link. Contributing to PEPs @@ -17,113 +26,50 @@ Contributing to PEPs See the `Contributing Guidelines <./CONTRIBUTING.rst>`_. -reStructuredText for PEPs -========================= - -Original PEP source should be written in reStructuredText format, -which is a constrained version of plaintext, and is described in -PEP 12. Older PEPs were often written in a more mildly restricted -plaintext format, as described in PEP 9. The ``pep2html.py`` -processing and installation script knows how to produce the HTML -for either PEP format. - -For processing reStructuredText format PEPs, you need the docutils -package, which is available from `PyPI <https://pypi.org/>`_. -If you have pip, ``pip install docutils`` should install it. - - -Generating the PEP Index -======================== - -PEP 0 is automatically generated based on the metadata headers in other -PEPs. The script handling this is ``genpepindex.py``, with supporting -libraries in the ``pep0`` directory. - - Checking PEP formatting and rendering ===================================== -Do not commit changes with bad formatting. To check the formatting of -a PEP, use the Makefile. In particular, to generate HTML for PEP 9999, -your source code should be in ``pep-9999.rst`` and the HTML will be -generated to ``pep-9999.html`` by the command ``make pep-9999.html``. -The default Make target generates HTML for all PEPs. - -If you don't have Make, use the ``pep2html.py`` script directly. +Please don't commit changes with reStructuredText syntax errors that cause PEP +generation to fail, or result in major rendering defects relative to what you +intend. -Generating HTML for python.org -============================== +Browse the ReadTheDocs preview +------------------------------ -python.org includes its own helper modules to render PEPs as HTML, with -suitable links back to the source pages in the version control repository. +For every PR, we automatically create a preview of the rendered PEPs using +`ReadTheDocs <https://readthedocs.org/>`_. +You can find it in the merge box at the bottom of the PR page: -These can be found at https://github.com/python/pythondotorg/tree/master/peps - -When making changes to the PEP management process that may impact python.org's -rendering pipeline: - -* Clone the python.org repository from https://github.com/python/pythondotorg/ -* Get set up for local python.org development as per - https://pythondotorg.readthedocs.io/install.html#manual-setup -* Adjust ``PEP_REPO_PATH`` in ``pydotorg/settings/local.py`` to refer to your - local clone of the PEP repository -* Run ``./manage.py generate_pep_pages`` as described in - https://pythondotorg.readthedocs.io/pep_generation.html +1. Click "Show all checks" to expand the checks section +2. Find the line for ``docs/readthedocs.org:pep-previews`` +3. Click on "Details" to the right -Rendering PEPs with Sphinx -========================== +Render PEPs locally +------------------- -There is a Sphinx-rendered version of the PEPs at https://python.github.io/peps/ -(updated on every push to ``master``) +See the `build documentation <./docs/build.rst>`__ for full +instructions on how to render PEPs locally. +In summary, run the following in a fresh, activated virtual environment: -**Warning:** This version is not, and should not be taken to be, a canonical -source for PEPs whilst it remains in preview (`please report any rendering bugs -<https://github.com/python/peps/issues/new>`_). The canonical source for PEPs remains -https://www.python.org/dev/peps/ +.. code-block:: bash -Build PEPs with Sphinx locally: -------------------------------- + # Install requirements + python -m pip install -U -r requirements.txt -1. Ensure you have Python >=3.9 and Sphinx installed -2. If you have access to ``make``, follow (i), otherwise (ii) + # Build the PEPs + make html - i. Run ``make sphinx-local`` - ii. Run ``python build.py -j 8 --build-files``. Note that the jobs argument - only takes effect on unix (non-mac) systems. -3. Wait for Sphinx to render the PEPs. There may be a series of warnings about - unreferenced citations or labels -- whilst these are valid warnings they do - not impact the build process. -4. Navigate to the ``build`` directory of your PEPs repo to find the HTML pages. - PEP 0 provides a formatted index, and may be a useful reference. + # Or, if you don't have 'make': + python build.py -Arguments to ``build.py``: --------------------------- +The output HTML is found under the ``build`` directory. -Renderers: -``-f`` or ``--build-files`` - Renders PEPs to ``pep-XXXX.html`` files +Check and lint PEPs +------------------- -``-d`` or ``--build-dirs`` - Renders PEPs to ``index.html`` files within ``pep-XXXX`` directories - -Options: - -``-i`` or ``--index-file`` - Copies PEP 0 to a base index file - -``-j`` or ``--jobs`` - How many parallel jobs to run (if supported). Integer, default 1 - -``-n`` or ``--nitpicky`` - Runs Sphinx in `nitpicky` mode - -``-w`` or ``--fail-on-warning`` - Fails Sphinx on warnings - -Tools: - -``-l`` or ``--check-links`` - Checks validity of links within PEP sources +You can check for and fix common linting and spelling issues, +either on-demand or automatically as you commit, with our pre-commit suite. +See the `Contributing Guide <./CONTRIBUTING.rst>`_ for details. diff --git a/build.py b/build.py old mode 100644 new mode 100755 index 97c014536..04f0b5fdb --- a/build.py +++ b/build.py @@ -1,6 +1,11 @@ +#!/usr/bin/env python3 +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + """Build script for Sphinx documentation""" import argparse +import os from pathlib import Path from sphinx.application import Sphinx @@ -9,17 +14,26 @@ from sphinx.application import Sphinx def create_parser(): parser = argparse.ArgumentParser(description="Build PEP documents") # alternative builders: - parser.add_argument("-l", "--check-links", action="store_true") - parser.add_argument("-f", "--build-files", action="store_true") - parser.add_argument("-d", "--build-dirs", action="store_true") + builders = parser.add_mutually_exclusive_group() + builders.add_argument("-l", "--check-links", action="store_const", + dest="builder", const="linkcheck", + help='Check validity of links within PEP sources. ' + 'Cannot be used with "-f" or "-d".') + builders.add_argument("-f", "--build-files", action="store_const", + dest="builder", const="html", + help='Render PEPs to "pep-NNNN.html" files (default). ' + 'Cannot be used with "-d" or "-l".') + builders.add_argument("-d", "--build-dirs", action="store_const", + dest="builder", const="dirhtml", + help='Render PEPs to "index.html" files within "pep-NNNN" directories. ' + 'Cannot be used with "-f" or "-l".') - # flags / options - parser.add_argument("-w", "--fail-on-warning", action="store_true") - parser.add_argument("-n", "--nitpicky", action="store_true") - parser.add_argument("-j", "--jobs", type=int, default=1) - - # extra build steps - parser.add_argument("-i", "--index-file", action="store_true") # for PEP 0 + parser.add_argument( + "-o", + "--output-dir", + default="build", + help="Output directory, relative to root. Default 'build'.", + ) return parser.parse_args() @@ -39,40 +53,24 @@ def create_index_file(html_root: Path, builder: str) -> None: if __name__ == "__main__": args = create_parser() - root_directory = Path(".").absolute() - source_directory = root_directory - build_directory = root_directory / "build" # synchronise with deploy-gh-pages.yaml -> deploy step - doctree_directory = build_directory / ".doctrees" + root_directory = Path(__file__).resolve().parent + source_directory = root_directory / "peps" + build_directory = root_directory / args.output_dir # builder configuration - if args.build_files: - sphinx_builder = "html" - elif args.build_dirs: - sphinx_builder = "dirhtml" - elif args.check_links: - sphinx_builder = "linkcheck" - else: - # default builder - sphinx_builder = "dirhtml" - - # other configuration - config_overrides = {} - if args.nitpicky: - config_overrides["nitpicky"] = True + sphinx_builder = args.builder or "html" app = Sphinx( source_directory, confdir=source_directory, - outdir=build_directory, - doctreedir=doctree_directory, + outdir=build_directory / sphinx_builder, + doctreedir=build_directory / "doctrees", buildername=sphinx_builder, - confoverrides=config_overrides, - warningiserror=args.fail_on_warning, - parallel=args.jobs, + warningiserror=True, + parallel=os.cpu_count() or 1, + tags=["internal_builder"], + keep_going=True, ) - app.builder.copysource = False # Prevent unneeded source copying - we link direct to GitHub - app.builder.search = False # Disable search app.build() - - if args.index_file: - create_index_file(build_directory, sphinx_builder) + + create_index_file(build_directory, sphinx_builder) diff --git a/check-peps.py b/check-peps.py new file mode 100755 index 000000000..ea45cd161 --- /dev/null +++ b/check-peps.py @@ -0,0 +1,605 @@ +#!/usr/bin/env python3 + +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +"""check-peps: Check PEPs for common mistakes. + +Usage: check-peps [-d | --detailed] <PEP files...> + +Only the PEPs specified are checked. +If none are specified, all PEPs are checked. + +Use "--detailed" to show the contents of lines where errors were found. +""" + +from __future__ import annotations + +import datetime as dt +import re +import sys +from pathlib import Path + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, KeysView, Sequence + from typing import TypeAlias + + # (line number, warning message) + Message: TypeAlias = tuple[int, str] + MessageIterator: TypeAlias = Iterator[Message] + + +# get the directory with the PEP sources +ROOT_DIR = Path(__file__).resolve().parent +PEP_ROOT = ROOT_DIR / "peps" + +# See PEP 12 for the order +# Note we retain "BDFL-Delegate" +ALL_HEADERS = ( + "PEP", + "Title", + "Version", + "Last-Modified", + "Author", + "Sponsor", + "BDFL-Delegate", "PEP-Delegate", + "Discussions-To", + "Status", + "Type", + "Topic", + "Content-Type", + "Requires", + "Created", + "Python-Version", + "Post-History", + "Replaces", + "Superseded-By", + "Resolution", +) +REQUIRED_HEADERS = frozenset({"PEP", "Title", "Author", "Status", "Type", "Created"}) + +# See PEP 1 for the full list +ALL_STATUSES = frozenset({ + "Accepted", + "Active", + "April Fool!", + "Deferred", + "Draft", + "Final", + "Provisional", + "Rejected", + "Superseded", + "Withdrawn", +}) + +# PEPs that are allowed to link directly to PEPs +SKIP_DIRECT_PEP_LINK_CHECK = frozenset({"0009", "0287", "0676", "0684", "8001"}) + +DEFAULT_FLAGS = re.ASCII | re.IGNORECASE # Insensitive latin + +# any sequence of letters or '-', followed by a single ':' and a space or end of line +HEADER_PATTERN = re.compile(r"^([a-z\-]+):(?: |$)", DEFAULT_FLAGS) +# any sequence of unicode letters or legal special characters +NAME_PATTERN = re.compile(r"(?:[^\W\d_]|[ ',\-.])+(?: |$)") +# any sequence of ASCII letters, digits, or legal special characters +EMAIL_LOCAL_PART_PATTERN = re.compile(r"[\w!#$%&'*+\-/=?^{|}~.]+", DEFAULT_FLAGS) + +DISCOURSE_THREAD_PATTERN = re.compile(r"([\w\-]+/)?\d+", DEFAULT_FLAGS) +DISCOURSE_POST_PATTERN = re.compile(r"([\w\-]+/)?\d+(/\d+)?", DEFAULT_FLAGS) + +MAILMAN_2_PATTERN = re.compile(r"[\w\-]+/\d{4}-[a-z]+/\d+\.html", DEFAULT_FLAGS) +MAILMAN_3_THREAD_PATTERN = re.compile(r"[\w\-]+@python\.org/thread/[a-z0-9]+/?", DEFAULT_FLAGS) +MAILMAN_3_MESSAGE_PATTERN = re.compile(r"[\w\-]+@python\.org/message/[a-z0-9]+/?(#[a-z0-9]+)?", DEFAULT_FLAGS) + +# Controlled by the "--detailed" flag +DETAILED_ERRORS = False + + +def check(filenames: Sequence[str] = (), /) -> int: + """The main entry-point.""" + if filenames: + filenames = map(Path, filenames) + else: + filenames = PEP_ROOT.glob("pep-????.rst") + if (count := sum(map(check_file, filenames))) > 0: + s = "s" * (count != 1) + print(f"check-peps failed: {count} error{s}", file=sys.stderr) + return 1 + return 0 + + +def check_file(filename: Path, /) -> int: + filename = filename.resolve() + try: + content = filename.read_text(encoding="utf-8") + except FileNotFoundError: + return _output_error(filename, [""], [(0, "Could not read PEP!")]) + else: + lines = content.splitlines() + return _output_error(filename, lines, check_peps(filename, lines)) + + +def check_peps(filename: Path, lines: Sequence[str], /) -> MessageIterator: + yield from check_headers(lines) + for line_num, line in enumerate(lines, start=1): + if filename.stem.removeprefix("pep-") in SKIP_DIRECT_PEP_LINK_CHECK: + continue + yield from check_direct_links(line_num, line.lstrip()) + + +def check_headers(lines: Sequence[str], /) -> MessageIterator: + yield from _validate_pep_number(next(iter(lines), "")) + + found_headers = {} + line_num = 0 + for line_num, line in enumerate(lines, start=1): + if line.strip() == "": + headers_end_line_num = line_num + break + if match := HEADER_PATTERN.match(line): + header = match[1] + if header in ALL_HEADERS: + if header not in found_headers: + found_headers[match[1]] = line_num + else: + yield line_num, f"Must not have duplicate header: {header} " + else: + yield line_num, f"Must not have invalid header: {header}" + else: + headers_end_line_num = line_num + + yield from _validate_required_headers(found_headers.keys()) + + shifted_line_nums = list(found_headers.values())[1:] + for i, (header, line_num) in enumerate(found_headers.items()): + start = line_num - 1 + end = headers_end_line_num - 1 + if i < len(found_headers) - 1: + end = shifted_line_nums[i] - 1 + remainder = "\n".join(lines[start:end]).removeprefix(f"{header}:") + if remainder != "": + if remainder[0] not in {" ", "\n"}: + yield line_num, f"Headers must have a space after the colon: {header}" + remainder = remainder.lstrip() + yield from _validate_header(header, line_num, remainder) + + +def _validate_header(header: str, line_num: int, content: str) -> MessageIterator: + if header == "Title": + yield from _validate_title(line_num, content) + elif header == "Author": + yield from _validate_author(line_num, content) + elif header == "Sponsor": + yield from _validate_sponsor(line_num, content) + elif header in {"BDFL-Delegate", "PEP-Delegate"}: + yield from _validate_delegate(line_num, content) + elif header == "Discussions-To": + yield from _validate_discussions_to(line_num, content) + elif header == "Status": + yield from _validate_status(line_num, content) + elif header == "Type": + yield from _validate_type(line_num, content) + elif header == "Topic": + yield from _validate_topic(line_num, content) + elif header == "Content-Type": + yield from _validate_content_type(line_num, content) + elif header in {"Requires", "Replaces", "Superseded-By"}: + yield from _validate_pep_references(line_num, content) + elif header == "Created": + yield from _validate_created(line_num, content) + elif header == "Python-Version": + yield from _validate_python_version(line_num, content) + elif header == "Post-History": + yield from _validate_post_history(line_num, content) + elif header == "Resolution": + yield from _validate_resolution(line_num, content) + + +def check_direct_links(line_num: int, line: str) -> MessageIterator: + """Check that PEPs and RFCs aren't linked directly""" + + line = line.lower() + if "dev/peps/pep-" in line or "peps.python.org/pep-" in line: + yield line_num, "Use the :pep:`NNN` role to refer to PEPs" + if "rfc-editor.org/rfc/" in line or "ietf.org/doc/html/rfc" in line: + yield line_num, "Use the :rfc:`NNN` role to refer to RFCs" + + +def _output_error(filename: Path, lines: Sequence[str], errors: Iterable[Message]) -> int: + relative_filename = filename.relative_to(ROOT_DIR) + err_count = 0 + for line_num, msg in errors: + err_count += 1 + + print(f"{relative_filename}:{line_num}: {msg}") + if not DETAILED_ERRORS: + continue + + line = lines[line_num - 1] + print(" |") + print(f"{line_num: >4} | '{line}'") + print(" |") + + return err_count + + +########################### +# PEP Header Validators # +########################### + + +def _validate_required_headers(found_headers: KeysView[str]) -> MessageIterator: + """PEPs must have all required headers, in the PEP 12 order""" + + if missing := REQUIRED_HEADERS.difference(found_headers): + for missing_header in sorted(missing, key=ALL_HEADERS.index): + yield 1, f"Must have required header: {missing_header}" + + ordered_headers = sorted(found_headers, key=ALL_HEADERS.index) + if list(found_headers) != ordered_headers: + order_str = ", ".join(ordered_headers) + yield 1, "Headers must be in PEP 12 order. Correct order: " + order_str + + +def _validate_pep_number(line: str) -> MessageIterator: + """'PEP' header must be a number 1-9999""" + + if not line.startswith("PEP: "): + yield 1, "PEP must begin with the 'PEP:' header" + return + + pep_number = line.removeprefix("PEP: ").lstrip() + yield from _pep_num(1, pep_number, "'PEP:' header") + + +def _validate_title(line_num: int, line: str) -> MessageIterator: + """'Title' must be 1-79 characters""" + + if len(line) == 0: + yield line_num, "PEP must have a title" + elif len(line) > 79: + yield line_num, "PEP title must be less than 80 characters" + + +def _validate_author(line_num: int, body: str) -> MessageIterator: + """'Author' must be list of 'Name <email@example.com>, 
'""" + + lines = body.split("\n") + for offset, line in enumerate(lines): + if offset >= 1 and line[:9].isspace(): + # Checks for: + # Author: Alice + # Bob + # ^^^^ + # Note that len("Author: ") == 8 + yield line_num + offset, "Author line must not be over-indented" + if offset < len(lines) - 1: + if not line.endswith(","): + yield line_num + offset, "Author continuation lines must end with a comma" + for part in line.removesuffix(",").split(", "): + yield from _email(line_num + offset, part, "Author") + + +def _validate_sponsor(line_num: int, line: str) -> MessageIterator: + """'Sponsor' must have format 'Name <email@example.com>'""" + + yield from _email(line_num, line, "Sponsor") + + +def _validate_delegate(line_num: int, line: str) -> MessageIterator: + """'Delegate' must have format 'Name <email@example.com>'""" + + if line == "": + return + + # PEP 451 + if ", " in line: + for part in line.removesuffix(",").split(", "): + yield from _email(line_num, part, "Delegate") + return + + yield from _email(line_num, line, "Delegate") + + +def _validate_discussions_to(line_num: int, line: str) -> MessageIterator: + """'Discussions-To' must be a thread URL""" + + yield from _thread(line_num, line, "Discussions-To", discussions_to=True) + if line.startswith("https://"): + return + for suffix in "@python.org", "@googlegroups.com": + if line.endswith(suffix): + remainder = line.removesuffix(suffix) + if re.fullmatch(r"[\w\-]+", remainder) is None: + yield line_num, "Discussions-To must be a valid mailing list" + return + yield line_num, "Discussions-To must be a valid thread URL or mailing list" + + +def _validate_status(line_num: int, line: str) -> MessageIterator: + """'Status' must be a valid PEP status""" + + if line not in ALL_STATUSES: + yield line_num, "Status must be a valid PEP status" + + +def _validate_type(line_num: int, line: str) -> MessageIterator: + """'Type' must be a valid PEP type""" + + if line not in {"Standards Track", "Informational", "Process"}: + yield line_num, "Type must be a valid PEP type" + + +def _validate_topic(line_num: int, line: str) -> MessageIterator: + """'Topic' must be for a valid sub-index""" + + topics = line.split(", ") + unique_topics = set(topics) + if len(topics) > len(unique_topics): + yield line_num, "Topic must not contain duplicates" + + if unique_topics - {"Governance", "Packaging", "Typing", "Release"}: + if not all(map(str.istitle, unique_topics)): + yield line_num, "Topic must be properly capitalised (Title Case)" + if unique_topics - {"governance", "packaging", "typing", "release"}: + yield line_num, "Topic must be for a valid sub-index" + if sorted(topics) != topics: + yield line_num, "Topic must be sorted lexicographically" + + +def _validate_content_type(line_num: int, line: str) -> MessageIterator: + """'Content-Type' must be 'text/x-rst'""" + + if line != "text/x-rst": + yield line_num, "Content-Type must be 'text/x-rst'" + + +def _validate_pep_references(line_num: int, line: str) -> MessageIterator: + """`Requires`/`Replaces`/`Superseded-By` must be 'NNN' PEP IDs""" + + line = line.removesuffix(",").rstrip() + if line.count(", ") != line.count(","): + yield line_num, "PEP references must be separated by comma-spaces (', ')" + return + + references = line.split(", ") + for reference in references: + yield from _pep_num(line_num, reference, "PEP reference") + + +def _validate_created(line_num: int, line: str) -> MessageIterator: + """'Created' must be a 'DD-mmm-YYYY' date""" + + yield from _date(line_num, line, "Created") + + +def _validate_python_version(line_num: int, line: str) -> MessageIterator: + """'Python-Version' must be an ``X.Y[.Z]`` version""" + + versions = line.split(", ") + for version in versions: + if version.count(".") not in {1, 2}: + yield line_num, f"Python-Version must have two or three segments: {version}" + continue + + try: + major, minor, micro = version.split(".", 2) + except ValueError: + major, minor = version.split(".", 1) + micro = "" + + if major not in "123": + yield line_num, f"Python-Version major part must be 1, 2, or 3: {version}" + if not _is_digits(minor) and minor != "x": + yield line_num, f"Python-Version minor part must be numeric: {version}" + elif minor != "0" and minor[0] == "0": + yield line_num, f"Python-Version minor part must not have leading zeros: {version}" + + if micro == "": + return + if minor == "x": + yield line_num, f"Python-Version micro part must be empty if minor part is 'x': {version}" + elif micro[0] == "0": + yield line_num, f"Python-Version micro part must not have leading zeros: {version}" + elif not _is_digits(micro): + yield line_num, f"Python-Version micro part must be numeric: {version}" + + +def _validate_post_history(line_num: int, body: str) -> MessageIterator: + """'Post-History' must be '`DD-mmm-YYYY <Thread URL>`__, 
'""" + + if body == "": + return + + for offset, line in enumerate(body.removesuffix(",").split("\n"), start=line_num): + for post in line.removesuffix(",").strip().split(", "): + if not post.startswith("`") and not post.endswith(">`__"): + yield from _date(offset, post, "Post-History") + else: + post_date, post_url = post[1:-4].split(" <") + yield from _date(offset, post_date, "Post-History") + yield from _thread(offset, post_url, "Post-History") + + +def _validate_resolution(line_num: int, line: str) -> MessageIterator: + """'Resolution' must be a direct thread/message URL""" + + yield from _thread(line_num, line, "Resolution", allow_message=True) + + +######################## +# Validation Helpers # +######################## + +def _pep_num(line_num: int, pep_number: str, prefix: str) -> MessageIterator: + if pep_number == "": + yield line_num, f"{prefix} must not be blank: {pep_number!r}" + return + if pep_number.startswith("0") and pep_number != "0": + yield line_num, f"{prefix} must not contain leading zeros: {pep_number!r}" + if not _is_digits(pep_number): + yield line_num, f"{prefix} must be numeric: {pep_number!r}" + elif not 0 <= int(pep_number) <= 9999: + yield line_num, f"{prefix} must be between 0 and 9999: {pep_number!r}" + + +def _is_digits(string: str) -> bool: + """Match a string of ASCII digits ([0-9]+).""" + return string.isascii() and string.isdigit() + + +def _email(line_num: int, author_email: str, prefix: str) -> MessageIterator: + author_email = author_email.strip() + + if author_email.count("<") > 1: + msg = f"{prefix} entries must not contain multiple '<': {author_email!r}" + yield line_num, msg + if author_email.count(">") > 1: + msg = f"{prefix} entries must not contain multiple '>': {author_email!r}" + yield line_num, msg + if author_email.count("@") > 1: + msg = f"{prefix} entries must not contain multiple '@': {author_email!r}" + yield line_num, msg + + author = author_email.split("<", 1)[0].rstrip() + if NAME_PATTERN.fullmatch(author) is None: + msg = f"{prefix} entries must begin with a valid 'Name': {author_email!r}" + yield line_num, msg + return + + email_text = author_email.removeprefix(author) + if not email_text: + # Does not have the optional email part + return + + if not email_text.startswith(" <") or not email_text.endswith(">"): + msg = f"{prefix} entries must be formatted as 'Name <email@example.com>': {author_email!r}" + yield line_num, msg + email_text = email_text.removeprefix(" <").removesuffix(">") + + if "@" in email_text: + local, domain = email_text.rsplit("@", 1) + elif " at " in email_text: + local, domain = email_text.rsplit(" at ", 1) + else: + yield line_num, f"{prefix} entries must contain a valid email address: {author_email!r}" + return + if EMAIL_LOCAL_PART_PATTERN.fullmatch(local) is None or _invalid_domain(domain): + yield line_num, f"{prefix} entries must contain a valid email address: {author_email!r}" + + +def _invalid_domain(domain_part: str) -> bool: + *labels, root = domain_part.split(".") + for label in labels: + if not label.replace("-", "").isalnum(): + return True + return not root.isalnum() or not root.isascii() + + +def _thread(line_num: int, url: str, prefix: str, *, allow_message: bool = False, discussions_to: bool = False) -> MessageIterator: + if allow_message and discussions_to: + msg = "allow_message and discussions_to cannot both be True" + raise ValueError(msg) + + msg = f"{prefix} must be a valid thread URL" + + if not url.startswith("https://"): + if not discussions_to: + yield line_num, msg + return + + if url.startswith("https://discuss.python.org/t/"): + remainder = url.removeprefix("https://discuss.python.org/t/").removesuffix("/") + + # Discussions-To links must be the thread itself, not a post + if discussions_to: + # The equivalent pattern is similar to '([\w\-]+/)?\d+', + # but the topic name must contain a non-numeric character + + # We use ``str.rpartition`` as the topic name is optional + topic_name, _, topic_id = remainder.rpartition("/") + if topic_name == '' and _is_digits(topic_id): + return + topic_name = topic_name.replace("-", "0").replace("_", "0") + # the topic name must not be entirely numeric + valid_topic_name = not _is_digits(topic_name) and topic_name.isalnum() + if valid_topic_name and _is_digits(topic_id): + return + else: + # The equivalent pattern is similar to '([\w\-]+/)?\d+(/\d+)?', + # but the topic name must contain a non-numeric character + if remainder.count("/") == 2: + # When there are three parts, the URL must be "topic-name/topic-id/post-id". + topic_name, topic_id, post_id = remainder.rsplit("/", 2) + topic_name = topic_name.replace("-", "0").replace("_", "0") + valid_topic_name = not _is_digits(topic_name) and topic_name.isalnum() + if valid_topic_name and _is_digits(topic_id) and _is_digits(post_id): + # the topic name must not be entirely numeric + return + elif remainder.count("/") == 1: + # When there are only two parts, there's an ambiguity between + # "topic-name/topic-id" and "topic-id/post-id". + # We disambiguate by checking if the LHS is a valid name and + # the RHS is a valid topic ID (for the former), + # and then if both the LHS and RHS are valid IDs (for the latter). + left, right = remainder.rsplit("/") + left = left.replace("-", "0").replace("_", "0") + # the topic name must not be entirely numeric + left_is_name = not _is_digits(left) and left.isalnum() + if left_is_name and _is_digits(right): + return + elif _is_digits(left) and _is_digits(right): + return + else: + # When there's only one part, it must be a valid topic ID. + if _is_digits(remainder): + return + + if url.startswith("https://mail.python.org/pipermail/"): + remainder = url.removeprefix("https://mail.python.org/pipermail/") + if MAILMAN_2_PATTERN.fullmatch(remainder) is not None: + return + + if url.startswith("https://mail.python.org/archives/list/"): + remainder = url.removeprefix("https://mail.python.org/archives/list/") + if allow_message and MAILMAN_3_MESSAGE_PATTERN.fullmatch(remainder) is not None: + return + if MAILMAN_3_THREAD_PATTERN.fullmatch(remainder) is not None: + return + + yield line_num, msg + + +def _date(line_num: int, date_str: str, prefix: str) -> MessageIterator: + try: + parsed_date = dt.datetime.strptime(date_str, "%d-%b-%Y") + except ValueError: + yield line_num, f"{prefix} must be a 'DD-mmm-YYYY' date: {date_str!r}" + return + else: + if date_str[1] == "-": # Date must be zero-padded + yield line_num, f"{prefix} must be a 'DD-mmm-YYYY' date: {date_str!r}" + return + + if parsed_date.year < 1990: + yield line_num, f"{prefix} must not be before Python was invented: {date_str!r}" + if parsed_date > (dt.datetime.now() + dt.timedelta(days=14)): + yield line_num, f"{prefix} must not be in the future: {date_str!r}" + + +if __name__ == "__main__": + if {"-h", "--help", "-?"}.intersection(sys.argv[1:]): + print(__doc__, file=sys.stderr) + raise SystemExit(0) + + files = {} + for arg in sys.argv[1:]: + if not arg.startswith("-"): + files[arg] = None + elif arg in {"-d", "--detailed"}: + DETAILED_ERRORS = True + else: + print(f"Unknown option: {arg!r}", file=sys.stderr) + raise SystemExit(1) + raise SystemExit(check(files)) diff --git a/conf.py b/conf.py deleted file mode 100644 index 422a65fd5..000000000 --- a/conf.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Configuration for building PEPs using Sphinx.""" - -from pathlib import Path -import sys - -sys.path.append(str(Path("pep_sphinx_extensions").absolute())) - -# -- Project information ----------------------------------------------------- - -project = "PEPs" -master_doc = "contents" - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. -extensions = ["pep_sphinx_extensions", "sphinx.ext.githubpages"] - -# The file extensions of source files. Sphinx uses these suffixes as sources. -source_suffix = { - ".rst": "pep", - ".txt": "pep", -} - -# List of patterns (relative to source dir) to ignore when looking for source files. -exclude_patterns = [ - # Windows: - "Thumbs.db", - ".DS_Store", - # Python: - "venv", - "requirements.txt", - # Sphinx: - "build", - "output.txt", # Link-check output - # PEPs: - "README.rst", - "CONTRIBUTING.rst", -] - -# -- Options for HTML output ------------------------------------------------- - -# HTML output settings -html_math_renderer = "maths_to_html" # Maths rendering -html_show_copyright = False # Turn off miscellany -html_show_sphinx = False -html_title = "peps.python.org" # Set <title/> - -# Theme settings -html_theme_path = ["pep_sphinx_extensions"] -html_theme = "pep_theme" # The actual theme directory (child of html_theme_path) -html_use_index = False # Disable index (we use PEP 0) -html_sourcelink_suffix = "" # Fix links to GitHub (don't append .txt) -html_style = "" # must be defined here or in theme.conf, but is unused -html_permalinks = False # handled in the PEPContents transform - -templates_path = ['pep_sphinx_extensions/pep_theme/templates'] # Theme template relative paths from `confdir` diff --git a/contents.rst b/contents.rst deleted file mode 100644 index 658655e40..000000000 --- a/contents.rst +++ /dev/null @@ -1,16 +0,0 @@ - -Python Enhancement Proposals (PEPs) -*********************************** - - -This is an internal Sphinx page, please go to the :doc:`PEP Index<pep-0000>`. - - -.. toctree:: - :maxdepth: 3 - :titlesonly: - :hidden: - :glob: - :caption: PEP Table of Contents (needed for Sphinx): - - pep-* \ No newline at end of file diff --git a/deploy.bash b/deploy.bash deleted file mode 100755 index c4350fa6c..000000000 --- a/deploy.bash +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -ex -make package -pip install awscli -aws s3 cp --acl public-read build/peps.tar.gz s3://pythondotorg-assets-staging/peps.tar.gz -aws s3 cp --acl public-read build/peps.tar.gz s3://pythondotorg-assets/peps.tar.gz diff --git a/docs/build.rst b/docs/build.rst new file mode 100644 index 000000000..d59b2f804 --- /dev/null +++ b/docs/build.rst @@ -0,0 +1,95 @@ +:author: Adam Turner + + +Building PEPs Locally +===================== + +Whilst editing a PEP, it is useful to review the rendered output locally. +This can also be used to check that the PEP is valid reStructuredText before +submission to the PEP editors. + +The rest of this document assumes you are working from a local clone of the +`PEPs repository <https://github.com/python/peps>`__, +with **Python 3.9 or later** installed. + + +Render PEPs locally +------------------- + +1. Create a virtual environment and install requirements: + + .. code-block:: shell + + make venv + + If you don't have access to ``make``, run: + + .. code-block:: ps1con + + PS> python -m venv .venv + PS> .\.venv\Scripts\activate + (venv) PS> python -m pip install --upgrade pip + (venv) PS> python -m pip install -r requirements.txt + +2. **(Optional)** Delete prior build files. + Generally only needed when making changes to the rendering system itself. + + .. code-block:: shell + + rm -rf build + +3. Run the build script: + + .. code-block:: shell + + make html + + If you don't have access to ``make``, run: + + .. code-block:: ps1con + + (venv) PS> python build.py + +4. Navigate to the ``build`` directory of your PEPs repo to find the HTML pages. + PEP 0 provides a formatted index, and may be a useful reference. + + +``build.py`` tools +------------------ + +Several additional tools can be run through ``build.py``, or the Makefile. + +Note that before using ``build.py`` you must activate the virtual environment +created earlier: + +.. code-block:: shell + + source .venv/bin/activate + +Or on Windows: + +.. code-block:: ps1con + + PS> .\.venv\Scripts\activate + + +Check links +''''''''''' + +Check the validity of links within PEP sources (runs the `Sphinx linkchecker +<https://www.sphinx-doc.org/en/master/usage/builders/index.html#sphinx.builders.linkcheck.CheckExternalLinksBuilder>`__). + +.. code-block:: shell + + python build.py --check-links + make check-links + + +``build.py`` usage +------------------ + +For details on the command-line options to the ``build.py`` script, run: + +.. code-block:: shell + + python build.py --help diff --git a/docs/rendering_system.rst b/docs/rendering_system.rst new file mode 100644 index 000000000..fc20d00df --- /dev/null +++ b/docs/rendering_system.rst @@ -0,0 +1,240 @@ +:author: Adam Turner + +.. + We can't use :pep:`N` references in this document, as they use links relative + to the current file, which doesn't work in a subdirectory like this one. + + +An Overview of the PEP Rendering System +======================================= + +This document provides an overview of the PEP rendering system, as a companion +to `PEP 676 <https://peps.python.org/pep-0676/>`__. + + +1. Configuration +---------------- + +Configuration is stored in three files: + +- ``peps/conf.py`` contains the majority of the Sphinx configuration +- ``peps/contents.rst`` contains the compulsory table of contents directive +- ``pep_sphinx_extensions/pep_theme/theme.conf`` sets the Pygments themes + +The configuration: + +- registers the custom Sphinx extension +- sets the ``.rst`` suffix to be parsed as PEPs +- tells Sphinx which source files to use +- registers the PEP theme, maths renderer, and template +- disables some default settings that are covered in the extension +- sets the default and "dark mode" code formatter styles + + +2. Orchestration +---------------- + +``build.py`` manages the rendering process. +Usage is covered in `Building PEPs Locally <./build.rst>`_. + + +3. Extension +------------ + +The Sphinx extension and theme are contained in the ``pep_sphinx_extensions`` +directory. +The following is a brief overview of the stages of the PEP rendering process, +and how the extension functions at each point. + + +3.1 Extension setup +''''''''''''''''''' + +The extension registers several objects: + +- ``FileBuilder`` and ``DirectoryBuilder`` run the build process for file- and + directory-based building, respectively. +- ``PEPParser`` registers the custom document transforms and parses PEPs to + a Docutils document. +- ``PEPTranslator`` converts a Docutils document into HTML. +- ``PEPRole`` handles ``:pep:`` roles in the reStructuredText source. + +The extension also patches default behaviour: + +- updating the default settings +- updating the Docutils inliner +- using HTML maths display over MathJax + + +3.2 Builder initialised +''''''''''''''''''''''' + +After the Sphinx builder object is created and initialised, we ensure the +configuration is correct for the builder chosen. + +Currently this involves updating the relative link template. +See ``_update_config_for_builder`` in ``pep_sphinx_extensions/__init__.py``. + + +3.3 Before documents are read +''''''''''''''''''''''''''''' + +The ``create_pep_zero`` hook is called. See `5. PEP 0`_. + + +3.4 Read document +''''''''''''''''' + +Parsing the document is handled by ``PEPParser`` +(``pep_sphinx_extensions.pep_processor.parsing.pep_parser.PEPParser``), a +lightweight wrapper over ``sphinx.parsers.RSTParser``. + +``PEPParser`` reads the document with leading :rfc:`2822` headers and registers +the transforms we want to apply. +These are: + +- ``PEPHeaders`` +- ``PEPTitle`` +- ``PEPContents`` +- ``PEPFooter`` + +Transforms are then applied in priority order. + + +3.4.1 ``PEPRole`` role +********************** + +This overrides the built-in ``:pep:`` role to return the correct URL. + + +3.4.2 ``PEPHeaders`` transform +****************************** + +PEPs start with a set of :rfc:`2822` headers, +per `PEP 1 <https://peps.python.org/pep-0001/>`__. +This transform validates that the required headers are present and of the +correct data type, and removes headers not for display. +It must run before the ``PEPTitle`` transform. + + +3.4.3 ``PEPTitle`` transform +**************************** + +We generate the title node from the parsed title in the PEP headers, and make +all nodes in the document children of the new title node. +This transform must also handle parsing reStructuredText markup within PEP +titles, such as `PEP 604 <https://peps.python.org/pep-0604/>`__. + + +3.4.4 ``PEPContents`` transform +******************************* + +The automatic table of contents (TOC) is inserted in this transform in a +two-part process. + +First, the transform inserts a placeholder for the TOC and a horizontal rule +after the document title and PEP headers. +A callback transform then recursively walks the document to create the TOC, +starting from after the placeholder node. +Whilst walking the document, all reference nodes in the titles are removed, and +titles are given a self-link. + + +3.4.5 ``PEPFooter`` transform +***************************** + +This first builds a map of file modification times from a single git call, as +a speed-up. This will return incorrect results on a shallow checkout of the +repository, as is the default on continuous integration systems. + +We then attempt to remove any empty references sections, and append metadata in +the footer (source link and last modified timestamp). + + +3.5 Prepare for writing +'''''''''''''''''''''''' + +``pep_html_builder.FileBuilder.prepare_writing`` initialises the bare minimum +of the Docutils writer and the settings for writing documents. +This provides a significant speed-up over the base Sphinx implementation, as +most of the data automatically initialised was unused. + + +3.6 Translate Docutils to HTML +''''''''''''''''''''''''''''''' + +``PEPTranslator`` overrides paragraph and reference logic to replicate +processing from the previous ``docutils.writers.pep``-based system. +Paragraphs are made compact where possible by omitting ``<p>`` tags, and +footnote references are be enclosed in square brackets. + + +3.7 Prepare for export to Jinja +''''''''''''''''''''''''''''''' + +Finally in ``pep_html_builder``, we gather all the parts to be passed to the +Jinja template. +This is also where we create the sidebar table of contents. + +The HTML files are then written out to the build directory. + + +4. Theme +-------- + +The theme is comprised of the HTML template in +``pep_sphinx_extensions/pep_theme/templates/page.html`` and the stylesheets in +``pep_sphinx_extensions/pep_theme/static``. + +The template is entirely self-contained, not relying on any default behaviour +from Sphinx. +It specifies the CSS files to include, the favicon, and basic semantic +information for the document structure. + +The styles are defined in two parts: + +- ``style.css`` handles the meat of the layout +- ``mq.css`` adds media queries for a responsive design + + +5. \PEP 0 +--------- + +The generation of the index, PEP 0, happens in three phases. +The reStructuredText source file is generated, it is then added to Sphinx, and +finally the data is post processed. + + +5.1 File creation +''''''''''''''''' + +``pep-0000.rst`` is created during a callback, before documents are loaded by +Sphinx. + +We first parse the individual PEP files to get the :rfc:`2822` header, and then +parse and validate that metadata. + +After collecting and validating all the PEP data, the index itself is created in +three steps: + +1. Output the header text +2. Output the category and numerical indices +3. Output the author index + +We then add the newly created PEP 0 file to two Sphinx variables so that it will +be processed as a normal source document. + + +5.2 Post processing +''''''''''''''''''' + +The ``PEPHeaders`` transform schedules the \PEP 0 post-processing code. +This serves two functions: masking email addresses and linking numeric +PEP references to the actual documents. + + +6. RSS Feed +----------- + +The RSS feed is created by extracting the header metadata and abstract from the +ten most recent PEPs. diff --git a/docutils.conf b/docutils.conf deleted file mode 100644 index e2f4c8cc1..000000000 --- a/docutils.conf +++ /dev/null @@ -1,21 +0,0 @@ -# Configuration file for Docutils. -# See http://docutils.sf.net/docs/tools.html - -[general] -# These entries are for the page footer: -source-link: 1 -datestamp: %Y-%m-%d %H:%M UTC -generator: 1 - -# use the local stylesheet -stylesheet: pep.css -template: pyramid-pep-template - -# link to the stylesheet; don't embed it -embed-stylesheet: 0 - -# path to PEPs, for template: -pep-home: /dev/peps/ - -# base URL for PEP references (no host so mirrors work): -pep-base-url: /dev/peps/ diff --git a/genpepindex.py b/genpepindex.py deleted file mode 100755 index 2ab6698a0..000000000 --- a/genpepindex.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -"""Auto-generate PEP 0 (PEP index). - -Generating the PEP index is a multi-step process. To begin, you must first -parse the PEP files themselves, which in and of itself takes a couple of steps: - - 1. Parse metadata. - 2. Validate metadata. - -With the PEP information collected, to create the index itself you must: - - 1. Output static text. - 2. Format an entry for the PEP. - 3. Output the PEP (both by category and numerical index). - -""" -from __future__ import absolute_import, with_statement -from __future__ import print_function - -import sys -import os -import codecs - -from operator import attrgetter - -from pep0.output import write_pep0 -from pep0.pep import PEP, PEPError - - -def main(argv): - if not argv[1:]: - path = '.' - else: - path = argv[1] - - peps = [] - if os.path.isdir(path): - for file_path in os.listdir(path): - if file_path.startswith('pep-0000.'): - continue - abs_file_path = os.path.join(path, file_path) - if not os.path.isfile(abs_file_path): - continue - if file_path.startswith("pep-") and file_path.endswith((".txt", "rst")): - with codecs.open(abs_file_path, 'r', encoding='UTF-8') as pep_file: - try: - pep = PEP(pep_file) - if pep.number != int(file_path[4:-4]): - raise PEPError('PEP number does not match file name', - file_path, pep.number) - peps.append(pep) - except PEPError as e: - errmsg = "Error processing PEP %s (%s), excluding:" % \ - (e.number, e.filename) - print(errmsg, e, file=sys.stderr) - sys.exit(1) - peps.sort(key=attrgetter('number')) - elif os.path.isfile(path): - with open(path, 'r') as pep_file: - peps.append(PEP(pep_file)) - else: - raise ValueError("argument must be a directory or file path") - - with codecs.open('pep-0000.rst', 'w', encoding='UTF-8') as pep0_file: - write_pep0(peps, pep0_file) - -if __name__ == "__main__": - main(sys.argv) diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 000000000..5dfe3103b --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,2 @@ +.terraform* +terraform.tfstate* diff --git a/infra/config.tf b/infra/config.tf new file mode 100644 index 000000000..2cc663d31 --- /dev/null +++ b/infra/config.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + fastly = { + source = "fastly/fastly" + version = "1.1.2" + } + } + required_version = ">= 1.1.8" + cloud { + organization = "psf" + workspaces { + name = "peps" + } + } +} +variable "fastly_token" { + type = string + sensitive = true +} +provider "fastly" { + api_key = var.fastly_token +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 000000000..a2169a81e --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,84 @@ +resource "fastly_service_vcl" "peps" { + name = "peps.python.org" + activate = true + domain { name = "peps.python.org" } + + backend { + name = "GitHub Pages" + address = "python.github.io" + port = 443 + override_host = "peps.python.org" + + use_ssl = true + ssl_check_cert = true + ssl_cert_hostname = "python.github.io" + ssl_sni_hostname = "python.github.io" + } + + header { + name = "HSTS" + type = "response" + action = "set" + destination = "http.Strict-Transport-Security" + ignore_if_set = false + source = "\"max-age=31536000; includeSubDomains; preload\"" + } + + request_setting { + name = "Force TLS" + force_ssl = true + } + + snippet { + name = "serve-rss" + type = "recv" + content = <<-EOT + if (req.url == "/peps.rss/") { + set req.url = "/peps.rss"; + } + EOT + } + + snippet { + name = "topics" + type = "recv" + content = <<-EOT + if (req.url ~ "^/topics($|/)") { + set req.http.Location = regsub(req.http.Location, "^/topics/?", "/topic/"); + error 618; + } + EOT + } + + snippet { + name = "redirect" + type = "error" + content = <<-EOT + if (obj.status == 618) { + set obj.status = 302; + set obj.http.Location = "https://" + req.http.host + req.http.Location; + return(deliver); + } + EOT + } + snippet { + name = "redirect-numbers" + type = "recv" + content = <<-EOT + if (req.url ~ "^/(\d|\d\d|\d\d\d|\d\d\d\d)/?$") { + set req.http.Location = "/pep-" + std.strpad(re.group.1, 4, "0") + "/"; + error 618; + } + EOT + } + snippet { + name = "left-pad-pep-numbers" + type = "recv" + content = <<-EOT + if (req.url ~ "^/pep-(\d|\d\d|\d\d\d)/?$") { + set req.http.Location = "/pep-" + std.strpad(re.group.1, 4, "0") + "/"; + error 618; + } + EOT + } +} diff --git a/pep-0001-process_flow.png b/pep-0001-process_flow.png deleted file mode 100644 index 0fc8176d2..000000000 Binary files a/pep-0001-process_flow.png and /dev/null differ diff --git a/pep-0002.txt b/pep-0002.txt deleted file mode 100644 index b47932179..000000000 --- a/pep-0002.txt +++ /dev/null @@ -1,213 +0,0 @@ -PEP: 2 -Title: Procedure for Adding New Modules -Version: $Revision$ -Last-Modified: $Date$ -Author: Martijn Faassen <faassen@infrae.com> -Status: Superseded -Type: Process -Content-Type: text/x-rst -Created: 07-Jul-2001 -Post-History: 07-Jul-2001, 09-Mar-2002 - - - -PEP Replacement -=============== - -This PEP has been superseded by the updated material in the Python -Developer's Guide [1]_. - -Introduction -============ - -The Python Standard Library contributes significantly to Python's -success. The language comes with "batteries included", so it is easy -for people to become productive with just the standard library alone. -It is therefore important that this library grows with the language, -and that such growth is supported and encouraged. - -Many contributions to the library are not created by core developers -but by people from the Python community who are experts in their -particular field. Furthermore, community members are also the users of -the standard library, applying it in a great diversity of settings. -This makes the community well equipped to detect and report gaps in -the library; things that are missing but should be added. - -New functionality is commonly added to the library in the form of new -modules. This PEP will describe the procedure for the *addition* of -new modules. PEP 4 deals with procedures for deprecation of modules; -the *removal* of old and unused modules from the standard library. -Finally there is also the issue of *changing* existing modules to make -the picture of library evolution complete. PEP 3 and PEP 5 give some -guidelines on this. The continued maintenance of existing modules is -an integral part of the decision on whether to add a new module to the -standard library. Therefore, this PEP also introduces concepts -(integrators, maintainers) relevant to the maintenance issue. - - -Integrators -=========== - -The integrators are a group of people with the following -responsibilities: - -* They determine if a proposed contribution should become part of the - standard library. - -* They integrate accepted contributions into the standard library. - -* They produce standard library releases. - -This group of people shall be PythonLabs, led by Guido. - - -Maintainer(s) -============= - -All contributions to the standard library need one or more -maintainers. This can be an individual, but it is frequently a group -of people such as the XML- SIG. Groups may subdivide maintenance -tasks among themselves. One or more maintainers shall be the *head -maintainer* (usually this is also the main developer). Head -maintainers are convenient people the integrators can address if they -want to resolve specific issues, such as the ones detailed later in -this document. - - -Developers(s) -============= - -Contributions to the standard library have been developed by one or -more developers. The initial maintainers are the original developers -unless there are special circumstances (which should be detailed in -the PEP proposing the contribution). - - -Acceptance Procedure -==================== - -When developers wish to have a contribution accepted into the standard -library, they will first form a group of maintainers (normally -initially consisting of themselves). - -Then, this group shall produce a PEP called a library PEP. A library -PEP is a special form of standards track PEP. The library PEP gives -an overview of the proposed contribution, along with the proposed -contribution as the reference implementation. This PEP should also -contain a motivation on why this contribution should be part of the -standard library. - -One or more maintainers shall step forward as PEP champion (the people -listed in the Author field are the champions). The PEP champion(s) -shall be the initial head maintainer(s). - -As described in PEP 1, a standards track PEP should consist of a -design document and a reference implementation. The library PEP -differs from a normal standard track PEP in that the reference -implementation should in this case always already have been written -before the PEP is to be reviewed for inclusion by the integrators and -to be commented upon by the community; the reference implementation -*is* the proposed contribution. - -This different requirement exists for the following reasons: - -* The integrators can only properly evaluate a contribution to the - standard library when there is source code and documentation to look - at; i.e. the reference implementation is always necessary to aid - people in studying the PEP. - -* Even rejected contributions will be useful outside the standard - library, so there will a lower risk of waste of effort by the - developers. - -* It will impress the integrators of the seriousness of contribution - and will help guard them against having to evaluate too many - frivolous proposals. - -Once the library PEP has been submitted for review, the integrators -will then evaluate it. The PEP will follow the normal PEP work flow -as described in PEP 1. If the PEP is accepted, they will work through -the head maintainers to make the contribution ready for integration. - - -Maintenance Procedure -===================== - -After a contribution has been accepted, the job is not over for both -integrators and maintainers. The integrators will forward any bug -reports in the standard library to the appropriate head maintainers. - -Before the feature freeze preparing for a release of the standard -library, the integrators will check with the head maintainers for all -contributions, to see if there are any updates to be included in the -next release. The integrators will evaluate any such updates for -issues like backwards compatibility and may require PEPs if the -changes are deemed to be large. - -The head maintainers should take an active role in keeping up to date -with the Python development process. If a head maintainer is unable -to function in this way, he or she should announce the intention to -step down to the integrators and the rest of the maintainers, so that -a replacement can step forward. The integrators should at all times -be capable of reaching the head maintainers by email. - -In the case where no head maintainer can be found (possibly because -there are no maintainers left), the integrators will issue a call to -the community at large asking for new maintainers to step forward. If -no one does, the integrators can decide to declare the contribution -deprecated as described in PEP 4. - - -Open issues -=========== - -There needs to be some procedure so that the integrators can always -reach the maintainers (or at least the head maintainers). This could -be accomplished by a mailing list to which all head maintainers should -be subscribed (this could be python-dev). Another possibility, which -may be useful in any case, is the maintenance of a list similar to -that of the list of PEPs which lists all the contributions and their -head maintainers with contact info. This could in fact be part of the -list of the PEPs, as a new contribution requires a PEP. But since the -authors/owners of a PEP introducing a new module may eventually be -different from those who maintain it, this wouldn't resolve all issues -yet. - -Should there be a list of what criteria integrators use for evaluating -contributions? (Source code but also things like documentation and a -test suite, as well as such vague things like 'dependability of the -maintainers'.) - -This relates to all the technical issues; check-in privileges, coding -style requirements, documentation requirements, test suite -requirements. These are preferably part of another PEP. - -Should the current standard library be subdivided among maintainers? -Many parts already have (informal) maintainers; it may be good to make -this more explicit. - -Perhaps there is a better word for 'contribution'; the word -'contribution' may not imply enough that the process (of development -and maintenance) does not stop after the contribution is accepted and -integrated into the library. - -Relationship to the mythical Catalog? - -References -========== - -.. [1] Adding to the Stdlib - (http://docs.python.org/devguide/stdlibchanges.html) - -Copyright -========= - -This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0004.txt b/pep-0004.txt deleted file mode 100644 index 353f4563c..000000000 --- a/pep-0004.txt +++ /dev/null @@ -1,321 +0,0 @@ -PEP: 4 -Title: Deprecation of Standard Modules -Version: $Revision$ -Last-Modified: $Date$ -Author: Brett Cannon <brett@python.org>, Martin von Löwis <martin@v.loewis.de> -Status: Active -Type: Process -Content-Type: text/x-rst -Created: 01-Oct-2000 -Post-History: - - -Introduction -============ - -When new modules were added to the standard Python library in the -past, it was not possible to foresee whether they would still be -useful in the future. Even though Python "Comes With Batteries -Included", batteries may discharge over time. Carrying old modules -around is a burden on the maintainer, especially when there is no -interest in the module anymore. - -At the same time, removing a module from the distribution is -difficult, as it is not known in general whether anybody is still -using it. This PEP defines a procedure for removing modules from the -standard Python library. Usage of a module may be 'deprecated', which -means that it may be removed from a future Python release. The -rationale for deprecating a module is also collected in this PEP. If -the rationale turns out faulty, the module may become 'undeprecated'. - - -Procedure for declaring a module deprecated -=========================================== - -Since the status of module deprecation is recorded in this PEP, -proposals for deprecating modules MUST be made by providing a change -to the text of this PEP. - -A proposal for deprecation of the module MUST include the date of the -proposed deprecation and a rationale for deprecating it. In addition, -the proposal MUST include a change to the documentation of the module; -deprecation is indicated by saying that the module is "obsolete" or -"deprecated". The proposal SHOULD include a patch for the module's -source code to indicate deprecation there as well, by raising a -DeprecationWarning. The proposal MUST include patches to remove any -use of the deprecated module from the standard library. - -It is expected that deprecated modules are included in the Python -release that immediately follows the deprecation; later releases may -ship without the deprecated modules. - - -For modules existing in both Python 2.7 and Python 3.5 ------------------------------------------------------- - -In order to facilitate writing code that works in both Python 2 & 3 -simultaneously, any module that exists in both Python 3.5 and -Python 2.7 will not be removed from the standard library until -Python 2.7 is no longer supported as specified by PEP 373. Exempted -from this rule is any module in the idlelib package as well as any -exceptions granted by the Python development team. - - -Procedure for declaring a module undeprecated -============================================= - -When a module becomes deprecated, a rationale is given for its -deprecation. In some cases, an alternative interface for the same -functionality is provided, so the old interface is deprecated. In -other cases, the need for having the functionality of the module may -not exist anymore. - -If the rationale is faulty, again a change to this PEP's text MUST be -submitted. This change MUST include the date of undeprecation and a -rationale for undeprecation. Modules that are undeprecated under this -procedure MUST be listed in this PEP for at least one major release of -Python. - - -Obsolete modules -================ - -A number of modules are already listed as obsolete in the library -documentation. These are listed here for completeness. - - cl, sv, timing - -All these modules have been declared as obsolete in Python 2.0, some -even earlier. - -The following obsolete modules were removed in Python 2.5: - - addpack, cmp, cmpcache, codehack, dircmp, dump, find, fmt, - grep, lockfile, newdir, ni, packmail, Para, poly, - rand, reconvert, regex, regsub, statcache, tb, tzparse, - util, whatsound, whrandom, zmod - -The following modules were removed in Python 2.6: - - gopherlib, rgbimg, macfs - -The following modules currently lack a DeprecationWarning: - - rfc822, mimetools, multifile - - -Deprecated modules -================== - -:: - - Module name: posixfile - Rationale: Locking is better done by fcntl.lockf(). - Date: Before 1-Oct-2000. - Documentation: Already documented as obsolete. Deprecation - warning added in Python 2.6. - - Module name: gopherlib - Rationale: The gopher protocol is not in active use anymore. - Date: 1-Oct-2000. - Documentation: Documented as deprecated since Python 2.5. Removed - in Python 2.6. - - Module name: rgbimgmodule - Rationale: In a 2001-04-24 c.l.py post, Jason Petrone mentions - that he occasionally uses it; no other references to - its use can be found as of 2003-11-19. - Date: 1-Oct-2000 - Documentation: Documented as deprecated since Python 2.5. Removed - in Python 2.6. - - Module name: pre - Rationale: The underlying PCRE engine doesn't support Unicode, and - has been unmaintained since Python 1.5.2. - Date: 10-Apr-2002 - Documentation: It was only mentioned as an implementation detail, - and never had a section of its own. This mention - has now been removed. - - Module name: whrandom - Rationale: The module's default seed computation was - inherently insecure; the random module should be - used instead. - Date: 11-Apr-2002 - Documentation: This module has been documented as obsolete since - Python 2.1, but listing in this PEP was neglected. - The deprecation warning will be added to the module - one year after Python 2.3 is released, and the - module will be removed one year after that. - - Module name: rfc822 - Rationale: Supplanted by Python 2.2's email package. - Date: 18-Mar-2002 - Documentation: Documented as "deprecated since release 2.3" since - Python 2.2.2. - - Module name: mimetools - Rationale: Supplanted by Python 2.2's email package. - Date: 18-Mar-2002 - Documentation: Documented as "deprecated since release 2.3" since - Python 2.2.2. - - Module name: MimeWriter - Rationale: Supplanted by Python 2.2's email package. - Date: 18-Mar-2002 - Documentation: Documented as "deprecated since release 2.3" since - Python 2.2.2. Raises a DeprecationWarning as of - Python 2.6. - - Module name: mimify - Rationale: Supplanted by Python 2.2's email package. - Date: 18-Mar-2002 - Documentation: Documented as "deprecated since release 2.3" since - Python 2.2.2. Raises a DeprecationWarning as of - Python 2.6. - - Module name: rotor - Rationale: Uses insecure algorithm. - Date: 24-Apr-2003 - Documentation: The documentation has been removed from the library - reference in Python 2.4. - - Module name: TERMIOS.py - Rationale: The constants in this file are now in the 'termios' module. - Date: 10-Aug-2004 - Documentation: This module has been documented as obsolete since - Python 2.1, but listing in this PEP was neglected. - Removed from the library reference in Python 2.4. - - Module name: statcache - Rationale: Using the cache can be fragile and error-prone; - applications should just use os.stat() directly. - Date: 10-Aug-2004 - Documentation: This module has been documented as obsolete since - Python 2.2, but listing in this PEP was neglected. - Removed from the library reference in Python 2.5. - - Module name: mpz - Rationale: Third-party packages provide similar features - and wrap more of GMP's API. - Date: 10-Aug-2004 - Documentation: This module has been documented as obsolete since - Python 2.2, but listing in this PEP was neglected. - Removed from the library reference in Python 2.4. - - Module name: xreadlines - Rationale: Using 'for line in file', introduced in 2.3, is preferable. - Date: 10-Aug-2004 - Documentation: This module has been documented as obsolete since - Python 2.3, but listing in this PEP was neglected. - Removed from the library reference in Python 2.4. - - Module name: multifile - Rationale: Supplanted by the email package. - Date: 21-Feb-2006 - Documentation: Documented as deprecated as of Python 2.5. - - Module name: sets - Rationale: The built-in set/frozenset types, introduced in - Python 2.4, supplant the module. - Date: 12-Jan-2007 - Documentation: Documented as deprecated as of Python 2.6. - - Module name: buildtools - Rationale: Unknown. - Date: 15-May-2007 - Documentation: Documented as deprecated as of Python 2.3, but - listing in this PEP was neglected. Raised a - DeprecationWarning as of Python 2.6. - - Module name: cfmfile - Rationale: Unknown. - Date: 15-May-2007 - Documentation: Documented as deprecated as of Python 2.4, but - listing in this PEP was neglected. A - DeprecationWarning was added in Python 2.6. - - Module name: macfs - Rationale: Unknown. - Date: 15-May-2007 - Documentation: Documented as deprecated as of Python 2.3, but - listing in this PEP was neglected. Removed in - Python 2.6. - - Module name: md5 - Rationale: Replaced by the 'hashlib' module. - Date: 15-May-2007 - Documentation: Documented as deprecated as of Python 2.5, but - listing in this PEP was neglected. - DeprecationWarning raised as of Python 2.6. - - Module name: sha - Rationale: Replaced by the 'hashlib' module. - Date: 15-May-2007 - Documentation: Documented as deprecated as of Python 2.5, but - listing in this PEP was neglected. - DeprecationWarning added in Python 2.6. - - Module name: plat-freebsd2/IN and plat-freebsd3/IN - Rationale: Platforms are obsolete (last released in 2000) - Removed from 2.6 - Date: 15-May-2007 - Documentation: None - - Module name: plat-freebsd4/IN and possibly plat-freebsd5/IN - Rationale: Platforms are obsolete/unsupported - Date: 15-May-2007 - Remove from 2.7 - Documentation: None - - Module name: imp - Rationale: Replaced by the importlib module. - Date: 2013-02-10 - Documentation: Deprecated as of Python 3.4. - - Module name: formatter - Rationale: Lack of use in the community, no tests to keep - code working. - Date: 2013-08-12 - Documentation: Deprecated as of Python 3.4. - - Module name: macpath - Rationale: Obsolete macpath module dangerously broken - and should be removed. - Date: 2017-05-15 - Documentation: Platform is obsolete/unsupported. - - Module name: xml.etree.cElementTree - Rationale: Obsolete, use xml.etree.ElementTree - Date: 2019-04-06 - Documentation: Documented as deprecated since 3.3 - -Deprecation of modules removed in Python 3.0 -============================================ - -PEP 3108 lists all modules that have been removed from Python 3.0. -They all are documented as deprecated in Python 2.6, and raise a -DeprecationWarning if the -3 flag is activated. - - -Undeprecated modules -==================== - -None. - - -Copyright -========= - -This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0011.txt b/pep-0011.txt deleted file mode 100644 index 482b7c894..000000000 --- a/pep-0011.txt +++ /dev/null @@ -1,297 +0,0 @@ -PEP: 11 -Title: Removing support for little used platforms -Version: $Revision$ -Last-Modified: $Date$ -Author: Martin von Löwis <martin@v.loewis.de>, - Brett Cannon <brett@python.org> -Status: Active -Type: Process -Content-Type: text/x-rst -Created: 07-Jul-2002 -Post-History: 18-Aug-2007 - 16-May-2014 - 20-Feb-2015 - - -Abstract --------- - -This PEP documents how an operating system (platform) becomes -supported in CPython and documents past support. - - -Rationale ---------- - -Over time, the CPython source code has collected various pieces of -platform-specific code, which, at some point in time, was -considered necessary to use Python on a specific platform. -Without access to this platform, it is not possible to determine -whether this code is still needed. As a result, this code may -either break during Python's evolution, or it may become -unnecessary as the platforms evolve as well. - -The growing amount of these fragments poses the risk of -unmaintainability: without having experts for a large number of -platforms, it is not possible to determine whether a certain -change to the CPython source code will work on all supported -platforms. - -To reduce this risk, this PEP specifies what is required for a -platform to be considered supported by Python as well as providing a -procedure to remove code for platforms with few or no Python -users. - -Supporting platforms --------------------- - -Gaining official platform support requires two things. First, a core -developer needs to volunteer to maintain platform-specific code. This -core developer can either already be a member of the Python -development team or be given contributor rights on the basis of -maintaining platform support (it is at the discretion of the Python -development team to decide if a person is ready to have such rights -even if it is just for supporting a specific platform). - -Second, a stable buildbot must be provided [2]_. This guarantees that -platform support will not be accidentally broken by a Python core -developer who does not have personal access to the platform. For a -buildbot to be considered stable it requires that the machine be -reliably up and functioning (but it is up to the Python core -developers to decide whether to promote a buildbot to being -considered stable). - -This policy does not disqualify supporting other platforms -indirectly. Patches which are not platform-specific but still done to -add platform support will be considered for inclusion. For example, -if platform-independent changes were necessary in the configure -script which were motivated to support a specific platform that could -be accepted. Patches which add platform-specific code such as the -name of a specific platform to the configure script will generally -not be accepted without the platform having official support. - -CPU architecture and compiler support are viewed in a similar manner -as platforms. For example, to consider the ARM architecture supported -a buildbot running on ARM would be required along with support from -the Python development team. In general it is not required to have -a CPU architecture run under every possible platform in order to be -considered supported. - -Unsupporting platforms ----------------------- - -If a certain platform that currently has special code in CPython is -deemed to be without enough Python users or lacks proper support from -the Python development team and/or a buildbot, a note must be posted -in this PEP that this platform is no longer actively supported. This -note must include: - -- the name of the system -- the first release number that does not support this platform - anymore, and -- the first release where the historical support code is actively - removed - -In some cases, it is not possible to identify the specific list of -systems for which some code is used (e.g. when autoconf tests for -absence of some feature which is considered present on all -supported systems). In this case, the name will give the precise -condition (usually a preprocessor symbol) that will become -unsupported. - -At the same time, the CPython source code must be changed to -produce a build-time error if somebody tries to install Python on -this platform. On platforms using autoconf, configure must fail. -This gives potential users of the platform a chance to step -forward and offer maintenance. - - -Re-supporting platforms ------------------------ - -If a user of a platform wants to see this platform supported -again, he may volunteer to maintain the platform support. Such an -offer must be recorded in the PEP, and the user can submit patches -to remove the build-time errors, and perform any other maintenance -work for the platform. - -Microsoft Windows ------------------ - -Microsoft has established a policy called product support lifecycle -[1]_. Each product's lifecycle has a mainstream support phase, where -the product is generally commercially available, and an extended -support phase, where paid support is still available, and certain bug -fixes are released (in particular security fixes). - -CPython's Windows support now follows this lifecycle. A new feature -release X.Y.0 will support all Windows releases whose extended support -phase is not yet expired. Subsequent bug fix releases will support -the same Windows releases as the original feature release (even if -the extended support phase has ended). - -Because of this policy, no further Windows releases need to be listed -in this PEP. - -Each feature release is built by a specific version of Microsoft -Visual Studio. That version should have mainstream support when the -release is made. Developers of extension modules will generally need -to use the same Visual Studio release; they are concerned both with -the availability of the versions they need to use, and with keeping -the zoo of versions small. The CPython source tree will keep -unmaintained build files for older Visual Studio releases, for which -patches will be accepted. Such build files will be removed from the -source tree 3 years after the extended support for the compiler has -ended (but continue to remain available in revision control). - -Legacy C Locale ---------------- - -Starting with CPython 3.7.0, \*nix platforms are expected to provide -at least one of ``C.UTF-8`` (full locale), ``C.utf8`` (full locale) or -``UTF-8`` (``LC_CTYPE``-only locale) as an alternative to the legacy ``C`` -locale. - -Any Unicode-related integration problems that occur only in the legacy ``C`` -locale and cannot be reproduced in an appropriately configured non-ASCII -locale will be closed as "won't fix". - -No-longer-supported platforms ------------------------------ - -* | Name: MS-DOS, MS-Windows 3.x - | Unsupported in: Python 2.0 - | Code removed in: Python 2.1 - -* | Name: SunOS 4 - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: DYNIX - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: dgux - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: Minix - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: Irix 4 and --with-sgi-dl - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: Linux 1 - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: Systems defining __d6_pthread_create (configure.in) - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: Systems defining PY_PTHREAD_D4, PY_PTHREAD_D6, - or PY_PTHREAD_D7 in thread_pthread.h - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: Systems using --with-dl-dld - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: Systems using --without-universal-newlines, - | Unsupported in: Python 2.3 - | Code removed in: Python 2.4 - -* | Name: MacOS 9 - | Unsupported in: Python 2.4 - | Code removed in: Python 2.4 - -* | Name: Systems using --with-wctype-functions - | Unsupported in: Python 2.6 - | Code removed in: Python 2.6 - -* | Name: Win9x, WinME, NT4 - | Unsupported in: Python 2.6 (warning in 2.5 installer) - | Code removed in: Python 2.6 - -* | Name: AtheOS - | Unsupported in: Python 2.6 (with "AtheOS" changed to "Syllable") - | Build broken in: Python 2.7 (edit configure to reenable) - | Code removed in: Python 3.0 - | Details: http://www.syllable.org/discussion.php?id=2320 - -* | Name: BeOS - | Unsupported in: Python 2.6 (warning in configure) - | Build broken in: Python 2.7 (edit configure to reenable) - | Code removed in: Python 3.0 - -* | Name: Systems using Mach C Threads - | Unsupported in: Python 3.2 - | Code removed in: Python 3.3 - -* | Name: SunOS lightweight processes (LWP) - | Unsupported in: Python 3.2 - | Code removed in: Python 3.3 - -* | Name: Systems using --with-pth (GNU pth threads) - | Unsupported in: Python 3.2 - | Code removed in: Python 3.3 - -* | Name: Systems using Irix threads - | Unsupported in: Python 3.2 - | Code removed in: Python 3.3 - -* | Name: OSF* systems (issue 8606) - | Unsupported in: Python 3.2 - | Code removed in: Python 3.3 - -* | Name: OS/2 (issue 16135) - | Unsupported in: Python 3.3 - | Code removed in: Python 3.4 - -* | Name: VMS (issue 16136) - | Unsupported in: Python 3.3 - | Code removed in: Python 3.4 - -* | Name: Windows 2000 - | Unsupported in: Python 3.3 - | Code removed in: Python 3.4 - -* | Name: Windows systems where COMSPEC points to command.com - | Unsupported in: Python 3.3 - | Code removed in: Python 3.4 - -* | Name: RISC OS - | Unsupported in: Python 3.0 (some code actually removed) - | Code removed in: Python 3.4 - -* | Name: IRIX - | Unsupported in: Python 3.7 - | Code removed in: Python 3.7 - -* | Name: Systems without multithreading support - | Unsupported in: Python 3.7 - | Code removed in: Python 3.7 - -References ----------- - -.. [1] http://support.microsoft.com/lifecycle/ -.. [2] http://buildbot.python.org/3.x.stable/ - -Copyright ---------- - -This document has been placed in the public domain. - - - .. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0458-1.png b/pep-0458-1.png deleted file mode 100644 index 52bea1b29..000000000 Binary files a/pep-0458-1.png and /dev/null differ diff --git a/pep-0480-1.png b/pep-0480-1.png deleted file mode 100644 index a31edf170..000000000 Binary files a/pep-0480-1.png and /dev/null differ diff --git a/pep-0495-daylightsavings.png b/pep-0495-daylightsavings.png deleted file mode 100644 index 18a1f6c2b..000000000 Binary files a/pep-0495-daylightsavings.png and /dev/null differ diff --git a/pep-0495-fold-2.png b/pep-0495-fold-2.png deleted file mode 100644 index d09eb41f7..000000000 Binary files a/pep-0495-fold-2.png and /dev/null differ diff --git a/pep-0495-gap.png b/pep-0495-gap.png deleted file mode 100644 index e3ba3cb77..000000000 Binary files a/pep-0495-gap.png and /dev/null differ diff --git a/pep-0495-gap.svg b/pep-0495-gap.svg deleted file mode 100644 index 658d9df55..000000000 --- a/pep-0495-gap.svg +++ /dev/null @@ -1,437 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="no"?> -<!-- Created with Inkscape (http://www.inkscape.org/) --> - -<svg - xmlns:dc="http://purl.org/dc/elements/1.1/" - xmlns:cc="http://creativecommons.org/ns#" - xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" - xmlns:svg="http://www.w3.org/2000/svg" - xmlns="http://www.w3.org/2000/svg" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" - xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" - width="150mm" - height="140mm" - viewBox="0 0 531.49606 496.06299" - id="svg14800" - version="1.1" - inkscape:version="0.91 r13725" - sodipodi:docname="pep-0495-gap.svg" - inkscape:export-filename="/Users/a/Work/peps/pep-0495-fold.png" - inkscape:export-xdpi="90" - inkscape:export-ydpi="90"> - <defs - id="defs14802"> - <marker - inkscape:stockid="DotM" - orient="auto" - refY="0" - refX="0" - id="DotM" - style="overflow:visible" - inkscape:isstock="true"> - <path - id="path6980" - d="m -2.5,-1 c 0,2.76 -2.24,5 -5,5 -2.76,0 -5,-2.24 -5,-5 0,-2.76 2.24,-5 5,-5 2.76,0 5,2.24 5,5 z" - style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1" - transform="matrix(0.4,0,0,0.4,2.96,0.4)" - inkscape:connector-curvature="0" /> - </marker> - <marker - inkscape:stockid="DiamondSstart" - orient="auto" - refY="0" - refX="0" - id="DiamondSstart" - style="overflow:visible" - inkscape:isstock="true"> - <path - id="path7010" - d="M 0,-7.0710768 -7.0710894,0 0,7.0710589 7.0710462,0 0,-7.0710768 Z" - style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1" - transform="matrix(0.2,0,0,0.2,1.2,0)" - inkscape:connector-curvature="0" /> - </marker> - <marker - inkscape:stockid="Arrow2Mend" - orient="auto" - refY="0" - refX="0" - id="Arrow2Mend" - style="overflow:visible" - inkscape:isstock="true"> - <path - id="path6943" - style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1" - d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z" - transform="scale(-0.6,-0.6)" - inkscape:connector-curvature="0" /> - </marker> - <pattern - inkscape:collect="always" - xlink:href="#pattern15623" - id="pattern15646" - patternTransform="translate(0,2.8515625e-5)" /> - <pattern - inkscape:collect="always" - xlink:href="#Strips1_1" - id="pattern15599" - patternTransform="matrix(10,0,0,10,424.80508,-468.3217)" /> - <pattern - inkscape:isstock="true" - inkscape:stockid="Stripes 1:1" - id="Strips1_1" - patternTransform="translate(0,0) scale(10,10)" - height="1" - width="2" - patternUnits="userSpaceOnUse" - inkscape:collect="always"> - <rect - id="rect6108" - height="2" - width="1" - y="-0.5" - x="0" - style="fill:black;stroke:none" /> - </pattern> - <marker - inkscape:stockid="Arrow1Lstart" - orient="auto" - refY="0" - refX="0" - id="Arrow1Lstart" - style="overflow:visible" - inkscape:isstock="true"> - <path - id="path6916" - d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z" - style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1" - transform="matrix(0.8,0,0,0.8,10,0)" - inkscape:connector-curvature="0" /> - </marker> - <marker - inkscape:stockid="Arrow1Lend" - orient="auto" - refY="0" - refX="0" - id="Arrow1Lend" - style="overflow:visible" - inkscape:isstock="true"> - <path - id="path6919" - d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z" - style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1" - transform="matrix(-0.8,0,0,-0.8,-10,0)" - inkscape:connector-curvature="0" /> - </marker> - <marker - inkscape:stockid="Arrow1Mend" - orient="auto" - refY="0" - refX="0" - id="Arrow1Mend" - style="overflow:visible" - inkscape:isstock="true"> - <path - id="path6925" - d="M 0,0 5,-5 -12.5,0 5,5 0,0 Z" - style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1" - transform="matrix(-0.4,0,0,-0.4,-4,0)" - inkscape:connector-curvature="0" /> - </marker> - <pattern - patternUnits="userSpaceOnUse" - width="265.19116" - height="51.983494" - patternTransform="translate(-424.80508,468.3217)" - id="pattern15596"> - <path - inkscape:connector-curvature="0" - id="path15588" - d="m 0.376692,25.991752 0,-25.61506 132.218888,0 132.21889,0 0,25.61506 0,25.61505 -132.21889,0 -132.218888,0 0,-25.61505 z" - style="opacity:0.5;fill:url(#pattern15599);fill-opacity:1;stroke:#ffd640;stroke-width:0.75338399;stroke-linecap:round;stroke-miterlimit:4;stroke-dasharray:0.75338398, 0.75338398;stroke-dashoffset:0;stroke-opacity:1" /> - </pattern> - <pattern - patternUnits="userSpaceOnUse" - width="213.59843" - height="36.4331" - patternTransform="translate(-0.5,1122.7283)" - id="pattern15623"> - <path - inkscape:connector-curvature="0" - id="path15613" - d="m 0.5,0.5 212.59843,0 0,17.7166 -212.59843,0 z" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:10;stroke-opacity:1" /> - <path - inkscape:connector-curvature="0" - id="path15615" - d="m 0.5,18.2166 0,17.7165 212.59843,0 0,-17.7165" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:10;stroke-opacity:1" /> - <path - inkscape:connector-curvature="0" - id="path15617" - d="m 0.98017929,9.3247 0,-7.9105 105.47376071,0 105.47375,0 0,7.9105 0,7.9105 -105.47375,0 -105.47376071,0 0,-7.9105 z" - style="opacity:0.5;fill:#ffd744;fill-opacity:1;stroke:#ffd744;stroke-width:0.75338399;stroke-linecap:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:7.5338397;stroke-opacity:0.50196078" /> - <path - inkscape:connector-curvature="0" - id="path15621" - d="m 0.98017929,27.0292 0,-8.2872 105.47376071,0 105.47375,0 0,8.2872 0,8.2872 -105.47375,0 -105.47376071,0 0,-8.2872 z" - style="opacity:0.5;fill:#326c9c;fill-opacity:1;stroke:#326c9b;stroke-width:0.75338399;stroke-linecap:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:7.5338397;stroke-opacity:0.50196078" /> - </pattern> - <pattern - patternUnits="userSpaceOnUse" - width="213.59843" - height="36.433102" - patternTransform="translate(-0.5,1122.7283)" - id="pattern15643"> - <rect - id="rect15629" - y="0" - x="0" - height="36.433102" - width="213.59843" - style="fill:url(#pattern15646);stroke:none" /> - </pattern> - </defs> - <sodipodi:namedview - id="base" - pagecolor="#ffffff" - bordercolor="#666666" - borderopacity="1.0" - inkscape:pageopacity="0.0" - inkscape:pageshadow="2" - inkscape:zoom="2.8284272" - inkscape:cx="215.26543" - inkscape:cy="232.89973" - inkscape:document-units="mm" - inkscape:current-layer="layer2" - showgrid="true" - inkscape:window-width="2556" - inkscape:window-height="1555" - inkscape:window-x="1" - inkscape:window-y="0" - inkscape:window-maximized="0" - objecttolerance="10000" - showborder="false" - fit-margin-top="0" - fit-margin-left="0" - fit-margin-right="0" - fit-margin-bottom="0"> - <inkscape:grid - type="xygrid" - id="grid14808" - originx="37.568003" - spacingx="17.716536" - spacingy="17.716536" - empspacing="3" - originy="-71.39131" /> - </sodipodi:namedview> - <metadata - id="metadata14805"> - <rdf:RDF> - <cc:Work - rdf:about=""> - <dc:format>image/svg+xml</dc:format> - <dc:type - rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> - <dc:title></dc:title> - </cc:Work> - </rdf:RDF> - </metadata> - <g - inkscape:label="Layer 1" - inkscape:groupmode="layer" - id="layer1" - transform="translate(37.568003,-484.90789)"> - <path - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.99921262;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow1Lstart);marker-end:url(#Arrow1Lend)" - d="M 476.5503,945.88825 0,946.42873 0,521.76422" - id="path14810" - inkscape:connector-curvature="0" - sodipodi:nodetypes="ccc" /> - <flowRoot - xml:space="preserve" - id="flowRoot15458" - style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"><flowRegion - id="flowRegion15460"><rect - id="rect15462" - width="159.44882" - height="106.29922" - x="-425.19687" - y="946.06299" /></flowRegion><flowPara - id="flowPara15464" /></flowRoot> <flowRoot - xml:space="preserve" - id="flowRoot15466" - style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:40px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"><flowRegion - id="flowRegion15468"><rect - id="rect15470" - width="159.44882" - height="88.58268" - x="212.59843" - y="1070.0787" - style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:40px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:center;writing-mode:lr-tb;text-anchor:middle" /></flowRegion><flowPara - id="flowPara15474" /></flowRoot> <flowRoot - xml:space="preserve" - id="flowRoot15480" - style="font-style:normal;font-weight:normal;font-size:40px;line-height:125%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"><flowRegion - id="flowRegion15482"><rect - id="rect15484" - width="70.866142" - height="53.149609" - x="212.59843" - y="1105.5118" /></flowRegion><flowPara - id="flowPara15486" /></flowRoot> <flowRoot - xml:space="preserve" - id="flowRoot15488" - style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - transform="translate(270.90867,-112.71393)"><flowRegion - id="flowRegion15490"><rect - id="rect15492" - width="265.74805" - height="88.58268" - x="159.44882" - y="1070.0787" - style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;writing-mode:lr-tb;text-anchor:start" /></flowRegion><flowPara - id="flowPara15496">UTC</flowPara></flowRoot> <text - xml:space="preserve" - style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:22.5px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - x="-570.61304" - y="-20.473276" - id="text15498" - sodipodi:linespacing="125%" - transform="matrix(0,-1,1,0,0,0)"><tspan - sodipodi:role="line" - id="tspan15500" - x="-570.61304" - y="-20.473276">local</tspan></text> - <path - style="fill:none;fill-rule:evenodd;stroke:#336d9c;stroke-width:2.12598419;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" - d="M 52.152923,893.91006 266.74473,679.31828" - id="path15502" - inkscape:connector-curvature="0" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#336d9c;stroke-width:2.12598419;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" - d="M 265.74804,733.46456 425.19686,574.01574" - id="path15504" - inkscape:connector-curvature="0" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#336d9c;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1, 12;stroke-dashoffset:0;stroke-opacity:1" - d="m 265.74804,680.31496 0,53.1496 z" - id="path15678" - inkscape:connector-curvature="0" - sodipodi:nodetypes="ccc" /> - <text - xml:space="preserve" - style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:20px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Italic';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - x="-17.04035" - y="703.841" - id="text16422" - sodipodi:linespacing="125%"><tspan - sodipodi:role="line" - id="tspan16424" - x="-17.04035" - y="703.841">t</tspan></text> - <text - xml:space="preserve" - style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:20px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Italic';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - x="240.81497" - y="962.27954" - id="text16438" - sodipodi:linespacing="125%"><tspan - sodipodi:role="line" - id="tspan16440" - x="240.81497" - y="962.27954">u<tspan - style="font-size:64.99999762%;baseline-shift:sub" - id="tspan16442">0</tspan></tspan></text> - <text - xml:space="preserve" - style="font-style:italic;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:20px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Italic';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - x="294.96457" - y="963.77954" - id="text16444" - sodipodi:linespacing="125%"><tspan - sodipodi:role="line" - id="tspan16446" - x="294.96457" - y="963.77954">u<tspan - style="font-size:64.99999762%;baseline-shift:sub" - id="tspan16448">1</tspan></tspan></text> - <path - style="fill:none;fill-rule:evenodd;stroke:#336d9c;stroke-width:7.08661413;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" - d="m 212.59843,941.81299 53.14961,0" - id="path16450" - inkscape:connector-curvature="0" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#336d9c;stroke-width:7.08661413;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" - d="m 4.2499999,733.46456 0,-53.1496" - id="path16452" - inkscape:connector-curvature="0" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#ffd847;stroke-width:7.08661413;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" - d="m 265.74804,941.81299 53.14961,0" - id="path16454" - inkscape:connector-curvature="0" /> - <text - xml:space="preserve" - style="font-style:italic;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:20px;line-height:125%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Bold Italic';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" - x="343.96481" - y="712.6087" - id="text16458" - sodipodi:linespacing="125%"><tspan - sodipodi:role="line" - id="tspan16460" - x="343.96481" - y="712.6087">Fold</tspan></text> - <path - style="fill:none;fill-rule:evenodd;stroke:#336d9c;stroke-width:2.12598425;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none" - d="m 265.74804,680.31492 0,53.14961" - id="path16481" - inkscape:connector-curvature="0" /> - <path - style="fill:none;fill-rule:evenodd;stroke:#ffd847;stroke-width:7.08661413;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" - d="m 11.716536,733.46456 0,-53.1496" - id="path16456" - inkscape:connector-curvature="0" /> - </g> - <g - inkscape:groupmode="layer" - id="layer2" - inkscape:label="Layer 2"> - <path - transform="translate(37.568003,-484.90789)" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 2;stroke-dashoffset:0;stroke-opacity:1" - d="m 0,698.03149 248.0315,0 0,247.85676" - id="path15680" - inkscape:connector-curvature="0" - sodipodi:nodetypes="ccc" /> - <path - transform="translate(37.568003,-484.90789)" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:4, 2;stroke-dashoffset:0;stroke-opacity:1" - d="m 248.0315,698.03149 53.14961,0 0,247.85676" - id="path15682" - inkscape:connector-curvature="0" - sodipodi:nodetypes="ccc" /> - <path - transform="translate(37.568003,-484.90789)" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.99921262;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:0.9992126, 11.99055118000000064;stroke-dashoffset:0;stroke-opacity:1" - d="m 0,680.31496 318.89765,0 0,265.57329" - id="path15566" - inkscape:connector-curvature="0" - sodipodi:nodetypes="ccc" /> - <path - transform="translate(37.568003,-484.90789)" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.99921262;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:0.99921262, 7.99370097999999984;stroke-dashoffset:0;stroke-opacity:1" - d="m 212.59843,733.46456 0,212.42369" - id="path15676" - inkscape:connector-curvature="0" - sodipodi:nodetypes="cc" /> - <path - transform="translate(37.568003,-484.90789)" - style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1, 12;stroke-dashoffset:0;stroke-opacity:1" - d="m 0,733.46456 265.74804,0 0,211.88321" - id="path15552" - inkscape:connector-curvature="0" - sodipodi:nodetypes="ccc" /> - </g> -</svg> diff --git a/pep-0525-1.png b/pep-0525-1.png deleted file mode 100644 index 0940967a4..000000000 Binary files a/pep-0525-1.png and /dev/null differ diff --git a/pep-0554.rst b/pep-0554.rst deleted file mode 100644 index 8e7b4c66f..000000000 --- a/pep-0554.rst +++ /dev/null @@ -1,1730 +0,0 @@ -PEP: 554 -Title: Multiple Interpreters in the Stdlib -Author: Eric Snow <ericsnowcurrently@gmail.com> -BDFL-Delegate: Antoine Pitrou <antoine@python.org> -Status: Draft -Type: Standards Track -Content-Type: text/x-rst -Created: 05-Sep-2017 -Python-Version: 3.10 -Post-History: 07-Sep-2017, 08-Sep-2017, 13-Sep-2017, 05-Dec-2017, - 09-May-2018, 20-Apr-2020, 04-May-2020 - - -Abstract -======== - -CPython has supported multiple interpreters in the same process (AKA -"subinterpreters") since version 1.5 (1997). The feature has been -available via the C-API. [c-api]_ Subinterpreters operate in -`relative isolation from one another <Interpreter Isolation_>`_, which -facilitates novel alternative approaches to -`concurrency <Concurrency_>`_. - -This proposal introduces the stdlib ``interpreters`` module. The module -will be `provisional <Provisional Status_>`_. It exposes the basic -functionality of subinterpreters already provided by the C-API, along -with new (basic) functionality for sharing data between interpreters. - - -A Disclaimer about the GIL -========================== - -To avoid any confusion up front: This PEP is unrelated to any efforts -to stop sharing the GIL between subinterpreters. At most this proposal -will allow users to take advantage of any results of work on the GIL. -The position here is that exposing subinterpreters to Python code is -worth doing, even if they still share the GIL. - - -Proposal -======== - -The ``interpreters`` module will be added to the stdlib. To help -authors of extension modules, a new page will be added to the -`Extending Python <extension-docs_>`_ docs. More information on both -is found in the immediately following sections. - -The "interpreters" Module -------------------------- - -The ``interpreters`` module will -provide a high-level interface to subinterpreters and wrap a new -low-level ``_interpreters`` (in the same way as the ``threading`` -module). See the `Examples`_ section for concrete usage and use cases. - -Along with exposing the existing (in CPython) subinterpreter support, -the module will also provide a mechanism for sharing data between -interpreters. This mechanism centers around "channels", which are -similar to queues and pipes. - -Note that *objects* are not shared between interpreters since they are -tied to the interpreter in which they were created. Instead, the -objects' *data* is passed between interpreters. See the `Shared data`_ -section for more details about sharing between interpreters. - -At first only the following types will be supported for sharing: - -* None -* bytes -* str -* int -* PEP 554 channels - -Support for other basic types (e.g. bool, float, Ellipsis) will be added later. - -API summary for interpreters module ------------------------------------ - -Here is a summary of the API for the ``interpreters`` module. For a -more in-depth explanation of the proposed classes and functions, see -the `"interpreters" Module API`_ section below. - -For creating and using interpreters: - -+---------------------------------------------+----------------------------------------------+ -| signature | description | -+=============================================+==============================================+ -| ``list_all() -> [Interpreter]`` | Get all existing interpreters. | -+---------------------------------------------+----------------------------------------------+ -| ``get_current() -> Interpreter`` | Get the currently running interpreter. | -+---------------------------------------------+----------------------------------------------+ -| ``get_main() -> Interpreter`` | Get the main interpreter. | -+---------------------------------------------+----------------------------------------------+ -| ``create(*, isolated=True) -> Interpreter`` | Initialize a new (idle) Python interpreter. | -+---------------------------------------------+----------------------------------------------+ - -| - -+----------------------------------------+-----------------------------------------------------+ -| signature | description | -+========================================+=====================================================+ -| ``class Interpreter(id)`` | A single interpreter. | -+----------------------------------------+-----------------------------------------------------+ -| ``.id`` | The interpreter's ID (read-only). | -+----------------------------------------+-----------------------------------------------------+ -| ``.isolated`` | The interpreter's mode (read-only). | -+----------------------------------------+-----------------------------------------------------+ -| ``.is_running() -> bool`` | Is the interpreter currently executing code? | -+----------------------------------------+-----------------------------------------------------+ -| ``.close()`` | Finalize and destroy the interpreter. | -+----------------------------------------+-----------------------------------------------------+ -| ``.run(src_str, /, *, channels=None)`` | | Run the given source code in the interpreter. | -| | | (This blocks the current thread until done.) | -+----------------------------------------+-----------------------------------------------------+ - -| - -+--------------------+------------------+------------------------------------------------------+ -| exception | base | description | -+====================+==================+======================================================+ -| ``RunFailedError`` | ``RuntimeError`` | Interpreter.run() resulted in an uncaught exception. | -+--------------------+------------------+------------------------------------------------------+ - -For sharing data between interpreters: - -+---------------------------------------------------------+--------------------------------------------+ -| signature | description | -+=========================================================+============================================+ -| ``is_shareable(obj) -> Bool`` | | Can the object's data be shared | -| | | between interpreters? | -+---------------------------------------------------------+--------------------------------------------+ -| ``create_channel() -> (RecvChannel, SendChannel)`` | | Create a new channel for passing | -| | | data between interpreters. | -+---------------------------------------------------------+--------------------------------------------+ -| ``list_all_channels() -> [(RecvChannel, SendChannel)]`` | Get all open channels. | -+---------------------------------------------------------+--------------------------------------------+ - -| - -+------------------------------------------+-----------------------------------------------+ -| signature | description | -+==========================================+===============================================+ -| ``class RecvChannel(id)`` | The receiving end of a channel. | -+------------------------------------------+-----------------------------------------------+ -| ``.id`` | The channel's unique ID. | -+------------------------------------------+-----------------------------------------------+ -| ``.recv() -> object`` | | Get the next object from the channel, | -| | | and wait if none have been sent. | -+------------------------------------------+-----------------------------------------------+ -| ``.recv_nowait(default=None) -> object`` | | Like recv(), but return the default | -| | | instead of waiting. | -+------------------------------------------+-----------------------------------------------+ - -| - -+------------------------------+--------------------------------------------------+ -| signature | description | -+==============================+==================================================+ -| ``class SendChannel(id)`` | The sending end of a channel. | -+------------------------------+--------------------------------------------------+ -| ``.id`` | The channel's unique ID. | -+------------------------------+--------------------------------------------------+ -| ``.send(obj)`` | | Send the object (i.e. its data) to the | -| | | receiving end of the channel and wait. | -+------------------------------+--------------------------------------------------+ -| ``.send_nowait(obj)`` | | Like send(), but return False if not received. | -+------------------------------+--------------------------------------------------+ - -| - -+--------------------------+------------------------+------------------------------------------------+ -| exception | base | description | -+==========================+========================+================================================+ -| ``ChannelError`` | ``Exception`` | The base class for channel-related exceptions. | -+--------------------------+------------------------+------------------------------------------------+ -| ``ChannelNotFoundError`` | ``ChannelError`` | The identified channel was not found. | -+--------------------------+------------------------+------------------------------------------------+ -| ``ChannelEmptyError`` | ``ChannelError`` | The channel was unexpectedly empty. | -+--------------------------+------------------------+------------------------------------------------+ -| ``ChannelNotEmptyError`` | ``ChannelError`` | The channel was unexpectedly not empty. | -+--------------------------+------------------------+------------------------------------------------+ -| ``NotReceivedError`` | ``ChannelError`` | Nothing was waiting to receive a sent object. | -+--------------------------+------------------------+------------------------------------------------+ - -Help for Extension Module Maintainers -------------------------------------- - -Many extension modules do not support use in subinterpreters yet. The -maintainers and users of such extension modules will both benefit when -they are updated to support subinterpreters. In the meantime users may -become confused by failures when using subinterpreters, which could -negatively impact extension maintainers. See `Concerns`_ below. - -To mitigate that impact and accelerate compatibility, we will do the -following: - -* be clear that extension modules are *not* required to support use in - subinterpreters -* raise ``ImportError`` when an incompatible (no PEP 489 support) module - is imported in a subinterpreter -* provide resources (e.g. docs) to help maintainers reach compatibility -* reach out to the maintainers of Cython and of the most used extension - modules (on PyPI) to get feedback and possibly provide assistance - - -Examples -======== - -Run isolated code ------------------ - -:: - - interp = interpreters.create() - print('before') - interp.run('print("during")') - print('after') - -Run in a thread ---------------- - -:: - - interp = interpreters.create() - def run(): - interp.run('print("during")') - t = threading.Thread(target=run) - print('before') - t.start() - print('after') - -Pre-populate an interpreter ---------------------------- - -:: - - interp = interpreters.create() - interp.run(tw.dedent(""" - import some_lib - import an_expensive_module - some_lib.set_up() - """)) - wait_for_request() - interp.run(tw.dedent(""" - some_lib.handle_request() - """)) - -Handling an exception ---------------------- - -:: - - interp = interpreters.create() - try: - interp.run(tw.dedent(""" - raise KeyError - """)) - except interpreters.RunFailedError as exc: - print(f"got the error from the subinterpreter: {exc}") - -Re-raising an exception ------------------------ - -:: - - interp = interpreters.create() - try: - try: - interp.run(tw.dedent(""" - raise KeyError - """)) - except interpreters.RunFailedError as exc: - raise exc.__cause__ - except KeyError: - print("got a KeyError from the subinterpreter") - -Note that this pattern is a candidate for later improvement. - -Synchronize using a channel ---------------------------- - -:: - - interp = interpreters.create() - r, s = interpreters.create_channel() - def run(): - interp.run(tw.dedent(""" - reader.recv() - print("during") - """), - shared=dict( - reader=r, - ), - ) - t = threading.Thread(target=run) - print('before') - t.start() - print('after') - s.send(b'') - -Sharing a file descriptor -------------------------- - -:: - - interp = interpreters.create() - r1, s1 = interpreters.create_channel() - r2, s2 = interpreters.create_channel() - def run(): - interp.run(tw.dedent(""" - fd = int.from_bytes( - reader.recv(), 'big') - for line in os.fdopen(fd): - print(line) - writer.send(b'') - """), - shared=dict( - reader=r, - writer=s2, - ), - ) - t = threading.Thread(target=run) - t.start() - with open('spamspamspam') as infile: - fd = infile.fileno().to_bytes(1, 'big') - s.send(fd) - r.recv() - -Passing objects via marshal ---------------------------- - -:: - - interp = interpreters.create() - r, s = interpreters.create_channel() - interp.run(tw.dedent(""" - import marshal - """), - shared=dict( - reader=r, - ), - ) - def run(): - interp.run(tw.dedent(""" - data = reader.recv() - while data: - obj = marshal.loads(data) - do_something(obj) - data = reader.recv() - """)) - t = threading.Thread(target=run) - t.start() - for obj in input: - data = marshal.dumps(obj) - s.send(data) - s.send(None) - -Passing objects via pickle --------------------------- - -:: - - interp = interpreters.create() - r, s = interpreters.create_channel() - interp.run(tw.dedent(""" - import pickle - """), - shared=dict( - reader=r, - ), - ) - def run(): - interp.run(tw.dedent(""" - data = reader.recv() - while data: - obj = pickle.loads(data) - do_something(obj) - data = reader.recv() - """)) - t = threading.Thread(target=run) - t.start() - for obj in input: - data = pickle.dumps(obj) - s.send(data) - s.send(None) - -Running a module ----------------- - -:: - - interp = interpreters.create() - main_module = mod_name - interp.run(f'import runpy; runpy.run_module({main_module!r})') - -Running as script (including zip archives & directories) --------------------------------------------------------- - -:: - - interp = interpreters.create() - main_script = path_name - interp.run(f"import runpy; runpy.run_path({main_script!r})") - -Running in a thread pool executor ---------------------------------- - -:: - - interps = [interpreters.create() for i in range(5)] - with concurrent.futures.ThreadPoolExecutor(max_workers=len(interps)) as pool: - print('before') - for interp in interps: - pool.submit(interp.run, 'print("starting"); print("stopping")' - print('after') - - -Rationale -========= - -Running code in multiple interpreters provides a useful level of -isolation within the same process. This can be leveraged in a number -of ways. Furthermore, subinterpreters provide a well-defined framework -in which such isolation may extended. - -Nick Coghlan explained some of the benefits through a comparison with -multi-processing [benefits]_:: - - [I] expect that communicating between subinterpreters is going - to end up looking an awful lot like communicating between - subprocesses via shared memory. - - The trade-off between the two models will then be that one still - just looks like a single process from the point of view of the - outside world, and hence doesn't place any extra demands on the - underlying OS beyond those required to run CPython with a single - interpreter, while the other gives much stricter isolation - (including isolating C globals in extension modules), but also - demands much more from the OS when it comes to its IPC - capabilities. - - The security risk profiles of the two approaches will also be quite - different, since using subinterpreters won't require deliberately - poking holes in the process isolation that operating systems give - you by default. - -CPython has supported subinterpreters, with increasing levels of -support, since version 1.5. While the feature has the potential -to be a powerful tool, subinterpreters have suffered from neglect -because they are not available directly from Python. Exposing the -existing functionality in the stdlib will help reverse the situation. - -This proposal is focused on enabling the fundamental capability of -multiple isolated interpreters in the same Python process. This is a -new area for Python so there is relative uncertainly about the best -tools to provide as companions to subinterpreters. Thus we minimize -the functionality we add in the proposal as much as possible. - -Concerns --------- - -* "subinterpreters are not worth the trouble" - -Some have argued that subinterpreters do not add sufficient benefit -to justify making them an official part of Python. Adding features -to the language (or stdlib) has a cost in increasing the size of -the language. So an addition must pay for itself. In this case, -subinterpreters provide a novel concurrency model focused on isolated -threads of execution. Furthermore, they provide an opportunity for -changes in CPython that will allow simultaneous use of multiple CPU -cores (currently prevented by the GIL). - -Alternatives to subinterpreters include threading, async, and -multiprocessing. Threading is limited by the GIL and async isn't -the right solution for every problem (nor for every person). -Multiprocessing is likewise valuable in some but not all situations. -Direct IPC (rather than via the multiprocessing module) provides -similar benefits but with the same caveat. - -Notably, subinterpreters are not intended as a replacement for any of -the above. Certainly they overlap in some areas, but the benefits of -subinterpreters include isolation and (potentially) performance. In -particular, subinterpreters provide a direct route to an alternate -concurrency model (e.g. CSP) which has found success elsewhere and -will appeal to some Python users. That is the core value that the -``interpreters`` module will provide. - -* "stdlib support for subinterpreters adds extra burden - on C extension authors" - -In the `Interpreter Isolation`_ section below we identify ways in -which isolation in CPython's subinterpreters is incomplete. Most -notable is extension modules that use C globals to store internal -state. PEP 3121 and PEP 489 provide a solution for most of the -problem, but one still remains. [petr-c-ext]_ Until that is resolved -(see PEP 573), C extension authors will face extra difficulty -to support subinterpreters. - -Consequently, projects that publish extension modules may face an -increased maintenance burden as their users start using subinterpreters, -where their modules may break. This situation is limited to modules -that use C globals (or use libraries that use C globals) to store -internal state. For numpy, the reported-bug rate is one every 6 -months. [bug-rate]_ - -Ultimately this comes down to a question of how often it will be a -problem in practice: how many projects would be affected, how often -their users will be affected, what the additional maintenance burden -will be for projects, and what the overall benefit of subinterpreters -is to offset those costs. The position of this PEP is that the actual -extra maintenance burden will be small and well below the threshold at -which subinterpreters are worth it. - -* "creating a new concurrency API deserves much more thought and - experimentation, so the new module shouldn't go into the stdlib - right away, if ever" - -Introducing an API for a new concurrency model, like happened with -asyncio, is an extremely large project that requires a lot of careful -consideration. It is not something that can be done a simply as this -PEP proposes and likely deserves significant time on PyPI to mature. -(See `Nathaniel's post <nathaniel-asyncio>`_ on python-dev.) - -However, this PEP does not propose any new concurrency API. At most -it exposes minimal tools (e.g. subinterpreters, channels) which may -be used to write code that follows patterns associated with (relatively) -new-to-Python `concurrency models <Concurrency_>`_. Those tools could -also be used as the basis for APIs for such concurrency models. -Again, this PEP does not propose any such API. - -* "there is no point to exposing subinterpreters if they still share - the GIL" -* "the effort to make the GIL per-interpreter is disruptive and risky" - -A common misconception is that this PEP also includes a promise that -subinterpreters will no longer share the GIL. When that is clarified, -the next question is "what is the point?". This is already answered -at length in this PEP. Just to be clear, the value lies in:: - - * increase exposure of the existing feature, which helps improve - the code health of the entire CPython runtime - * expose the (mostly) isolated execution of subinterpreters - * preparation for per-interpreter GIL - * encourage experimentation - -* "data sharing can have a negative impact on cache performance - in multi-core scenarios" - -(See [cache-line-ping-pong]_.) - -This shouldn't be a problem for now as we have no immediate plans -to actually share data between interpreters, instead focusing -on copying. - - -About Subinterpreters -===================== - -Concurrency ------------ - -Concurrency is a challenging area of software development. Decades of -research and practice have led to a wide variety of concurrency models, -each with different goals. Most center on correctness and usability. - -One class of concurrency models focuses on isolated threads of -execution that interoperate through some message passing scheme. A -notable example is `Communicating Sequential Processes`_ (CSP) (upon -which Go's concurrency is roughly based). The isolation inherent to -subinterpreters makes them well-suited to this approach. - -Shared data ------------ - -Subinterpreters are inherently isolated (with caveats explained below), -in contrast to threads. So the same communicate-via-shared-memory -approach doesn't work. Without an alternative, effective use of -concurrency via subinterpreters is significantly limited. - -The key challenge here is that sharing objects between interpreters -faces complexity due to various constraints on object ownership, -visibility, and mutability. At a conceptual level it's easier to -reason about concurrency when objects only exist in one interpreter -at a time. At a technical level, CPython's current memory model -limits how Python *objects* may be shared safely between interpreters; -effectively objects are bound to the interpreter in which they were -created. Furthermore, the complexity of *object* sharing increases as -subinterpreters become more isolated, e.g. after GIL removal. - -Consequently,the mechanism for sharing needs to be carefully considered. -There are a number of valid solutions, several of which may be -appropriate to support in Python. This proposal provides a single basic -solution: "channels". Ultimately, any other solution will look similar -to the proposed one, which will set the precedent. Note that the -implementation of ``Interpreter.run()`` will be done in a way that -allows for multiple solutions to coexist, but doing so is not -technically a part of the proposal here. - -Regarding the proposed solution, "channels", it is a basic, opt-in data -sharing mechanism that draws inspiration from pipes, queues, and CSP's -channels. [fifo]_ - -As simply described earlier by the API summary, -channels have two operations: send and receive. A key characteristic -of those operations is that channels transmit data derived from Python -objects rather than the objects themselves. When objects are sent, -their data is extracted. When the "object" is received in the other -interpreter, the data is converted back into an object owned by that -interpreter. - -To make this work, the mutable shared state will be managed by the -Python runtime, not by any of the interpreters. Initially we will -support only one type of objects for shared state: the channels provided -by ``create_channel()``. Channels, in turn, will carefully manage -passing objects between interpreters. - -This approach, including keeping the API minimal, helps us avoid further -exposing any underlying complexity to Python users. Along those same -lines, we will initially restrict the types that may be passed through -channels to the following: - -* None -* bytes -* str -* int -* channels - -Limiting the initial shareable types is a practical matter, reducing -the potential complexity of the initial implementation. There are a -number of strategies we may pursue in the future to expand supported -objects and object sharing strategies. - -Interpreter Isolation ---------------------- - -CPython's interpreters are intended to be strictly isolated from each -other. Each interpreter has its own copy of all modules, classes, -functions, and variables. The same applies to state in C, including in -extension modules. The CPython C-API docs explain more. [caveats]_ - -However, there are ways in which interpreters share some state. First -of all, some process-global state remains shared: - -* file descriptors -* builtin types (e.g. dict, bytes) -* singletons (e.g. None) -* underlying static module data (e.g. functions) for - builtin/extension/frozen modules - -There are no plans to change this. - -Second, some isolation is faulty due to bugs or implementations that did -not take subinterpreters into account. This includes things like -extension modules that rely on C globals. [cryptography]_ In these -cases bugs should be opened (some are already): - -* readline module hook functions (http://bugs.python.org/issue4202) -* memory leaks on re-init (http://bugs.python.org/issue21387) - -Finally, some potential isolation is missing due to the current design -of CPython. Improvements are currently going on to address gaps in this -area: - -* GC is not run per-interpreter [global-gc]_ -* at-exit handlers are not run per-interpreter [global-atexit]_ -* extensions using the ``PyGILState_*`` API are incompatible [gilstate]_ -* interpreters share memory management (e.g. allocators, gc) -* interpreters share the GIL - -Existing Usage --------------- - -Subinterpreters are not a widely used feature. In fact, the only -documented cases of widespread usage are -`mod_wsgi <https://github.com/GrahamDumpleton/mod_wsgi>`_, -`OpenStack Ceph <https://github.com/ceph/ceph/pull/14971>`_, and -`JEP <https://github.com/ninia/jep>`_. On the one hand, these cases -provide confidence that existing subinterpreter support is relatively -stable. On the other hand, there isn't much of a sample size from which -to judge the utility of the feature. - - -Provisional Status -================== - -The new ``interpreters`` module will be added with "provisional" status -(see PEP 411). This allows Python users to experiment with the feature -and provide feedback while still allowing us to adjust to that feedback. -The module will be provisional in Python 3.9 and we will make a decision -before the 3.10 release whether to keep it provisional, graduate it, or -remove it. This PEP will be updated accordingly. - -While the module is provisional, any changes to the API (or to behavior) -do not need to be reflected here, nor get approval by the BDFL-delegate. -However, such changes will still need to go through the normal processes -(BPO for smaller changes and python-dev/PEP for substantial ones). - - -Alternate Python Implementations -================================ - -I've solicited feedback from various Python implementors about support -for subinterpreters. Each has indicated that they would be able to -support subinterpreters (if they choose to) without a lot of -trouble. Here are the projects I contacted: - -* jython ([jython]_) -* ironpython (personal correspondence) -* pypy (personal correspondence) -* micropython (personal correspondence) - - -.. _interpreters-list-all: -.. _interpreters-get-current: -.. _interpreters-create: -.. _interpreters-Interpreter: - -"interpreters" Module API -========================= - -The module provides the following functions:: - - list_all() -> [Interpreter] - - Return a list of all existing interpreters. - - get_current() => Interpreter - - Return the currently running interpreter. - - get_main() => Interpreter - - Return the main interpreter. If the Python implementation - has no concept of a main interpreter then return None. - - create(*, isolated=True) -> Interpreter - - Initialize a new Python interpreter and return it. The - interpreter will be created in the current thread and will remain - idle until something is run in it. The interpreter may be used - in any thread and will run in whichever thread calls - ``interp.run()``. See "Interpreter Isolated Mode" below for - an explanation of the "isolated" parameter. - - -The module also provides the following class:: - - class Interpreter(id): - - id -> int: - - The interpreter's ID. (read-only) - - isolated -> bool: - - Whether or not the interpreter is operating in "isolated" mode. - (read-only) - - is_running() -> bool: - - Return whether or not the interpreter is currently executing - code. Calling this on the current interpreter will always - return True. - - close(): - - Finalize and destroy the interpreter. - - This may not be called on an already running interpreter. - Doing so results in a RuntimeError. - - run(source_str, /, *, channels=None): - - Run the provided Python source code in the interpreter. If - the "channels" keyword argument is provided (and is a mapping - of attribute names to channels) then it is added to the - interpreter's execution namespace (the interpreter's - "__main__" module). If any of the values are not RecvChannel - or SendChannel instances then ValueError gets raised. - - This may not be called on an already running interpreter. - Doing so results in a RuntimeError. - - A "run()" call is similar to a function call. Once it - completes, the code that called "run()" continues executing - (in the original interpreter). Likewise, if there is any - uncaught exception then it effectively (see below) propagates - into the code where ``run()`` was called. However, unlike - function calls (but like threads), there is no return value. - If any value is needed, pass it out via a channel. - - The big difference from functions is that "run()" executes - the code in an entirely different interpreter, with entirely - separate state. The state of the current interpreter in the - current OS thread is swapped out with the state of the target - interpreter (the one that will execute the code). When the - target finishes executing, the original interpreter gets - swapped back in and its execution resumes. - - So calling "run()" will effectively cause the current Python - thread to pause. Sometimes you won't want that pause, in - which case you should make the "run()" call in another thread. - To do so, add a function that calls "run()" and then run that - function in a normal "threading.Thread". - - Note that the interpreter's state is never reset, neither - before "run()" executes the code nor after. Thus the - interpreter state is preserved between calls to "run()". - This includes "sys.modules", the "builtins" module, and the - internal state of C extension modules. - - Also note that "run()" executes in the namespace of the - "__main__" module, just like scripts, the REPL, "-m", and - "-c". Just as the interpreter's state is not ever reset, the - "__main__" module is never reset. You can imagine - concatenating the code from each "run()" call into one long - script. This is the same as how the REPL operates. - - Supported code: source text. - -Uncaught Exceptions -------------------- - -Regarding uncaught exceptions in ``Interpreter.run()``, we noted that -they are "effectively" propagated into the code where ``run()`` was -called. To prevent leaking exceptions (and tracebacks) between -interpreters, we create a surrogate of the exception and its traceback -(see ``traceback.TracebackException``), set it to ``__cause__`` on a -new ``RunFailedError``, and raise that. - -Raising (a proxy of) the exception directly is problematic since it's -harder to distinguish between an error in the ``run()`` call and an -uncaught exception from the subinterpreter. - -.. _interpreters-is-shareable: -.. _interpreters-create-channel: -.. _interpreters-list-all-channels: -.. _interpreters-RecvChannel: -.. _interpreters-SendChannel: - -API for sharing data --------------------- - -Subinterpreters are less useful without a mechanism for sharing data -between them. Sharing actual Python objects between interpreters, -however, has enough potential problems that we are avoiding support -for that here. Instead, only minimum set of types will be supported. -Initially this will include ``None``, ``bytes``, ``str``, ``int``, -and channels. Further types may be supported later. - -The ``interpreters`` module provides a function that users may call -to determine whether an object is shareable or not:: - - is_shareable(obj) -> bool: - - Return True if the object may be shared between interpreters. - This does not necessarily mean that the actual objects will be - shared. Insead, it means that the objects' underlying data will - be shared in a cross-interpreter way, whether via a proxy, a - copy, or some other means. - -This proposal provides two ways to share such objects between -interpreters. - -First, channels may be passed to ``run()`` via the ``channels`` -keyword argument, where they are effectively injected into the target -interpreter's ``__main__`` module. While passing arbitrary shareable -objects this way is possible, doing so is mainly intended for sharing -meta-objects (e.g. channels) between interpreters. It is less useful -to pass other objects (like ``bytes``) to ``run`` directly. - -Second, the main mechanism for sharing objects (i.e. their data) between -interpreters is through channels. A channel is a simplex FIFO similar -to a pipe. The main difference is that channels can be associated with -zero or more interpreters on either end. Like queues, which are also -many-to-many, channels are buffered (though they also offer methods -with unbuffered semantics). - -Python objects are not shared between interpreters. However, in some -cases data those objects wrap is actually shared and not just copied. -One example might be PEP 3118 buffers. In those cases the object in the -original interpreter is kept alive until the shared data in the other -interpreter is no longer used. Then object destruction can happen like -normal in the original interpreter, along with the previously shared -data. - -The ``interpreters`` module provides the following functions related -to channels:: - - create_channel() -> (RecvChannel, SendChannel): - - Create a new channel and return (recv, send), the RecvChannel - and SendChannel corresponding to the ends of the channel. - - Both ends of the channel are supported "shared" objects (i.e. - may be safely shared by different interpreters. Thus they - may be passed as keyword arguments to "Interpreter.run()". - - list_all_channels() -> [(RecvChannel, SendChannel)]: - - Return a list of all open channel-end pairs. - -The module also provides the following channel-related classes:: - - class RecvChannel(id): - - The receiving end of a channel. An interpreter may use this to - receive objects from another interpreter. At first only a few - of the simple, immutable builtin types will be supported. - - id -> int: - - The channel's unique ID. This is shared with the "send" end. - - recv(): - - Return the next object from the channel. If none have been - sent then wait until the next send. - - At the least, the object will be equivalent to the sent object. - That will almost always mean the same type with the same data, - though it could also be a compatible proxy. Regardless, it may - use a copy of that data or actually share the data. - - recv_nowait(default=None): - - Return the next object from the channel. If none have been - sent then return the default. Otherwise, this is the same - as the "recv()" method. - - - class SendChannel(id): - - The sending end of a channel. An interpreter may use this to - send objects to another interpreter. At first only a few of - the simple, immutable builtin types will be supported. - - id -> int: - - The channel's unique ID. This is shared with the "recv" end. - - send(obj): - - Send the object (i.e. its data) to the "recv" end of the - channel. Wait until the object is received. If the object - is not shareable then ValueError is raised. - - send_nowait(obj): - - Send the object to the "recv" end of the channel. This - behaves the same as "send()", except for the waiting part. - If no interpreter is currently receiving (waiting on the - other end) then queue the object and return False. Otherwise - return True. - -Channel Lifespan ----------------- - -A channel is automatically closed and destroyed once there are no more -Python objects (e.g. ``RecvChannel`` and ``SendChannel``) referring -to it. So it is effectively triggered via garbage-collection of those -objects.. - - -.. _isolated-mode: - -Interpreter "Isolated" Mode -=========================== - -By default, every new interpreter created by ``interpreters.create()`` -has specific restrictions on any code it runs. This includes the -following: - -* importing an extension module fails if it does not implement the - PEP 489 API -* new threads of any kind are not allowed -* ``os.fork()`` is not allowed (so no ``multiprocessing``) -* ``os.exec*()``, AKA "fork+exec", is not allowed (so no ``subprocess``) - -This represents the full "isolated" mode of subinterpreters. It is -applied when ``interpreters.create()`` is called with the "isolated" -keyword-only argument set to ``True`` (the default). If -``interpreters.create(isolated=False)`` is called then none of those -restrictions is applied. - -One advantage of this approach is that it allows extension maintainers -to check subinterpreter compatibility before they implement the PEP 489 -API. Also note that ``isolated=False`` represents the historical -behavior when using the existing subinterpreters C-API, thus providing -backward compatibility. For the existing C-API itself, the default -remains ``isolated=False``. The same is true for the "main" module, so -existing use of Python will not change. - -We may choose to later loosen some of the above restrictions or provide -a way to enable/disable granular restrictions individually. Regardless, -requiring PEP 489 support from extension modules will always be a -default restriction. - - -Documentation -============= - -The new stdlib docs page for the ``interpreters`` module will include -the following: - -* (at the top) a clear note that subinterpreter support in extension - modules is not required -* some explanation about what subinterpreters are -* brief examples of how to use subinterpreters and channels -* a summary of the limitations of subinterpreters -* (for extension maintainers) a link to the resources for ensuring - subinterpreter compatibility -* much of the API information in this PEP - -A separate page will be added to the docs for resources to help -extension maintainers ensure their modules can be used safely in -subinterpreters, under `Extending Python <extension-docs>`_. The page -will include the following information: - -* a summary about subinterpreters (similar to the same in the new - ``interpreters`` module page and in the C-API docs) -* an explanation of how extension modules can be impacted -* how to implement PEP 489 support -* how to move from global module state to per-interpreter -* how to take advantage of PEP 384 (heap types), PEP 3121 - (module state), and PEP 573 -* strategies for dealing with 3rd party C libraries that keep their - own subinterpreter-incompatible global state - -Note that the documentation will play a large part in mitigating any -negative impact that the new ``interpreters`` module might have on -extension module maintainers. - -Also, the ``ImportError`` for incompatible extgension modules will have -a message that clearly says it is due to missing subinterpreter -compatibility and that extensions are not required to provide it. This -will help set user expectations properly. - - -Deferred Functionality -====================== - -In the interest of keeping this proposal minimal, the following -functionality has been left out for future consideration. Note that -this is not a judgement against any of said capability, but rather a -deferment. That said, each is arguably valid. - -Interpreter.call() ------------------- - -It would be convenient to run existing functions in subinterpreters -directly. ``Interpreter.run()`` could be adjusted to support this or -a ``call()`` method could be added:: - - Interpreter.call(f, *args, **kwargs) - -This suffers from the same problem as sharing objects between -interpreters via queues. The minimal solution (running a source string) -is sufficient for us to get the feature out where it can be explored. - -timeout arg to recv() and send() --------------------------------- - -Typically functions that have a ``block`` argument also have a -``timeout`` argument. It sometimes makes sense to do likewise for -functions that otherwise block, like the channel ``recv()`` and -``send()`` methods. We can add it later if needed. - -Interpreter.run_in_thread() ---------------------------- - -This method would make a ``run()`` call for you in a thread. Doing this -using only ``threading.Thread`` and ``run()`` is relatively trivial so -we've left it out. - -Synchronization Primitives --------------------------- - -The ``threading`` module provides a number of synchronization primitives -for coordinating concurrent operations. This is especially necessary -due to the shared-state nature of threading. In contrast, -subinterpreters do not share state. Data sharing is restricted to -channels, which do away with the need for explicit synchronization. If -any sort of opt-in shared state support is added to subinterpreters in -the future, that same effort can introduce synchronization primitives -to meet that need. - -CSP Library ------------ - -A ``csp`` module would not be a large step away from the functionality -provided by this PEP. However, adding such a module is outside the -minimalist goals of this proposal. - -Syntactic Support ------------------ - -The ``Go`` language provides a concurrency model based on CSP, so -it's similar to the concurrency model that subinterpreters support. -However, ``Go`` also provides syntactic support, as well several builtin -concurrency primitives, to make concurrency a first-class feature. -Conceivably, similar syntactic (and builtin) support could be added to -Python using subinterpreters. However, that is *way* outside the scope -of this PEP! - -Multiprocessing ---------------- - -The ``multiprocessing`` module could support subinterpreters in the same -way it supports threads and processes. In fact, the module's -maintainer, Davin Potts, has indicated this is a reasonable feature -request. However, it is outside the narrow scope of this PEP. - -C-extension opt-in/opt-out --------------------------- - -By using the ``PyModuleDef_Slot`` introduced by PEP 489, we could easily -add a mechanism by which C-extension modules could opt out of support -for subinterpreters. Then the import machinery, when operating in -a subinterpreter, would need to check the module for support. It would -raise an ImportError if unsupported. - -Alternately we could support opting in to subinterpreter support. -However, that would probably exclude many more modules (unnecessarily) -than the opt-out approach. Also, note that PEP 489 defined that an -extension's use of the PEP's machinery implies support for -subinterpreters. - -The scope of adding the ModuleDef slot and fixing up the import -machinery is non-trivial, but could be worth it. It all depends on -how many extension modules break under subinterpreters. Given that -there are relatively few cases we know of through mod_wsgi, we can -leave this for later. - -Poisoning channels ------------------- - -CSP has the concept of poisoning a channel. Once a channel has been -poisoned, any ``send()`` or ``recv()`` call on it would raise a special -exception, effectively ending execution in the interpreter that tried -to use the poisoned channel. - -This could be accomplished by adding a ``poison()`` method to both ends -of the channel. The ``close()`` method can be used in this way -(mostly), but these semantics are relatively specialized and can wait. - -Resetting __main__ ------------------- - -As proposed, every call to ``Interpreter.run()`` will execute in the -namespace of the interpreter's existing ``__main__`` module. This means -that data persists there between ``run()`` calls. Sometimes this isn't -desirable and you want to execute in a fresh ``__main__``. Also, -you don't necessarily want to leak objects there that you aren't using -any more. - -Note that the following won't work right because it will clear too much -(e.g. ``__name__`` and the other "__dunder__" attributes:: - - interp.run('globals().clear()') - -Possible solutions include: - -* a ``create()`` arg to indicate resetting ``__main__`` after each - ``run`` call -* an ``Interpreter.reset_main`` flag to support opting in or out - after the fact -* an ``Interpreter.reset_main()`` method to opt in when desired -* ``importlib.util.reset_globals()`` [reset_globals]_ - -Also note that resetting ``__main__`` does nothing about state stored -in other modules. So any solution would have to be clear about the -scope of what is being reset. Conceivably we could invent a mechanism -by which any (or every) module could be reset, unlike ``reload()`` -which does not clear the module before loading into it. Regardless, -since ``__main__`` is the execution namespace of the interpreter, -resetting it has a much more direct correlation to interpreters and -their dynamic state than does resetting other modules. So a more -generic module reset mechanism may prove unnecessary. - -This isn't a critical feature initially. It can wait until later -if desirable. - -Resetting an interpreter's state --------------------------------- - -It may be nice to re-use an existing subinterpreter instead of -spinning up a new one. Since an interpreter has substantially more -state than just the ``__main__`` module, it isn't so easy to put an -interpreter back into a pristine/fresh state. In fact, there *may* -be parts of the state that cannot be reset from Python code. - -A possible solution is to add an ``Interpreter.reset()`` method. This -would put the interpreter back into the state it was in when newly -created. If called on a running interpreter it would fail (hence the -main interpreter could never be reset). This would likely be more -efficient than creating a new subinterpreter, though that depends on -what optimizations will be made later to subinterpreter creation. - -While this would potentially provide functionality that is not -otherwise available from Python code, it isn't a fundamental -functionality. So in the spirit of minimalism here, this can wait. -Regardless, I doubt it would be controversial to add it post-PEP. - -File descriptors and sockets in channels ----------------------------------------- - -Given that file descriptors and sockets are process-global resources, -support for passing them through channels is a reasonable idea. They -would be a good candidate for the first effort at expanding the types -that channels support. They aren't strictly necessary for the initial -API. - -Integration with async ----------------------- - -Per Antoine Pitrou [async]_:: - - Has any thought been given to how FIFOs could integrate with async - code driven by an event loop (e.g. asyncio)? I think the model of - executing several asyncio (or Tornado) applications each in their - own subinterpreter may prove quite interesting to reconcile multi- - core concurrency with ease of programming. That would require the - FIFOs to be able to synchronize on something an event loop can wait - on (probably a file descriptor?). - -A possible solution is to provide async implementations of the blocking -channel methods (``recv()``, and ``send()``). However, -the basic functionality of subinterpreters does not depend on async and -can be added later. - -Alternately, "readiness callbacks" could be used to simplify use in -async scenarios. This would mean adding an optional ``callback`` -(kw-only) parameter to the ``recv_nowait()`` and ``send_nowait()`` -channel methods. The callback would be called once the object was sent -or received (respectively). - -(Note that making channels buffered makes readiness callbacks less -important.) - -Support for iteration ---------------------- - -Supporting iteration on ``RecvChannel`` (via ``__iter__()`` or -``_next__()``) may be useful. A trivial implementation would use the -``recv()`` method, similar to how files do iteration. Since this isn't -a fundamental capability and has a simple analog, adding iteration -support can wait until later. - -Channel context managers ------------------------- - -Context manager support on ``RecvChannel`` and ``SendChannel`` may be -helpful. The implementation would be simple, wrapping a call to -``close()`` (or maybe ``release()``) like files do. As with iteration, -this can wait. - -Pipes and Queues ----------------- - -With the proposed object passing mechanism of "channels", other similar -basic types aren't required to achieve the minimal useful functionality -of subinterpreters. Such types include pipes (like unbuffered channels, -but one-to-one) and queues (like channels, but more generic). See below -in `Rejected Ideas`_ for more information. - -Even though these types aren't part of this proposal, they may still -be useful in the context of concurrency. Adding them later is entirely -reasonable. The could be trivially implemented as wrappers around -channels. Alternatively they could be implemented for efficiency at the -same low level as channels. - -Return a lock from send() -------------------------- - -When sending an object through a channel, you don't have a way of knowing -when the object gets received on the other end. One way to work around -this is to return a locked ``threading.Lock`` from ``SendChannel.send()`` -that unlocks once the object is received. - -Alternately, the proposed ``SendChannel.send()`` (blocking) and -``SendChannel.send_nowait()`` provide an explicit distinction that is -less likely to confuse users. - -Note that returning a lock would matter for buffered channels -(i.e. queues). For unbuffered channels it is a non-issue. - -Support prioritization in channels ----------------------------------- - -A simple example is ``queue.PriorityQueue`` in the stdlib. - -Support inheriting settings (and more?) ---------------------------------------- - -Folks might find it useful, when creating a new subinterpreter, to be -able to indicate that they would like some things "inherited" by the -new interpreter. The mechanism could be a strict copy or it could be -copy-on-write. The motivating example is with the warnings module -(e.g. copy the filters). - -The feature isn't critical, nor would it be widely useful, so it -can wait until there's interest. Notably, both suggested solutions -will require significant work, especially when it comes to complex -objects and most especially for mutable containers of mutable -complex objects. - -Make exceptions shareable -------------------------- - -Exceptions are propagated out of ``run()`` calls, so it isn't a big -leap to make them shareable in channels. However, as noted elsewhere, -it isn't essential or (particularly common) so we can wait on doing -that. - -Make RunFailedError.__cause__ lazy ----------------------------------- - -An uncaught exception in a subinterpreter (from ``run()``) is copied -to the calling interpreter and set as ``__cause__`` on a -``RunFailedError`` which is then raised. That copying part involves -some sort of deserialization in the calling interpreter, which can be -expensive (e.g. due to imports) yet is not always necessary. - -So it may be useful to use an ``ExceptionProxy`` type to wrap the -serialized exception and only deserialize it when needed. That could -be via ``ExceptionProxy__getattribute__()`` or perhaps through -``RunFailedError.resolve()`` (which would raise the deserialized -exception and set ``RunFailedError.__cause__`` to the exception. - -It may also make sense to have ``RunFailedError.__cause__`` be a -descriptor that does the lazy deserialization (and set ``__cause__``) -on the ``RunFailedError`` instance. - -Serialize everything through channels -------------------------------------- - -We could use pickle (or marshal) to serialize everything sent through -channels. Doing this is potentially inefficient, but it may be a -matter of convenience in the end. We can add it later, but trying to -remove it later would be significantly more painful. - -Return a value from ``run()`` ------------------------------ - -Currently ``run()`` always returns None. One idea is to return the -return value from whatever the subinterpreter ran. However, for now -it doesn't make sense. The only thing folks can run is a string of -code (i.e. a script). This is equivalent to ``PyRun_StringFlags()``, -``exec()``, or a module body. None of those "return" anything. We can -revisit this once ``run()`` supports functions, etc. - -Add a "tp_share" type slot --------------------------- - -This would replace the current global registry for shareable types. - -Expose which interpreters have actually *used* a channel end. -------------------------------------------------------------- - -Currently we associate interpreters upon access to a channel. We would -keep a separate association list for "upon use" and expose that. - -Add a shareable synchronization primitive ------------------------------------------ - -This would be ``_threading.Lock`` (or something like it) where -interpreters would actually share the underlying mutex. This would -provide much better efficiency than blocking channel ops. The main -concern is that locks and channels don't mix well (as learned in Go). - -Note that the same functionality as a lock can be achieved by passing -some sort of "token" object through a channel. "send()" would be -equivalent to releasing the lock and "recv()" to acquiring the lock. - -We can add this later if it proves desirable without much trouble. - -Propagate SystemExit and KeyboardInterrupt Differently ------------------------------------------------------- - -The exception types that inherit from ``BaseException`` (aside from -``Exception``) are usually treated specially. These types are: -``KeyboardInterrupt``, ``SystemExit``, and ``GeneratorExit``. It may -make sense to treat them specially when it comes to propagation from -``run()``. Here are some options:: - - * propagate like normal via RunFailedError - * do not propagate (handle them somehow in the subinterpreter) - * propagate them directly (avoid RunFailedError) - * propagate them directly (set RunFailedError as __cause__) - -We aren't going to worry about handling them differently. Threads -already ignore ``SystemExit``, so for now we will follow that pattern. - -Add an explicit release() and close() to channel end classes ------------------------------------------------------------- - -It can be convenient to have an explicit way to close a channel against -further global use. Likewise it could be useful to have an explicit -way to release one of the channel ends relative to the current -interpreter. Among other reasons, such a mechanism is useful for -communicating overall state between interpreters without the extra -boilerplate that passing objects through a channel directly would -require. - -The challenge is getting automatic release/close right without making -it hard to understand. This is especially true when dealing with a -non-empty channel. We should be able to get by without release/close -for now. - -Add SendChannel.send_buffer() ------------------------------ - -This method would allow no-copy sending of an object through a channel -if it supports the PEP 3118 buffer protocol (e.g. memoryview). - -Support for this is not fundamental to channels and can be added on -later without much disruption. - -Auto-run in a thread --------------------- - -The PEP proposes a hard separation between subinterpreters and threads: -if you want to run in a thread you must create the thread yourself and -call ``run()`` in it. However, it might be convenient if ``run()`` -could do that for you, meaning there would be less boilerplate. - -Furthermore, we anticipate that users will want to run in a thread much -more often than not. So it would make sense to make this the default -behavior. We would add a kw-only param "threaded" (default ``True``) -to ``run()`` to allow the run-in-the-current-thread operation. - - -Rejected Ideas -============== - -Explicit channel association ----------------------------- - -Interpreters are implicitly associated with channels upon ``recv()`` and -``send()`` calls. They are de-associated with ``release()`` calls. The -alternative would be explicit methods. It would be either -``add_channel()`` and ``remove_channel()`` methods on ``Interpreter`` -objects or something similar on channel objects. - -In practice, this level of management shouldn't be necessary for users. -So adding more explicit support would only add clutter to the API. - -Use pipes instead of channels ------------------------------ - -A pipe would be a simplex FIFO between exactly two interpreters. For -most use cases this would be sufficient. It could potentially simplify -the implementation as well. However, it isn't a big step to supporting -a many-to-many simplex FIFO via channels. Also, with pipes the API -ends up being slightly more complicated, requiring naming the pipes. - -Use queues instead of channels ------------------------------- - -Queues and buffered channels are almost the same thing. The main -difference is that channels have a stronger relationship with context -(i.e. the associated interpreter). - -The name "Channel" was used instead of "Queue" to avoid confusion with -the stdlib ``queue.Queue``. - -"enumerate" ------------ - -The ``list_all()`` function provides the list of all interpreters. -In the threading module, which partly inspired the proposed API, the -function is called ``enumerate()``. The name is different here to -avoid confusing Python users that are not already familiar with the -threading API. For them "enumerate" is rather unclear, whereas -"list_all" is clear. - -Alternate solutions to prevent leaking exceptions across interpreters ---------------------------------------------------------------------- - -In function calls, uncaught exceptions propagate to the calling frame. -The same approach could be taken with ``run()``. However, this would -mean that exception objects would leak across the inter-interpreter -boundary. Likewise, the frames in the traceback would potentially leak. - -While that might not be a problem currently, it would be a problem once -interpreters get better isolation relative to memory management (which -is necessary to stop sharing the GIL between interpreters). We've -resolved the semantics of how the exceptions propagate by raising a -``RunFailedError`` instead, for which ``__cause__`` wraps a safe proxy -for the original exception and traceback. - -Rejected possible solutions: - -* reproduce the exception and traceback in the original interpreter - and raise that. -* raise a subclass of RunFailedError that proxies the original - exception and traceback. -* raise RuntimeError instead of RunFailedError -* convert at the boundary (a la ``subprocess.CalledProcessError``) - (requires a cross-interpreter representation) -* support customization via ``Interpreter.excepthook`` - (requires a cross-interpreter representation) -* wrap in a proxy at the boundary (including with support for - something like ``err.raise()`` to propagate the traceback). -* return the exception (or its proxy) from ``run()`` instead of - raising it -* return a result object (like ``subprocess`` does) [result-object]_ - (unnecessary complexity?) -* throw the exception away and expect users to deal with unhandled - exceptions explicitly in the script they pass to ``run()`` - (they can pass error info out via channels); with threads you have - to do something similar - -Always associate each new interpreter with its own thread ---------------------------------------------------------- - -As implemented in the C-API, a subinterpreter is not inherently tied to -any thread. Furthermore, it will run in any existing thread, whether -created by Python or not. You only have to activate one of its thread -states (``PyThreadState``) in the thread first. This means that the -same thread may run more than one interpreter (though obviously -not at the same time). - -The proposed module maintains this behavior. Subinterpreters are not -tied to threads. Only calls to ``Interpreter.run()`` are. However, -one of the key objectives of this PEP is to provide a more human- -centric concurrency model. With that in mind, from a conceptual -standpoint the module *might* be easier to understand if each -subinterpreter were associated with its own thread. - -That would mean ``interpreters.create()`` would create a new thread -and ``Interpreter.run()`` would only execute in that thread (and -nothing else would). The benefit is that users would not have to -wrap ``Interpreter.run()`` calls in a new ``threading.Thread``. Nor -would they be in a position to accidentally pause the current -interpreter (in the current thread) while their subinterpreter -executes. - -The idea is rejected because the benefit is small and the cost is high. -The difference from the capability in the C-API would be potentially -confusing. The implicit creation of threads is magical. The early -creation of threads is potentially wasteful. The inability to run -arbitrary interpreters in an existing thread would prevent some valid -use cases, frustrating users. Tying interpreters to threads would -require extra runtime modifications. It would also make the module's -implementation overly complicated. Finally, it might not even make -the module easier to understand. - -Only associate interpreters upon use ------------------------------------- - -Associate interpreters with channel ends only once ``recv()``, -``send()``, etc. are called. - -Doing this is potentially confusing and also can lead to unexpected -races where a channel is auto-closed before it can be used in the -original (creating) interpreter. - -Add a "reraise" method to RunFailedError ----------------------------------------- - -While having ``__cause__`` set on ``RunFailedError`` helps produce a -more useful traceback, it's less helpful when handling the original -error. To help facilitate this, we could add -``RunFailedError.reraise()``. This method would enable the following -pattern:: - - try: - try: - interp.run(script) - except RunFailedError as exc: - exc.reraise() - except MyException: - ... - -This would be made even simpler if there existed a ``__reraise__`` -protocol. - -All that said, this is completely unnecessary. Using ``__cause__`` -is good enough:: - - try: - try: - interp.run(script) - except RunFailedError as exc: - raise exc.__cause__ - except MyException: - ... - -Note that in extreme cases it may require a little extra boilerplate:: - - try: - try: - interp.run(script) - except RunFailedError as exc: - if exc.__cause__ is not None: - raise exc.__cause__ - raise # re-raise - except MyException: - ... - - -Implementation -============== - -The implementation of the PEP has 4 parts: - -* the high-level module described in this PEP (mostly a light wrapper - around a low-level C extension -* the low-level C extension module -* additions to the ("private") C=API needed by the low-level module -* secondary fixes/changes in the CPython runtime that facilitate - the low-level module (among other benefits) - -These are at various levels of completion, with more done the lower -you go: - -* the high-level module has been, at best, roughly implemented. - However, fully implementing it will be almost trivial. -* the low-level module is mostly complete. The bulk of the - implementation was merged into master in December 2018 as the - "_xxsubinterpreters" module (for the sake of testing subinterpreter - functionality). Only 3 parts of the implementation remain: - "send_wait()", "send_buffer()", and exception propagation. All three - have been mostly finished, but were blocked by work related to ceval. - That blocker is basically resolved now and finishing the low-level - will not require extensive work. -* all necessary C-API work has been finished -* all anticipated work in the runtime has been finished - -The implementation effort for PEP 554 is being tracked as part of -a larger project aimed at improving multi-core support in CPython. -[multi-core-project]_ - - -References -========== - -.. [c-api] - https://docs.python.org/3/c-api/init.html#sub-interpreter-support - -.. _Communicating Sequential Processes: - -.. [CSP] - https://en.wikipedia.org/wiki/Communicating_sequential_processes - https://github.com/futurecore/python-csp - -.. [fifo] - https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Pipe - https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue - https://docs.python.org/3/library/queue.html#module-queue - http://stackless.readthedocs.io/en/2.7-slp/library/stackless/channels.html - https://golang.org/doc/effective_go.html#sharing - http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/ - -.. [caveats] - https://docs.python.org/3/c-api/init.html#bugs-and-caveats - -.. [petr-c-ext] - https://mail.python.org/pipermail/import-sig/2016-June/001062.html - https://mail.python.org/pipermail/python-ideas/2016-April/039748.html - -.. [cryptography] - https://github.com/pyca/cryptography/issues/2299 - -.. [global-gc] - http://bugs.python.org/issue24554 - -.. [gilstate] - https://bugs.python.org/issue10915 - http://bugs.python.org/issue15751 - -.. [global-atexit] - https://bugs.python.org/issue6531 - -.. [mp-conn] - https://docs.python.org/3/library/multiprocessing.html#connection-objects - -.. [bug-rate] - https://mail.python.org/pipermail/python-ideas/2017-September/047094.html - -.. [benefits] - https://mail.python.org/pipermail/python-ideas/2017-September/047122.html - -.. [main-thread] - https://mail.python.org/pipermail/python-ideas/2017-September/047144.html - https://mail.python.org/pipermail/python-dev/2017-September/149566.html - -.. [reset_globals] - https://mail.python.org/pipermail/python-dev/2017-September/149545.html - -.. [async] - https://mail.python.org/pipermail/python-dev/2017-September/149420.html - https://mail.python.org/pipermail/python-dev/2017-September/149585.html - -.. [result-object] - https://mail.python.org/pipermail/python-dev/2017-September/149562.html - -.. [jython] - https://mail.python.org/pipermail/python-ideas/2017-May/045771.html - -.. [multi-core-project] - https://github.com/ericsnowcurrently/multi-core-python - -.. [cache-line-ping-pong] - https://mail.python.org/archives/list/python-dev@python.org/message/3HVRFWHDMWPNR367GXBILZ4JJAUQ2STZ/ - -.. [nathaniel-asyncio] - https://mail.python.org/archives/list/python-dev@python.org/message/TUEAZNZHVJGGLL4OFD32OW6JJDKM6FAS/ - -.. [extension-docs] - https://docs.python.org/3/extending/index.html - - -Copyright -========= - -This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0582.rst b/pep-0582.rst deleted file mode 100644 index 6781fabef..000000000 --- a/pep-0582.rst +++ /dev/null @@ -1,205 +0,0 @@ -PEP: 582 -Title: Python local packages directory -Version: $Revision$ -Last-Modified: $Date$ -Author: Kushal Das <mail@kushaldas.in>, Steve Dower <steve.dower@python.org>, - Donald Stufft <donald@stufft.io>, Nick Coghlan <ncoghlan@gmail.com> -Discussions-To: https://discuss.python.org/t/pep-582-python-local-packages-directory/963/ -Status: Draft -Type: Standards Track -Content-Type: text/x-rst -Created: 16-May-2018 -Python-Version: 3.8 - - -Abstract -======== - -This PEP proposes to add to Python a mechanism to automatically recognize a -``__pypackages__`` directory and prefer importing packages installed in this -location over user or global site-packages. This will avoid the steps to create, -activate or deactivate "virtual environments". Python will use the -``__pypackages__`` from the base directory of the script when present. - - - -Motivation -========== - -Python virtual environments have become an essential part of development and -teaching workflow in the community, but at the same time, they create a barrier -to entry for many. The following are a few of the issues people run into while -being introduced to Python (or programming for the first time). - -- How virtual environments work is a lot of information for anyone new. It takes - a lot of extra time and effort to explain them. - -- Different platforms and shell environments require different sets of commands - to activate the virtual environments. Any workshop or teaching environment with - people coming with different operating systems installed on their laptops create a - lot of confusion among the participants. - -- Virtual environments need to be activated on each opened terminal. If someone - creates/opens a new terminal, that by default does not get the same environment - as in a previous terminal with virtual environment activated. - - -Specification -============= - -When the Python binary is executed, it attempts to determine its prefix (as -stored in ``sys.prefix``), which is then used to find the standard library and -other key files, and by the ``site`` module to determine the location of the -``site-package`` directories. Currently the prefix is found -- assuming -``PYTHONHOME`` is not set -- by first walking up the filesystem tree looking for -a marker file (``os.py``) that signifies the presence of the standard library, -and if none is found, falling back to the build-time prefix hard coded in the -binary. The result of this process is the contents of ``sys.path`` - a list of -locations that the Python import system will search for modules. - -This PEP proposes to add a new step in this process. If a ``__pypackages__`` -directory is found in the current working directory, then it will be included in -``sys.path`` after the current working directory and just before the system -site-packages. This way, if the Python executable starts in the given project -directory, it will automatically find all the dependencies inside of -``__pypackages__``. - -In case of Python scripts, Python will try to find ``__pypackages__`` in the -same directory as the script. If found (along with the current Python version -directory inside), then it will be used, otherwise Python will behave as it does -currently. - -If any package management tool finds the same ``__pypackages__`` directory in -the current working directory, it will install any packages there and also -create it if required based on Python version. - -Projects that use a source management system can include a ``__pypackages__`` -directory (empty or with e.g. a file like ``.gitignore``). After doing a fresh -check out the source code, a tool like ``pip`` can be used to install the -required dependencies directly into this directory. - -Example -------- - -The following shows an example project directory structure, and different ways -the Python executable and any script will behave. - -:: - - foo - __pypackages__ - 3.8 - lib - bottle - myscript.py - - /> python foo/myscript.py - sys.path[0] == 'foo' - sys.path[1] == 'foo/__pypackages__/3.8/lib' - - - cd foo - - foo> /usr/bin/ansible - #! /usr/bin/env python3 - foo> python /usr/bin/ansible - - foo> python myscript.py - - foo> python - sys.path[0] == '.' - sys.path[1] == './__pypackages__/3.8/lib' - - foo> python -m bottle - -We have a project directory called ``foo`` and it has a ``__pypackages__`` -inside of it. We have ``bottle`` installed in that -``__pypackages__/3.8/lib``, and have a ``myscript.py`` file inside of the -project directory. We have used whatever tool we generally use to install ``bottle`` -in that location. - -For invoking a script, Python will try to find a ``__pypackages__`` inside of -the directory that the script resides[1]_, ``/usr/bin``. The same will happen -in case of the last example, where we are executing ``/usr/bin/ansible`` from -inside of the ``foo`` directory. In both cases, it will **not** use the -``__pypackages__`` in the current working directory. - -Similarly, if we invoke ``myscript.py`` from the first example, it will use the -``__pypackages__`` directory that was in the ``foo`` directory. - -If we go inside of the ``foo`` directory and start the Python executable (the -interpreter), it will find the ``__pypackages__`` directory inside of the -current working directory and use it in the ``sys.path``. The same happens if we -try to use the ``-m`` and use a module. In our example, ``bottle`` module will -be found inside of the ``__pypackages__`` directory. - -The above two examples are only cases where ``__pypackages__`` from current -working directory is used. - -In another example scenario, a trainer of a Python class can say "Today we are -going to learn how to use Twisted! To start, please checkout our example -project, go to that directory, and then run ``python3 -m pip install twisted``." - -That will install Twisted into a directory separate from ``python3``. There's no -need to discuss virtual environments, global versus user installs, etc. as the -install will be local by default. The trainer can then just keep telling them to -use ``python3`` without any activation step, etc. - - -.. [1]_: In the case of symlinks, it is the directory where the actual script - resides, not the symlink pointing to the script - - -Security Considerations -======================= - -While executing a Python script, it will not consider the ``__pypackages__`` in -the current directory, instead if there is a ``__pypackages__`` directory in the -same path of the script, that will be used. - -For example, if we execute ``python /usr/share/myproject/fancy.py`` from the -``/tmp`` directory and if there is a ``__pypackages__`` directory inside of -``/usr/share/myproject/`` directory, it will be used. Any potential -``__pypackages__`` directory in ``/tmp`` will be ignored. - - -Backwards Compatibility -======================= - -This does not affect any older version of Python implementation. - -Impact on other Python implementations --------------------------------------- - -Other Python implementations will need to replicate the new behavior of the -interpreter bootstrap, including locating the ``__pypackages__`` directory and -adding it the ``sys.path`` just before site packages, if it is present. - - -Reference Implementation -======================== - -`Here <https://github.com/kushaldas/cpython/tree/pypackages>`_ is a PoC -implementation (in the ``pypackages`` branch). - - -Rejected Ideas -============== - -``__pylocal__`` and ``python_modules``. - - -Copyright -========= - -This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 80 - coding: utf-8 - End: diff --git a/pep-0602-example-release-calendar.png b/pep-0602-example-release-calendar.png deleted file mode 100644 index 1db4825f6..000000000 Binary files a/pep-0602-example-release-calendar.png and /dev/null differ diff --git a/pep-0602-overlapping-support-matrix.png b/pep-0602-overlapping-support-matrix.png deleted file mode 100644 index 52392848f..000000000 Binary files a/pep-0602-overlapping-support-matrix.png and /dev/null differ diff --git a/pep-0605-example-release-calendar.png b/pep-0605-example-release-calendar.png deleted file mode 100644 index 034556346..000000000 Binary files a/pep-0605-example-release-calendar.png and /dev/null differ diff --git a/pep-0605-overlapping-support-matrix.png b/pep-0605-overlapping-support-matrix.png deleted file mode 100644 index 609193ac2..000000000 Binary files a/pep-0605-overlapping-support-matrix.png and /dev/null differ diff --git a/pep-0625.rst b/pep-0625.rst deleted file mode 100644 index 4b39fba1d..000000000 --- a/pep-0625.rst +++ /dev/null @@ -1,168 +0,0 @@ -PEP: 625 -Title: File name of a Source Distribution -Author: Tzu-ping Chung <uranusjr@gmail.com>, - Paul Moore <p.f.moore@gmail.com> -Discussions-To: https://discuss.python.org/t/draft-pep-file-name-of-a-source-distribution/4686 -Status: Draft -Type: Standards Track -Content-Type: text/x-rst -Created: 08-Jul-2020 -Post-History: 08-Jul-2020 - -Abstract -======== - -This PEP describes a standard naming scheme for a Source Distribution, also -known as an *sdist*. This scheme distinguishes an sdist from an arbitrary -archive file containing source code of Python packages, and can be used to -communicate information about the distribution to packaging tools. - -A standard sdist specified here is a gzipped tar file with a specially -formatted file stem and a ``.sdist`` suffix. This PEP does not specify the -contents of the tarball. - - -Motivation -========== - -An sdist is a Python package distribution that contains "source code" of the -Python package, and requires a build step to be turned into a wheel on -installation. This format is often considered as an unbuilt counterpart of a -:pep:`427` wheel, and given special treatments in various parts of the -packaging ecosystem. - -Compared to wheel, however, the sdist is entirely unspecified, and currently -works by convention. The widely accepted format of an sdist is defined by the -implementation of distutils and setuptools, which creates a source code -archive in a predictable format and file name scheme. Installers exploit this -predictability to assign this format certain contextual information that helps -the installation process. pip, for example, parses the file name of an sdist -from a :pep:`503` index, to obtain the distribution's project name and version -for dependency resolution purposes. But due to the lack of specification, -the installer does not have any guarantee as to the correctness of the inferred -message, and must verify it at some point by locally building the distribution -metadata. - -This build step is awkward for a certain class of operations, when the user -does not expect the build process to occur. `pypa/pip#8387`_ describes an -example. The command ``pip download --no-deps --no-binary=numpy numpy`` is -expected to only download an sdist for numpy, since we do not need to check -for dependencies, and both the name and version are available by introspecting -the downloaded file name. pip, however, cannot assume the downloaded archive -follows the convention, and must build and check the metadata. For a :pep:`518` -project, this means running the ``prepare_metadata_for_build_wheel`` hook -specified in :pep:`517`, which incurs significant overhead. - - -Rationale -========= - -By creating a special file name scheme for the sdist format, this PEP frees up -tools from the time-consuming metadata verification step when they only need -the metadata available in the file name. - -This PEP also serves as the formal specification to the long-standing -file name convention used by the current sdist implementations. The file name -contains the distribution name and version, to aid tools identifying a -distribution without needing to download, unarchive the file, and perform -costly metadata generation for introspection, if all the information they need -is available in the file name. - - -Specification -============= - -The name of an sdist should be ``{distribution}-{version}.sdist``. - -* ``distribution`` is the name of the distribution as defined in :pep:`345`, - and normalised according to :pep:`503`, e.g. ``'pip'``, ``'flit-core'``. -* ``version`` is the version of the distribution as defined in :pep:`440`, - e.g. ``20.2``. - -Each component is escaped according to the same rules as :pep:`427`. - -An sdist must be a gzipped tar archive that is able to be extracted by the -standard library ``tarfile`` module with the open flag ``'r:gz'``. - - -Backwards Compatibility -======================= - -The new file name scheme should not incur backwards incompatibility in -existing tools. Installers are likely to have already implemented logic to -exclude extensions they do not understand, since they already need to deal -with legacy formats on PyPI such as ``.rpm`` and ``.egg``. They should be able -to correctly ignore files with extension ``.sdist``. - -pip, for example, skips this extension with the following debug message:: - - Skipping link: unsupported archive format: sdist: <URL to file> - -While setuptools ignores it silently. - - -Rejected Ideas -============== - -Create specification for sdist metadata ---------------------------------------- - -The topic of creating a trustworthy, standard sdist metadata format as a means -to distinguish sdists from arbitrary archive files has been raised and -discussed multiple times, but has yet to make significant progress due to -the complexity of potential metadata inconsistency between an sdist and a -wheel built from it. - -This PEP does not exclude the possibility of creating a metadata specification -for sdists in the future. But by specifying only the file name of an sdist, a -tool can reliably identify an sdist, and perform useful introspection on its -identity, without going into the details required for metadata specification. - -Use a currently common sdist naming scheme ------------------------------------------- - -There is a currently established practice to name an sdist in the format of -``{distribution}-{version}.[tar.gz|zip]``. - -Popular source code management services use a similar scheme to name the -downloaded source archive. GitHub, for example, uses ``distribution-1.0.zip`` -as the archive name containing source code of repository ``distribution`` on -branch ``1.0``. Giving this scheme a special meaning would cause confusion -since a source archive may not a valid sdist. - -Augment a currently common sdist naming scheme ----------------------------------------------- - -A scheme ``{distribution}-{version}.sdist.tar.gz`` was raised during the -initial discussion. This was abandoned due to backwards compatibility issues -with currently available installation tools. pip 20.1, for example, would -parse ``distribution-1.0.sdist.tar.gz`` as project ``distribution`` with -version ``1.0.sdist``. This would cause the sdist to be downloaded, but fail to -install due to inconsistent metadata. - -The same problem exists for all common archive suffixes. To avoid confusing -old installers, the sdist scheme must use a suffix that they do not identify -as an archive. - - -References -========== - -.. _`pypa/pip#8387`: https://github.com/pypa/pip/issues/8387 - - -Copyright -========= - -This document is placed in the public domain or under the CC0-1.0-Universal -license, whichever is more permissive. - - - .. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0639.rst b/pep-0639.rst deleted file mode 100644 index 48cfeb7c6..000000000 --- a/pep-0639.rst +++ /dev/null @@ -1,904 +0,0 @@ -PEP: 639 -Title: Metadata for Python Software Packages 2.2 -Version: $Revision$ -Last-Modified: $Date$ -Author: Philippe Ombredanne <pombredanne at nexb.com> -Sponsor: Paul Moore <p.f.moore at gmail.com> -BDFL-Delegate: Paul Moore <p.f.moore at gmail.com> -Discussions-To: https://discuss.python.org/t/2154 -Status: Draft -Type: Standards Track -Content-Type: text/x-rst -Created: 15-Aug-2019 -Python-Version: 3.x -Post-History: -Replaces: 566 -Resolution: - - -Abstract -======== - -This PEP describes the changes between versions 2.1 and 2.2 of the `Core -Metadata Specification` [#cms]_ for Python packages. Version 2.1 is specified in -PEP 566. - -The primary change introduced in this PEP updates how licenses are documented in -core metadata via the ``License`` field with license expression strings using -SPDX license identifiers [#spdxlist]_ such that license documentation is simpler -and less ambiguous: - -- for package authors to create, -- for package users to read and understand, and, -- for tools to process package license information mechanically. - -The other changes include: - -- specifying a ``License-File`` field which is already used by ``wheel`` and - ``setuptools`` to include license files in built distributions. -- defining how tools can validate license expressions and report warnings to - users for invalid expressions (but still accept any string as ``License``). - - -Goals -===== - -This PEP's scope is limited strictly to how we document the license of a -distribution: - -- with an improved and structured way to document a license expression, and, -- by including license texts in a built package. - -The core metadata specification updates that are part of this PEP have been -designed to have minimal impact and to be backward compatible with v2.1. These -changes utilize emerging new ways to document licenses that are already in use -in some tools (e.g. by adding the ``License-File`` field already used in -``wheel`` and ``setuptools``) or by some package authors (e.g. storing an SPDX -license expression in the existing ``License`` field). - -In addition to an update to the metadata specification, this PEP contains: - -- recommendations for publishing tools on how to validate the ``License`` and - ``Classifier`` fields and report informational warnings when a package uses an - older, non-structured style of license documentation conventions. - -- informational appendices that contain surveys of how we document licenses - today in Python packages and elsewhere, and a reference Python library to - parse, validate and build correct license expressions. - -It is the intent of the PEP authors to work closely with tool authors to -implement the recommendations for validation and warnings specified in this PEP. - - -Non-Goals -========= - -This PEP is neutral regarding the choice of license by any package author. - -In particular, the SPDX license expression syntax proposed in this PEP provides -simpler and more expressive conventions to document accurately any kind of -license that applies to a Python package, whether it is an open source license, -a free or libre software license, a proprietary license, or a combination of -such licenses. - -This PEP makes no recommendation for specific licenses and does not require the -use of specific license documentation conventions. This PEP also does not impose -any restrictions when uploading to PyPI. - -Instead, this PEP is intended to document common practices already in use, and -recommends that publishing tools should encourage users via informational -warnings when they do not follow this PEP's recommendations. - -This PEP is not about license documentation in files inside packages, even -though this is a surveyed topic in the appendix. - - -Possible future PEPs --------------------- - -It is the intention of the authors of this PEP to consider the submission of -related but separate PEPs in the future such as: - -- make ``License`` and new ``License-File`` fields mandatory including - stricter enforcement in tools and PyPI publishing. - -- require uploads to PyPI to use only FOSS (Free and Open Source software) - licenses. - - -Motivation -========== - -Software is licensed, and providing accurate licensing information to Python -packages users is an important matter. Today, there are multiple places where -licenses are documented in package metadata and there are limitations to what -can be documented. This is often leading to confusion or a lack of clarity both -for package authors and package users. - -Several package authors have expressed difficulty and/or frustrations due to the -limited capabilities to express licensing in package metadata. This also applies -to Linux and BSD* distribution packagers. This has triggered several -license-related discussions and issues, in particular: - -- https://github.com/pypa/trove-classifiers/issues/17 -- https://github.com/pypa/interoperability-peps/issues/46 -- https://github.com/pypa/packaging-problems/issues/41 -- https://github.com/pypa/wheel/issues/138 -- https://github.com/pombredanne/spdx-pypi-pep/issues/1 - -On average, Python packages tend to have more ambiguous, or missing, license -information than other common application package formats (such as npm, Maven or -Gem) as can be seen in the statistics [#cdstats]_ page of the ClearlyDefined -[#cd]_ project that cover all packages from PyPI, Maven, npm and Rubygems. -ClearlyDefined is an open source project to help improve clarity of other open -source projects that is incubating at the OSI (Open Source Initiative) [#osi]_. - - -Rationale -========= - -A mini-survey of existing license metadata definitions in use in the Python -ecosystem today and documented in several other system/distro and application -package formats is provided in Appendix 2 of this PEP. - -There are a few takeaways from the survey: - -- Most package formats use a single ``License`` field. - -- Many modern package formats use some form of license expression syntax to - optionally combine more than one license identifier together. SPDX and - SPDX-like syntaxes are the most popular in use. - -- SPDX license identifiers are becoming a de facto way to reference common - licenses everywhere, whether or not a license expression syntax is used. - -- Several package formats support documenting both a license expression and the - paths of the corresponding files that contain the license text. Most free and - open source software licenses require package authors to include their full - text in a distribution. - -These considerations have guided the design and recommendations of this PEP. - -The reuse of the ``License`` field with license expressions will provide an -intuitive and more structured way to express the license of a distribution using -a well-defined syntax and well-known license identifiers. - -Over time, recommending the usage of these expressions will help Python package -publishers improve the clarity of their license documentation to the benefit of -package authors, consumers and redistributors. - - -Core Metadata Specification updates -=================================== - -The canonical source for the names and semantics of each of the supported -metadata fields is the Core Metadata Specification [#cms]_ document. - -The details of the updates considered to the Core Metadata Specification [#cms]_ -document as part of this PEP are described here and will be added to the -canonical source once this PEP is approved. - - -Added in Version 2.2 --------------------- - -License-File (multiple use) -::::::::::::::::::::::::::: - -The License-File is a string that is a path, relative to``.dist-info``, to a -license file. The license file content MUST be UTF-8 encoded text. - -Build tools SHOULD honor this field and include the corresponding license -file(s) in the built package. - - -Changed in Version 2.2 ----------------------- - -License (optional) -:::::::::::::::::: - -Text indicating the license covering the distribution. This text can be either a -valid license expression as defined here or any free text. - -Publishing tools SHOULD issue an informational warning if this field is empty, -missing, or is not a valid license expression as defined here. Build tools MAY -issue a similar warning. - - -License Expression syntax -''''''''''''''''''''''''' - -A license expression is a string using the SPDX license expression syntax as -documented in the SPDX specification [#spdx]_ using either Version 2.2 -[#spdx22]_ or a later compatible version. SPDX is a working group at the Linux -Foundation that defines a standard way to exchange package information. - -When used in the ``License`` field and as a specialization of the SPDX license -expression definition, a license expression can use the following license -identifiers: - -- any SPDX-listed license short-form identifiers that are published in the SPDX - License List [#spdxlist]_ using either Version 3.10 or any later compatible - version. Note that the SPDX working group never removes any license - identifiers: instead they may choose to mark an identifier as "deprecated". - -- the ``LicenseRef-Public-Domain`` and ``LicenseRef-Proprietary`` strings to - identify licenses that are not included in the SPDX license list. - -When processing the ``License`` field to determine if it contains a valid -license expression, tools: - -- SHOULD report an informational warning if one or more of the following - applies: - - - the field does not contain a license expression, - - - the license expression syntax is invalid, - - - the license expression syntax is valid but some license identifiers are - unknown as defined here or the license identifiers have been marked as - deprecated in the SPDX License List [#spdxlist]_ - -- SHOULD store a case-normalized version of the ``License`` field using the - reference case for each SPDX license identifier and uppercase for the AND, OR - and WITH keywords. - -- SHOULD report an informational warning if normalization process results in - changes to the ``License`` field contents. - -License expression examples:: - - License: MIT - - License: BSD-3-Clause - - License: MIT OR GPL-2.0-or-later OR (FSFUL AND BSD-2-Clause) - - License: GPL-3.0-only WITH Classpath-Exception-2.0 OR BSD-3-Clause - - License: This software may only be obtained by sending the - author a postcard, and then the user promises not - to redistribute it. - - License: LicenseRef-Proprietary AND LicenseRef-Public-Domain - - -Classifier (multiple use) -::::::::::::::::::::::::: - -Each entry is a string giving a single classification value for the -distribution. Classifiers are described in PEP 301. - -Examples:: - - Classifier: Development Status :: 4 - Beta - Classifier: Environment :: Console (Text Based) - -Tools SHOULD issue an informational warning if this field contains a licensing- -related classifier string starting with the ``License ::`` prefix and SHOULD -suggest the use of a license expression in the ``License`` field instead. - -If the ``License`` field is present and contains a valid license expression, -publishing tools MUST NOT also provide any licensing-related classifier entries -[#classif]_. - -However, for compatibility with existing publishing and installation processes, -licensing-related classifier entries SHOULD continue to be accepted if the -``License`` field is absent or does not contain a valid license expression. - -Publishing tools MAY infer a license expression from the provided classifier -entries if they are able to do so unambiguously. - -However, no new licensing related classifiers will be added; anyone -requesting them will be directed to use a license expression in the ``License`` -field instead. Note that the licensing-related classifiers may be deprecated in -a future PEP. - - -Mapping Legacy Classifiers to New License Expressions -''''''''''''''''''''''''''''''''''''''''''''''''''''' - -Publishing tools MAY infer or suggest an equivalent license expression from the -provided ``License`` or ``Classifier`` information if they are able to do so -unambiguously. For instance, if a package only has this license classifier:: - - Classifier: License :: OSI Approved :: MIT License - -Then the corresponding value for a ``License`` field using a valid license -expression to suggest would be:: - - License: MIT - - -Here are mapping guidelines for the legacy classifiers: - -- Classifier ``License :: Other/Proprietary License`` becomes License: - ``LicenseRef-Proprietary`` expression. - -- Classifier ``License :: Public Domain`` becomes License: ``LicenseRef-Public-Domain`` - expression, though tools should encourage the use of more explicit and legally - portable license identifiers such as ``CC0-1.0`` [#cc0]_, the ``Unlicense`` - [#unlic]_ since the meaning associated with the term "public domain" is thoroughly - dependent on the specific legal jurisdiction involved and some jurisdictions - have no concept of Public Domain as it exists in the USA. - -- The generic and ambiguous classifiers ``License :: OSI Approved`` and - ``License :: DFSG approved`` do not have an equivalent license expression. - -- The generic and sometimes ambiguous classifiers - ``License :: Free For Educational Use``, ``License :: Free For Home Use``, - ``License :: Free for non-commercial use``, ``License :: Freely Distributable``, - ``License :: Free To Use But Restricted``, and ``License :: Freeware`` are mapped - to the generic License: ``LicenseRef-Proprietary`` expression. - -- Classifiers ``License :: GUST*`` have no mapping to SPDX license identifierss - for now and no package uses them in PyPI as of the writing of this PEP. - -The remainder of the classifiers using a ``License ::`` prefix map to a simple -single-identifier license expression using the corresponding SPDX license identifiers. - -When multiple license-related classifiers are used, their relation is ambiguous -and it is typically not possible to determine if all the licenses apply or if -there is a choice that is possible among the licenses. In this case, tools -cannot reliably infer a license expression and should suggest that the package -author construct a license expression which expresses their intent. - - -Summary of Differences From PEP 566 -=================================== - -* Metadata-Version is now 2.2. -* Added one new field: ``License-File`` -* Updated the documentation of two fields: ``License`` and ``Classifier`` - - -Backwards Compatibility -======================= - -The reuse of the ``License`` field means that we keep backward -compatibility. The specification of the ``License-File`` field is only writing -down the practices of the ``wheel`` and ``setuptools`` tools and is backward -compatible with their support for that field. - -The "soft" validation of the ``License`` field when it does not contain a valid -license expression and when the ``Classifier`` field is used with legacy -license-related classifiers means that we can gently prepare users for possible -strict and incompatible validation of these fields in the future. - - -Security Implications -===================== - -This PEP has no foreseen security implications: the License field is a plain -string and the License-File(s) are file paths. None of them introduces any new -security concern. - - -How to Teach Users to Use License Expressions -============================================= - -The simple cases are simple: a single license identifier is a valid license -expression and a large majority of packages use a single license. - -The plan to teach users of packaging tools how to express their package's -license with a valid license expression is to have tools issue informative -messages when they detect invalid license expressions or when a license-related -classifier is used in the ``Classifier`` field. - -With a warning message that does not terminate processing, publishing tools will -gently teach users how to provide correct license expressions over time. - -Tools may also help with the conversion and suggest a license expression in some -cases: - -1. The section `Mapping Legacy Classifiers to New License expressions`_ provides - tool authors with guidelines on how to suggest a license expression produced - from legacy classifiers. - -2. Tools may also be able to infer and suggest how to update an existing - incorrect ``License`` value and convert that to a correct license expression. - For instance a tool may suggest to correct a ``License`` field from - ``Apache2`` (which is not a valid license expression as defined in this PEP) - to ``Apache-2.0`` (which is a valid license expression using an SPDX license - identifier as defined in this PEP). - - -Reference Implementation -======================== - -Tools will need to support parsing and validating license expressions in the -``License`` field. - -The ``license-expression`` library [#licexp]_ is a reference Python -implementation of a library that handles license expressions including parsing, -validating and formatting license expressions using flexible lists of license -symbols (including SPDX license identifiers and any extra identifiers referenced -here). It is licensed under the Apache-2.0 license and is used in a few projects -such as the SPDX Python tools [#spdxpy]_, the ScanCode toolkit [#scancodetk]_ -and the Free Software Foundation Europe (FSFE) Reuse project [#reuse]_. - - -Rejected ideas -============== - -1. Use a new ``License-Expression`` field and deprecate the ``License`` field. - -Adding a new field would introduce backward incompatible changes when the -``License`` field would be retired later and require having more complex -validation. The use of such a field would further introduce a new concept that -is not seen anywhere else in any other package metadata (e.g. a new field only -for license expression) and possibly be a source of confusion. Also, users are -less likely to start using a new field than make small adjustments to their use -of existing fields. - - -2. Mapping licenses used in the license expression to specific files in the - license files (or vice versa). - -This would require using a mapping (two parallel lists would be too prone to -alignment errors) and a mapping would bring extra complication to how license -are documented by adding an additional nesting level. - -A mapping would be needed as you cannot guarantee that all expressions (e.g. -GPL with an exception may be in a single file) or all the license keys have a -single license file and that any expression does not have more than one. (e.g. -an Apache license ``LICENSE`` and its ``NOTICE`` file for instance are two -distinct files). Yet in most cases, there is a simpler "one license", "one or -more license files". In the rarer and more complex cases where there are many -licenses involved you can still use the proposed conventions at the cost of a -slight loss of clarity by not specifying which text file is for which license -identifier, but you are not forcing the more complex data model (e.g. a mapping) -on everyone that may not need it. - -We could of course have a data field with multiple possible value types (it’s a -string, it’s a list, it’s a mapping!) but this could be a source of confusion. -This is what has been done for instance in npm (historically) and in Rubygems -(still today) and as result you need to test the type of the metadata field -before using it in code and users are confused about when to use a list or a -string. - - -3. Mapping licenses to specific source files and/or directories of source files - (or vice versa). - -File-level notices are not considered as part of the scope of this PEP and the -existing ``SPDX-License-Identifier`` [#spdxids]_ convention can be used and -may not need further specification as a PEP. - - -Appendix 1. License Expression example -====================================== - -The current version of ``setuptools`` metadata [#setuptools5030]_ does not use -the ``License`` field. It uses instead this license-related information in -``setup.cfg``:: - - license_file = LICENSE - classifiers = - License :: OSI Approved :: MIT License - -The simplest migration to this PEP would consist of using this instead:: - - license = MIT - license_files = - LICENSE - -Another possibility would be to include the licenses of the third-party packages -that are vendored in the ``setuptools/_vendor/`` and ``pkg_resources/_vendor`` -directories:: - - appdirs==1.4.3 - packaging==20.4 - pyparsing==2.2.1 - ordered-set==3.1.1 - -These license expressions for these packages are:: - - appdirs: MIT - packaging: Apache-2.0 OR BSD-2-Clause - pyparsing: MIT - ordered-set: MIT - -Therefore, a comprehensive license expression covering both ``setuptools`` proper -and its vendored packages could contain these metadata, combining all the -license expressions in one expression:: - - license = MIT AND (Apache-2.0 OR BSD-2-Clause) - license_files = - LICENSE.MIT - LICENSE.packaging - -Here we would assume that the ``LICENSE.MIT`` file contains the text of the MIT -license and the copyrights used by ``setuptools``, ``appdirs``, ``pyparsing`` and -``ordered-set``, and that the ``LICENSE.packaging`` file contains the texts of the -Apache and BSD license, its copyrights and its license choice notice [#packlic]_. - - -Appendix 2. Surveying how we document licenses today in Python -============================================================== - -There are multiple ways used or recommended to document Python package -licenses today: - - -In Core metadata ----------------- - -There are two overlapping core metadata fields to document a license: the -license-related ``Classifier`` strings [#classif]_ prefixed with ``License ::`` and -the ``License`` field as free text [#licfield]_. - - -The core metadata documentation ``License`` field documentation is currently:: - - License (optional) - :::::::::::::::::: - - Text indicating the license covering the distribution where the license - is not a selection from the "License" Trove classifiers. See - "Classifier" below. This field may also be used to specify a - particular version of a license which is named via the ``Classifier`` - field, or to indicate a variation or exception to such a license. - - Examples:: - - License: This software may only be obtained by sending the - author a postcard, and then the user promises not - to redistribute it. - - License: GPL version 3, excluding DRM provisions - -Even though there are two fields, it is at times difficult to convey anything -but simpler licensing. For instance some classifiers lack accuracy (GPL -without a version) and when you have multiple License-related classifiers it is -not clear if this is a choice or all these apply and which ones. Furthermore, -the list of available license-related classifiers is often out-of-date. - - -In the PyPA ``sampleproject`` ------------------------------ - -The latest PyPA ``sampleproject`` recommends only to use classifiers in -``setup.py`` and does not list the ``license`` field in its example -``setup.py`` [#samplesetup]_. - - -The License Files in wheels and setuptools ------------------------------------------- - -Beyond a license code or qualifier, license text files are documented and -included in a built package either implicitly or explicitly and this is another -possible source of confusion: - -- In wheels [#wheels]_ license files are automatically added to the ``.dist-info`` - directory if they match one of a few common license file name patterns (such - as LICENSE*, COPYING*). Alternatively a package author can specify a list of - license file paths to include in the built wheel using in the - ``license_files`` field in the ``[metadata]`` section of the project's - ``setup.cfg``. Previously this was a (singular) ``license_file`` file attribute - that is now deprecated but is still in common use. See [#pipsetup]_ for - instance. - -- In ``setuptools`` [#setuptoolssdist]_, a ``license_file`` attribute is used to add - a single license file to a source distribution. This singular version is - still honored by ``wheels`` for backward compatibility. - -- Using a LICENSE.txt file is encouraged in the packaging guide [#packaging]_ - paired with a ``MANIFEST.in`` entry to ensure that the license file is included - in a built source distribution (sdist). - -Note: the License-File field proposed in this PEP already exists in ``wheel`` and -``setuptools`` with the same behaviour as explained above. This PEP is only -recognizing and documenting the existing practice as used in ``wheel`` (with the -``license_file`` and ``license_files`` ``setup.cfg`` ``[metadata]`` entries) and in -``setuptools`` ``license_file`` ``setup()`` argument. - - -In Python code files --------------------- - -(Note: Documenting licenses in source code is not in the scope of this PEP) - -Beside using comments and/or ``SPDX-License-Identifier`` conventions, the license -is sometimes documented in Python code files using "dunder" variables typically -named after one of the lower cased Core Metadata fields such as ``__license__`` -[#pycode]_. - -This convention (dunder global variables) is recognized by the built-in ``help()`` -function and the standard ``pydoc`` module. The dunder variable(s) will show up in -the ``help()`` DATA section for a module. - - -In some other Python packaging tools ------------------------------------- - -- Conda package manifest [#conda]_ has support for ``license`` and ``license_file`` - fields as well as a ``license_family`` license grouping field. - -- ``flit`` [#flit]_ recommends to use classifiers instead of License (as per the - current metadata spec). - -- ``pbr`` [#pbr]_ uses similar data as setuptools but always stored setup.cfg. - -- ``poetry`` [#poetry]_ specifies the use of the ``license`` field in - ``pyproject.toml`` with SPDX license identifiers. - - -Appendix 3. Surveying how other package formats document licenses -================================================================= - -Here is a survey of how things are done elsewhere. - -License in Linux distribution packages ---------------------------------------- - -Note: in most cases the license texts of the most common licenses are included -globally once in a shared documentation directory (e.g. /usr/share/doc). - -- Debian document package licenses with machine readable copyright files - [#dep5]_. This specification defines its own license expression syntax that is - very similar to the SDPX syntax and use its own list of license identifiers - for common licenses (also closely related to SPDX identifiers). - -- Fedora packages [#fedora]_ specify how to include ``License Texts`` - [#fedoratext]_ and how use a ``License`` field [#fedoralic]_ that must be filled - with an appropriate license Short License identifier(s) from an extensive list - of "Good Licenses" identifiers [#fedoralist]_. Fedora also defines its own - license expression syntax very similar to the SDPX syntax. - -- openSUSE packages [#opensuse]_ use SPDX license expressions with - SPDX license identifiers and a list of extra license identifiers - [#opensuselist]_. - -- Gentoo ebuild uses a ``LICENSE`` variable [#gentoo]_. This field is specified - in GLEP-0023 [#glep23]_ and in the Gentoo development manual [#gentoodev]_. - Gentoo also defines a license expression syntax and a list of allowed - licenses. The expression syntax is rather different from SPDX. - -- FreeBSD package Makefile [#freebsd]_ provides ``LICENSE`` and - ``LICENSE_FILE`` fields with a list of custom license symbols. For - non-standard licenses, FreeBSD recommend to use ``LICENSE=UNKNOWN`` and add - ``LICENSE_NAME`` and ``LICENSE_TEXT`` fields, as well as sophisticated - ``LICENSE_PERMS`` to qualify the license permissions and ``LICENSE_GROUPS`` - to document a license grouping. The ``LICENSE_COMB`` allows to document more - than one license and how they apply together, forming a custom license - expression syntax. FreeBSD also recommends the use of - ``SPDX-License-Identifier`` in source code files. - -- Archlinux PKGBUILD [#archinux]_ define its own license identifiers - [#archlinuxlist]_. The value ``'unknown'`` can be used if the license is not - defined. - -- OpenWRT ipk packages [#openwrt]_ use the ``PKG_LICENSE`` and - ``PKG_LICENSE_FILES`` variables and recommend the use of SPDX License - identifiers. - -- NixOS uses SPDX identifiers [#nixos]_ and some extra license identifiers in - its license field. - -- GNU Guix (based on NixOS) has a single License field, uses its own license - symbols list [#guix]_ and specifies to use one license or a list of licenses - [#guixlic]_. - -- Alpine Linux packages [#alpine]_ recommend using SPDX identifiers in the - license field. - - -License in Language and Application packages --------------------------------------------- - -- In Java, Maven POM [#maven]_ defines a ``licenses`` XML tag with a list of license - items each with a name, URL, comments and "distribution" type. This is not - mandatory and the content of each field is not specified. - -- JavaScript npm package.json [#npm]_ use a single license field with SPDX - license expression or the ``UNLICENSED`` id if no license is specified. - A license file can be referenced as an alternative using "SEE LICENSE IN - <filename>" in the single ``license`` field. - -- Rubygems gemspec [#gem]_ specifies either a singular license string or a list - of license strings. The relationship between multiple licenses in a list is - not specified. They recommend using SPDX license identifiers. - -- CPAN Perl modules [#perl]_ use a single license field which is either a single - string or a list of strings. The relationship between the licenses in a list - is not specified. There is a list of custom license identifiers plus - these generic identifiers: ``open_source``, ``restricted``, ``unrestricted``, - ``unknown``. - -- Rust Cargo [#cargo]_ specifies the use of an SPDX license expression (v2.1) in - the ``license`` field. It also supports an alternative expression syntax using - slash-separated SPDX license identifiers. There is also a ``license_file`` - field. The crates.io package registry [#cratesio]_ requires that either - ``license`` or ``license_file`` fields are set when you upload a package. - -- PHP Composer composer.json [#composer]_ uses a ``license`` field with an SPDX - license id or "proprietary". The ``license`` field is either a single string - that can use something which resembles the SPDX license expression syntax with - "and" and "or" keywords; or is a list of strings if there is a choice of - licenses (aka. a "disjunctive" choice of license). - -- NuGet packages [#nuget]_ were using only a simple license URL and are now - specifying to use an SPDX License expression and/or the path to a license - file within the package. The NuGet.org repository states that they only - accepts license expressions that are `approved by the Open Source Initiative - or the Free Software Foundation.` - -- Go language modules ``go.mod`` have no provision for any metadata beyond - dependencies. Licensing information is left for code authors and other - community package managers to document. - -- Dart/Flutter spec [#flutter]_ recommends to use a single ``LICENSE`` file - that should contain all the license texts each separated by a line with 80 - hyphens. - -- JavaScript Bower [#bower]_ ``license`` field is either a single string or a list - of strings using either SPDX license identifiers, or a path or a URL to a - license file. - -- Cocoapods podspec [#cocoapod]_ ``license`` field is either a single string or a - mapping with attributes of type, file and text keys. This is mandatory unless - there is a LICENSE or LICENCE file provided. - -- Haskell Cabal [#cabal]_ accepts an SPDX license expression since version 2.2. - The version of the SPDX license list used is a function of the ``cabal`` version. - The specification also provides a mapping between pre-SPDX Legacy license - Identifiers and SPDX identifiers. Cabal also specifies a ``license-file(s)`` - field that lists license files that will be installed with the package. - -- Erlang/Elixir mix/hex package [#mix]_ specifies a ``licenses`` field as a - required list of license strings and recommends to use SPDX license - identifiers. - -- D lang dub package [#dub]_ defines its own list of license identifiers and - its own license expression syntax and both are similar to the SPDX conventions. - -- R Package DESCRIPTION [#cran]_ defines its own sophisticated license - expression syntax and list of licenses identifiers. R has a unique way to - support specifiers for license versions such as ``LGPL (>= 2.0, < 3)`` in its - license expression syntax. - - -Conventions used by other ecosystems ------------------------------------- - -- ``SPDX-License-Identifier`` [#spdxids]_ is a simple convention to document the - license inside a file. - -- The Free Software Foundation (FSF) promotes the use of SPDX license identifiers - for clarity in the GPL and other versioned free software licenses [#gnu]_ - [#fsf]_. - -- The Free Software Foundation Europe (FSFE) REUSE project [#reuse]_ promotes - using ``SPDX-License-Identifier``. - -- The Linux kernel uses ``SPDX-License-Identifier`` and parts of the FSFE REUSE - conventions to document its licenses [#linux]_. - -- U-Boot spearheaded using ``SPDX-License-Identifier`` in code and now follows the - Linux ways [#uboot]_. - -- The Apache Software Foundation projects use RDF DOAP [#apache]_ with a single - license field pointing to SPDX license identifiers. - -- The Eclipse Foundation promotes using ``SPDX-license-Identifiers`` [#eclipse]_ - -- The ClearlyDefined project [#cd]_ promotes using SPDX license identifiers and - expressions to improve license clarity. - -- The Android Open Source Project [#android]_ use ``MODULE_LICENSE_XXX`` empty - tag files where ``XXX`` is a license code such as BSD, APACHE, GPL, etc. And - side by side with this ``MODULE_LICENSE`` file there is a ``NOTICE`` file - that contains license and notices texts. - - -References -========== - -This document specifies version 2.2 of the metadata format. - -- Version 1.0 is specified in PEP 241. -- Version 1.1 is specified in PEP 314. -- Version 1.2 is specified in PEP 345. -- Version 2.0, while not formally accepted, was specified in PEP 426. -- Version 2.1 is specified in PEP 566. - -.. [#cms] https://packaging.python.org/specifications/core-metadata -.. [#cdstats] https://clearlydefined.io/stats -.. [#cd] https://clearlydefined.io -.. [#osi] http://opensource.org -.. [#classif] https://pypi.org/classifiers -.. [#spdxlist] https://spdx.org/licenses -.. [#spdx] https://spdx.org -.. [#spdx22] https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/ -.. [#wheels] https://github.com/pypa/wheel/blob/b8b21a5720df98703716d3cd981d8886393228fa/docs/user_guide.rst#including-license-files-in-the-generated-wheel-file -.. [#reuse] https://reuse.software/ -.. [#licexp] https://github.com/nexB/license-expression/ -.. [#spdxpy] https://github.com/spdx/tools-python/ -.. [#scancodetk] https://github.com/nexB/scancode-toolkit -.. [#licfield] https://packaging.python.org/guides/distributing-packages-using-setuptools/?highlight=MANIFEST.in#license -.. [#samplesetup] https://github.com/pypa/sampleproject/blob/52966defd6a61e97295b0bb82cd3474ac3e11c7a/setup.py#L98 -.. [#pipsetup] https://github.com/pypa/pip/blob/476606425a08c66b9c9d326994ff5cf3f770926a/setup.cfg#L40 -.. [#setuptoolssdist] https://github.com/pypa/setuptools/blob/97e8ad4f5ff7793729e9c8be38e0901e3ad8d09e/setuptools/command/sdist.py#L202 -.. [#packaging] https://packaging.python.org/guides/distributing-packages-using-setuptools/?highlight=MANIFEST.in#license-txt -.. [#pycode] https://github.com/search?l=Python&q=%22__license__%22&type=Code -.. [#setuptools5030] https://github.com/pypa/setuptools/blob/v50.3.0/setup.cfg#L17 -.. [#packlic] https://github.com/pypa/packaging/blob/19.1/LICENSE -.. [#conda] https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#about-section -.. [#flit] https://github.com/takluyver/flit -.. [#poetry] https://poetry.eustace.io/docs/pyproject/#license -.. [#pbr] https://docs.openstack.org/pbr/latest/user/features.html -.. [#dep5] https://dep-team.pages.debian.net/deps/dep5/ -.. [#fedora] https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/ -.. [#fedoratext] https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_license_text -.. [#fedoralic] https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_valid_license_short_names -.. [#fedoralist] https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#Good_Licenses -.. [#opensuse] https://en.opensuse.org/openSUSE:Packaging_guidelines#Licensing -.. [#opensuselist] https://docs.google.com/spreadsheets/d/14AdaJ6cmU0kvQ4ulq9pWpjdZL5tkR03exRSYJmPGdfs/pub -.. [#gentoo] https://devmanual.gentoo.org/ebuild-writing/variables/index.html#license -.. [#glep23] https://www.gentoo.org/glep/glep-0023.html -.. [#gentoodev] https://devmanual.gentoo.org/general-concepts/licenses/index.html -.. [#freebsd] https://www.freebsd.org/doc/en_US.ISO8859-1/books/porters-handbook/licenses.html -.. [#archinux] https://wiki.archlinux.org/index.php/PKGBUILD#license -.. [#archlinuxlist] https://wiki.archlinux.org/index.php/PKGBUILD#license -.. [#openwrt] https://openwrt.org/docs/guide-developer/packages#buildpackage_variables -.. [#nixos] https://github.com/NixOS/nixpkgs/blob/master/lib/licenses.nix -.. [#guix] http://git.savannah.gnu.org/cgit/guix.git/tree/guix/licenses.scm -.. [#guixlic] https://guix.gnu.org/manual/en/html_node/package-Reference.html#index-license_002c-of-packages -.. [#alpine] https://wiki.alpinelinux.org/wiki/Creating_an_Alpine_package#license -.. [#maven] https://maven.apache.org/pom.html#Licenses -.. [#npm] https://docs.npmjs.com/files/package.json#license -.. [#gem] https://guides.rubygems.org/specification-reference/#license= -.. [#perl] https://metacpan.org/pod/CPAN::Meta::Spec#license -.. [#cargo] https://doc.rust-lang.org/cargo/reference/manifest.html#package-metadata -.. [#cratesio] https://doc.rust-lang.org/cargo/reference/registries.html#publish -.. [#composer] https://getcomposer.org/doc/04-schema.md#license -.. [#nuget] https://docs.microsoft.com/en-us/nuget/reference/nuspec#licenseurl -.. [#flutter] https://flutter.dev/docs/development/packages-and-plugins/developing-packages#adding-licenses-to-the-license-file -.. [#bower] https://github.com/bower/spec/blob/master/json.md#license -.. [#cocoapod] https://guides.cocoapods.org/syntax/podspec.html#license -.. [#cabal] https://cabal.readthedocs.io/en/latest/developing-packages.html#pkg-field-license -.. [#mix] https://hex.pm/docs/publish -.. [#dub] https://dub.pm/package-format-json.html#licenses -.. [#cran] https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Licensing -.. [#spdxids] https://spdx.org/using-spdx-license-identifier -.. [#gnu] https://www.gnu.org/licenses/identify-licenses-clearly.html -.. [#fsf] https://www.fsf.org/blogs/rms/rms-article-for-claritys-sake-please-dont-say-licensed-under-gnu-gpl-2 -.. [#linux] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/process/license-rules.rst -.. [#uboot] https://www.denx.de/wiki/U-Boot/Licensing -.. [#apache] https://svn.apache.org/repos/asf/allura/doap_Allura.rdf -.. [#eclipse] https://www.eclipse.org/legal/epl-2.0/faq.php -.. [#android] https://github.com/aosp-mirror/platform_external_tcpdump/blob/master/MODULE_LICENSE_BSD -.. [#cc0] https://creativecommons.org/publicdomain/zero/1.0/ -.. [#unlic] https://unlicense.org/ - - -Copyright -========= - -This document is placed in the public domain or under the CC0-1.0-Universal -license [#cc0]_, whichever is more permissive. - - -Acknowledgements -================ - -- Nick Coghlan -- Kevin P. Fleming -- Pradyun Gedam -- Oleg Grenrus -- Dustin Ingram -- Chris Jerdonek -- Cyril Roelandt -- Luis Villa - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 80 - End: diff --git a/pep-0649.rst b/pep-0649.rst deleted file mode 100644 index c5a0a2d28..000000000 --- a/pep-0649.rst +++ /dev/null @@ -1,993 +0,0 @@ -PEP: 649 -Title: Deferred Evaluation Of Annotations Using Descriptors -Author: Larry Hastings <larry@hastings.org> -Status: Draft -Type: Standards Track -Content-Type: text/x-rst -Created: 11-Jan-2021 -Post-History: 11-Jan-2021, 11-Apr-2021 - - -Abstract -======== - -As of Python 3.9, Python supports two different behaviors -for annotations: - -* original or "stock" Python semantics, in which annotations - are evaluated at the time they are bound, and -* PEP 563 semantics, currently enabled per-module by - ``from __future__ import annotations``, in which annotations - are converted back into strings and must be reparsed and - executed by ``eval()`` to be used. - -Original Python semantics created a circular references problem -for static typing analysis. PEP 563 solved that problem--but -its novel semantics introduced new problems, including its -restriction that annotations can only reference names at -module-level scope. - -This PEP proposes a third way that embodies the best of both -previous approaches. It solves the same circular reference -problems solved by PEP 563, while otherwise preserving Python's -original annotation semantics, including allowing annotations -to refer to local and class variables. - -In this new approach, the code to generate the annotations -dict is written to its own function which computes and returns -the annotations dict. Then, ``__annotations__`` is a "data -descriptor" which calls this annotation function once and -retains the result. This delays the evaluation of annotations -expressions until the annotations are examined, at which point -all circular references have likely been resolved. And if -the annotations are never examined, the function is never -called and the annotations are never computed. - -Annotations defined using this PEP's semantics have the same -visibility into the symbol table as annotations under "stock" -semantics--any name visible to an annotation in Python 3.9 -is visible to an annotation under this PEP. In addition, -annotations under this PEP can refer to names defined *after* -the annotation is defined, as long as the name is defined in -a scope visible to the annotation. Specifically, when this PEP -is active: - -* An annotation can refer to a local variable defined in the - current function scope. -* An annotation can refer to a local variable defined in an - enclosing function scope. -* An annotation can refer to a class variable defined in the - current class scope. -* An annotation can refer to a global variable. - -And in all four of these cases, the variable referenced by -the annotation needn't be defined at the time the annotation -is defined--it can be defined afterwards. The only restriction -is that the name or variable be defined before the annotation -is *evaluated.* - -If accepted, these new semantics for annotations would initially -be gated behind ``from __future__ import co_annotations``. -However, these semantics would eventually be promoted to be -Python's default behavior. Thus this PEP would *supersede* -PEP 563, and PEP 563's behavior would be deprecated and -eventually removed. - -Overview -======== - -.. note:: The code presented in this section is simplified - for clarity. The intention is to communicate the high-level - concepts involved without getting lost in with the details. - The actual details are often quite different. See the - Implementation_ section later in this PEP for a much more - accurate description of how this PEP works. - -Consider this example code: - -.. code-block:: - - def foo(x: int = 3, y: MyType = None) -> float: - ... - class MyType: - ... - foo_y_type = foo.__annotations__['y'] - -As we see here, annotations are available at runtime through an -``__annotations__`` attribute on functions, classes, and modules. -When annotations are specified on one of these objects, -``__annotations__`` is a dictionary mapping the names of the -fields to the value specified as that field's annotation. - -The default behavior in Python 3.9 is to evaluate the expressions -for the annotations, and build the annotations dict, at the time -the function, class, or module is bound. At runtime the above -code actually works something like this: - -.. code-block:: - - annotations = {'x': int, 'y': MyType, 'return': float} - def foo(x = 3, y = "abc"): - ... - foo.__annotations__ = annotations - class MyType: - ... - foo_y_type = foo.__annotations__['y'] - -The crucial detail here is that the values ``int``, ``MyType``, -and ``float`` are looked up at the time the function object is -bound, and these values are stored in the annotations dict. -But this code doesn't run—it throws a ``NameError`` on the first -line, because ``MyType`` hasn't been defined yet. - -PEP 563's solution is to decompile the expressions back -into strings, and store those *strings* in the annotations dict. -The equivalent runtime code would look something like this: - -.. code-block:: - - annotations = {'x': 'int', 'y': 'MyType', 'return': 'float'} - def foo(x = 3, y = "abc"): - ... - foo.__annotations__ = annotations - class MyType: - ... - foo_y_type = foo.__annotations__['y'] - -This code now runs successfully. However, ``foo_y_type`` -is no longer a reference to ``MyType``, it is the *string* -``'MyType'``. The code would have to be further modified to -call ``eval()`` or ``typing.get_type_hints()`` to convert -the string into a useful reference to the actual ``MyType`` -object. - -This PEP proposes a third approach, delaying the evaluation of -the annotations by computing them in their own function. If -this PEP was active, the generated code would work something -like this: - -.. code-block:: - - class function: - # __annotations__ on a function object is already a - # "data descriptor" in Python, we're just changing what it does - @property - def __annotations__(self): - return self.__co_annotations__() - - # ... - - def foo_annotations_fn(): - return {'x': int, 'y': MyType, 'return': float} - def foo(x = 3, y = "abc"): - ... - foo.__co_annotations__ = foo_annotations_fn - class MyType: - ... - foo_y_type = foo.__annotations__['y'] - -The important change is that the code constructing the -annotations dict now lives in a function—here, called -``foo_annotations_fn()``. But this function isn't called -until we ask for the value of ``foo.__annotations__``, -and we don't do that until *after* the definition of ``MyType``. -So this code also runs successfully, and ``foo_y_type`` now -has the correct value--the class ``MyType``--even though -``MyType`` wasn't defined until *after* the annotation was -defined. - - -Motivation -========== - -Python's original semantics for annotations made its use for -static type analysis painful due to forward reference problems. -This was the main justification for PEP 563, and we need not -revisit those arguments here. - -However, PEP 563's solution was to decompile code for Python -annotations back into strings at compile time, requiring -users of annotations to ``eval()`` those strings to restore -them to their actual Python values. This has several drawbacks: - -* It requires Python implementations to stringize their - annotations. This is surprising behavior—unprecedented - for a language-level feature. Also, adding this feature - to CPython was complicated, and this complicated code would - need to be reimplemented independently by every other Python - implementation. -* It requires that all annotations be evaluated at module-level - scope. Annotations under PEP 563 can no longer refer to - * class variables, - * local variables in the current function, or - * local variables in enclosing functions. -* It requires a code change every time existing code uses an - annotation, to handle converting the stringized - annotation back into a useful value. -* ``eval()`` is slow. -* ``eval()`` isn't always available; it's sometimes removed - from Python for space reasons. -* In order to evaluate the annotations on a class, - it requires obtaining a reference to that class's globals, - which PEP 563 suggests should be done by looking up that class - by name in ``sys.modules``—another surprising requirement for - a language-level feature. -* It adds an ongoing maintenance burden to Python implementations. - Every time the language adds a new feature available in expressions, - the implementation's stringizing code must be updated in - tandem in order to support decompiling it. - -This PEP also solves the forward reference problem outlined in -PEP 563 while avoiding the problems listed above: - -* Python implementations would generate annotations as code - objects. This is simpler than stringizing, and is something - Python implementations are already quite good at. This means: - - alternate implementations would need to write less code to - implement this feature, and - - the implementation would be simpler overall, which should - reduce its ongoing maintenance cost. -* Existing annotations would not need to be changed to only - use global scope. Actually, annotations would become much - easier to use, as they would now also handle forward - references. -* Code examining annotations at runtime would no longer need - to use ``eval()`` or anything else—it would automatically - see the correct values. This is easier, faster, and - removes the dependency on ``eval()``. - - -Backwards Compatibility -======================= - -PEP 563 changed the semantics of annotations. When its semantics -are active, annotations must assume they will be evaluated in -*module-level* scope. They may no longer refer directly -to local variables or class attributes. - -This PEP removes that restriction; annotations may refer to globals, -local variables inside functions, local variables defined in enclosing -functions, and class members in the current class. In addition, -annotations may refer to any of these that haven't been defined yet -at the time the annotation is defined, as long as the not-yet-defined -name is created normally (in such a way that it is known to the symbol -table for the relevant block, or is a global or class variable found -using normal name resolution). Thus, this PEP demonstrates *improved* -backwards compatibility over PEP 563. - -PEP 563 also requires using ``eval()`` or ``typing.get_type_hints()`` -to examine annotations. Code updated to work with PEP 563 that calls -``eval()`` directly would have to be updated simply to remove the -``eval()`` call. Code using ``typing.get_type_hints()`` would -continue to work unchanged, though future use of that function -would become optional in most cases. - -Because this PEP makes semantic changes to how annotations are -evaluated, this PEP will be initially gated with a per-module -``from __future__ import co_annotations`` before it eventually -becomes the default behavior. - -Apart from the delay in evaluating values stored in annotations -dicts, this PEP preserves nearly all existing behavior of -annotations dicts. Specifically: - -* Annotations dicts are mutable, and any changes to them are - preserved. -* The ``__annotations__`` attribute can be explicitly set, - and any value set this way will be preserved. -* The ``__annotations__`` attribute can be deleted using - the ``del`` statement. - -However, there are two uncommon interactions possible with class -and module annotations that work today—both with stock semantics, -and with PEP 563 semantics—that would no longer work when this PEP -was active. These two interactions would have to be prohibited. -The good news is, neither is common, and neither is considered good -practice. In fact, they're rarely seen outside of Python's own -regression test suite. They are: - -* *Code that sets annotations on module or class attributes - from inside any kind of flow control statement.* It's - currently possible to set module and class attributes with - annotations inside an ``if`` or ``try`` statement, and it works - as one would expect. It's untenable to support this behavior - when this PEP is active. -* *Code in module or class scope that references or modifies the - local* ``__annotations__`` *dict directly.* Currently, when - setting annotations on module or class attributes, the generated - code simply creates a local ``__annotations__`` dict, then sets - mappings in it as needed. It's also possible for user code - to directly modify this dict, though this doesn't seem like it's - an intentional feature. Although it would be possible to support - this after a fashion when this PEP was active, the semantics - would likely be surprising and wouldn't make anyone happy. - -Note that these are both also pain points for static type checkers, -and are unsupported by those checkers. It seems reasonable to -declare that both are at the very least unsupported, and their -use results in undefined behavior. It might be worth making a -small effort to explicitly prohibit them with compile-time checks. - -In addition, there are a few operators that would no longer be -valid for use in annotations, because their side effects would -affect the *annotation function* instead of the -class/function/module the annotation was nominally defined in: - -* ``:=`` (aka the "walrus operator"), -* ``yield`` and ``yield from``, and -* ``await``. - -Use of any of these operators in an annotation will result in a -compile-time error. - -Since delaying the evaluation of annotations until they are -evaluated changes the semantics of the language, it's observable -from within the language. Therefore it's possible to write code -that behaves differently based on whether annotations are -evaluated at binding time or at access time, e.g. - -.. code-block:: - - mytype = str - def foo(a:mytype): pass - mytype = int - print(foo.__annotations__['a']) - -This will print ``<class 'str'>`` with stock semantics -and ``<class 'int'>`` when this PEP is active. Since -this is poor programming style to begin with, it seems -acceptable that this PEP changes its behavior. - -Finally, there's a standard idiom that's actually somewhat common -when accessing class annotations, and which will become more -problematic when this PEP is active: code often accesses class -annotations via ``cls.__dict__.get("__annotations__", {})`` -rather than simply ``cls.__annotations__``. It's due to a flaw -in the original design of annotations themselves. This topic -will be examined in a separate discussion; the outcome of -that discussion will likely guide the future evolution of this -PEP. - - -Mistaken Rejection Of This Approach In November 2017 -==================================================== - -During the early days of discussion around PEP 563, -using code to delay the evaluation of annotations was -briefly discussed, in a November 2017 thread in -``comp.lang.python-dev``. At the time the -technique was termed an "implicit lambda expression". - -Guido van Rossum—Python's BDFL at the time—replied, -asserting that these "implicit lambda expression" wouldn't -work, because they'd only be able to resolve symbols at -module-level scope: - - IMO the inability of referencing class-level definitions - from annotations on methods pretty much kills this idea. - -https://mail.python.org/pipermail/python-dev/2017-November/150109.html - -This led to a short discussion about extending lambda-ized -annotations for methods to be able to refer to class-level -definitions, by maintaining a reference to the class-level -scope. This idea, too, was quickly rejected. - -PEP 563 summarizes the above discussion here: - -https://www.python.org/dev/peps/pep-0563/#keeping-the-ability-to-use-function-local-state-when-defining-annotations - -What's puzzling is PEP 563's own changes to the scoping rules -of annotations—it *also* doesn't permit annotations to reference -class-level definitions. It's not immediately clear why an -inability to reference class-level definitions was enough to -reject using "implicit lambda expressions" for annotations, -but was acceptable for stringized annotations. - -In retrospect there was probably a pivot during the development -of PEP 563. It seems that, early on, there was a prevailing -assumption that PEP 563 would support references to class-level -definitions. But by the time PEP 563 was finalized, this -assumption had apparently been abandoned. And it looks like -"implicit lambda expressions" were never reconsidered in this -new light. - -In any case, annotations are still able to refer to class-level -definitions under this PEP, rendering the objection moot. - -.. _Implementation: - -Implementation -============== - -There's a prototype implementation of this PEP, here: - -https://github.com/larryhastings/co_annotations/ - -As of this writing, all features described in this PEP are -implemented, and there are some rudimentary tests in the -test suite. There are still some broken tests, and the -``co_annotations`` repo is many months behind the -CPython repo. - - -from __future__ import co_annotations -------------------------------------- - -In the prototype, the semantics presented in this PEP are gated with: - -.. code-block:: - - from __future__ import co_annotations - - - -__co_annotations__ ------------------- - -Python supports runtime metadata for annotations for three different -types: function, classes, and modules. The basic approach to -implement this PEP is much the same for all three with only minor -variations. - -With this PEP, each of these types adds a new attribute, -``__co_annotations__``. ``__co_annotations__`` is a function: -it takes no arguments, and must return either ``None`` or a dict -(or subclass of dict). It adds the following semantics: - -* ``__co_annotations__`` is always set, and may contain either - ``None`` or a callable. -* ``__co_annotations__`` cannot be deleted. -* ``__annotations__`` and ``__co_annotations__`` can't both - be set to a useful value simultaneously: - - - If you set ``__annotations__`` to a dict, this also sets - ``__co_annotations__`` to None. - - If you set ``__co_annotations__`` to a callable, this also - deletes ``__annotations__`` - -Internally, ``__co_annotations__`` is a "data descriptor", -where functions are called whenever user code gets, sets, -or deletes the attribute. In all three cases, the object -has separate internal storage for the current value -of the ``__co_annotations__`` attribute. - -``__annotations__`` is also as a data descriptor, with its own -separate internal storage for its internal value. The code -implementing the "get" for ``__annotations__`` works something -like this: - -.. code-block:: - - if (the internal value is set) - return the internal annotations dict - if (__co_annotations__ is not None) - call the __co_annotations__ function - if the result is a dict: - store the result as the internal value - set __co_annotations__ to None - return the internal value - do whatever this object does when there are no annotations - - -Unbound code objects --------------------- - -When Python code defines one of these three objects with -annotations, the Python compiler generates a separate code -object which builds and returns the appropriate annotations -dict. Wherever possible, the "annotation code object" is -then stored *unbound* as the internal value of -``__co_annotations__``; it is then bound on demand when -the user asks for ``__annotations__``. - -This is a useful optimization for both speed and memory -consumption. Python processes rarely examine annotations -at runtime. Therefore, pre-binding these code objects to -function objects would usually be a waste of resources. - -When is this optimization not possible? - -* When an annotation function contains references to - free variables, in the current function or in an - outer function. -* When an annotation function is defined on a method - (a function defined inside a class) and the annotations - possibly refer directly to class variables. - -Note that user code isn't permitted to directly access these -unbound code objects. If the user "gets" the value of -``__co_annotations__``, and the internal value of -``__co_annotations__`` is an unbound code object, -it immediately binds the code object, and the resulting -function object is stored as the new value of -``__co_annotations__`` and returned. - -(However, these unbound code objects *are* stored in the -``.pyc`` file. So a determined user could examine them -should that be necessary for some reason.) - - - - -Function Annotations --------------------- - -When compiling a function, the CPython bytecode compiler -visits the annotations for the function all in one place, -starting with ``compiler_visit_annotations()`` in ``compile.c``. -If there are any annotations, they create the scope for -the annotations function on demand, and -``compiler_visit_annotations()`` assembles it. - -The code object is passed in in place of the annotations dict -for the ``MAKE_FUNCTION`` bytecode instruction. -``MAKE_FUNCTION`` supports a new bit in its oparg -bitfield, ``0x10``, which tells it to expect a -``co_annotations`` code object on the stack. -The bitfields for ``annotations`` (``0x04``) and -``co_annotations`` (``0x10``) are mutually exclusive. - -When binding an unbound annotation code object, a function will -use its own ``__globals__`` as the new function's globals. - -One quirk of Python: you can't actually remove the annotations -from a function object. If you delete the ``__annotations__`` -attribute of a function, then get its ``__annotations__`` member, -it will create an empty dict and use that as its -``__annotations__``. The implementation of this PEP maintains -this quirk for backwards compatibility. - - -Class Annotations ------------------ - -When compiling a class body, the compiler maintains two scopes: -one for the normal class body code, and one for annotations. -(This is facilitated by four new functions: ``compiler.c`` -adds ``compiler_push_scope()`` and ``compiler_pop_scope()``, -and ``symtable.c`` adds ``symtable_push_scope()`` and -``symtable_pop_scope()``.) -Once the code generator reaches the end of the class body, -but before it generates the bytecode for the class body, -it assembles the bytecode for ``__co_annotations__``, then -assigns that to ``__co_annotations__`` using ``STORE_NAME``. - -It also sets a new ``__globals__`` attribute. Currently it -does this by calling ``globals()`` and storing the result. -(Surely there's a more elegant way to find the class's -globals--but this was good enough for the prototype.) When -binding an unbound annotation code object, a class will use -the value of this ``__globals__`` attribute. When the class -drops its reference to the unbound code object--either because -it has bound it to a function, or because ``__annotations__`` -has been explicitly set--it also deletes its ``__globals__`` -attribute. - -As discussed above, examination or modification of -``__annotations__`` from within the class body is no -longer supported. Also, any flow control (``if`` or ``try`` blocks) -around declarations of members with annotations is unsupported. - -If you delete the ``__annotations__`` attribute of a class, -then get its ``__annotations__`` member, it will return the -annotations dict of the first base class with annotations set. -If no base classes have annotations set, it will raise -``AttributeError``. - -Although it's an implementation-specific detail, currently -classes store the internal value of ``__co_annotations__`` -in their ``tp_dict`` under the same name. - - -Module Annotations ------------------- - -Module annotations work much the same as class annotations. -The main difference is, a module uses its own dict as the -``__globals__`` when binding the function. - -If you delete the ``__annotations__`` attribute of a class, -then get its ``__annotations__`` member, the module will -raise ``AttributeError``. - -Annotations With Closures -------------------------- - -It's possible to write annotations that refer to -free variables, and even free variables that have yet -to be defined. For example: - -.. code-block:: - - from __future__ import co_annotations - - def outer(): - def middle(): - def inner(a:mytype, b:mytype2): pass - mytype = str - return inner - mytype2 = int - return middle() - - fn = outer() - print(fn.__annotations__) - -At the time ``fn`` is set, ``inner.__co_annotations__()`` -hasn't been run. So it has to retain a reference to -the *future* definitions of ``mytype`` and ``mytype2`` if -it is to correctly evaluate its annotations. - -If an annotation function refers to a local variable -from the current function scope, or a free variable -from an enclosing function scope--if, in CPython, the -annotation function code object contains one or more -``LOAD_DEREF`` opcodes--then the annotation code object -is bound at definition time with references to these -variables. ``LOAD_DEREF`` instructions require the annotation -function to be bound with special run-time information -(in CPython, a ``freevars`` array). Rather than store -that separately and use that to later lazy-bind the -function object, the current implementation simply -early-binds the function object. - -Note that, since the annotation function ``inner.__co_annotations__()`` -is defined while parsing ``outer()``, from Python's perspective -the annotation function is a "nested function". So "local -variable inside the 'current' function" and "free variable -from an enclosing function" are, from the perspective of -the annotation function, the same thing. - - -Annotations That Refer To Class Variables ------------------------------------------ - -It's possible to write annotations that refer to -class variables, and even class variables that haven't -yet been defined. For example: - -.. code-block:: - - from __future__ import co_annotations - - class C: - def method(a:mytype): pass - mytype = str - - print(C.method.__annotations__) - -Internally, annotation functions are defined as -a new type of "block" in CPython's symbol table -called an ``AnnotationBlock``. An ``AnnotationBlock`` -is almost identical to a ``FunctionBlock``. It differs -in that it's permitted to see names from an enclosing -class scope. (Again: annotation functions are functions, -and they're defined *inside* the same scope as -the thing they're being defined on. So in the above -example, the annotation function for ``C.method()`` -is defined inside ``C``.) - -If it's possible that an annotation function refers -to class variables--if all these conditions are true: - -* The annotation function is being defined inside - a class scope. -* The generated code for the annotation function - has at least one ``LOAD_NAME`` instruction. - -Then the annotation function is bound at the time -it's set on the class/function, and this binding -includes a reference to the class dict. The class -dict is pushed on the stack, and the ``MAKE_FUNCTION`` -bytecode instruction takes a new second bitfield (0x20) -indicating that it should consume that stack argument -and store it as ``__locals__`` on the newly created -function object. - -Then, at the time the function is executed, the -``f_locals`` field of the frame object is set to -the function's ``__locals__``, if set. This permits -``LOAD_NAME`` opcodes to work normally, which means -the code generated for annotation functions is nearly -identical to that generated for conventional Python -functions. - - -Interactive REPL Shell ----------------------- - -Everything works the same inside Python's interactive REPL shell, -except for module annotations in the interactive module (``__main__``) -itself. Since that module is never "finished", there's no specific -point where we can compile the ``__co_annotations__`` function. - -For the sake of simplicity, in this case we forego delayed evaluation. -Module-level annotations in the REPL shell will continue to work -exactly as they do today, evaluating immediately and setting the -result directly inside the ``__annotations__`` dict. - -(It might be possible to support delayed evaluation here. -But it gets complicated quickly, and for a nearly-non-existent -use case.) - - -Annotations On Local Variables Inside Functions ------------------------------------------------ - -Python supports syntax for local variable annotations inside -functions. However, these annotations have no runtime -effect--they're discarded at compile-time. Therefore, this -PEP doesn't need to do anything to support them, the same -as stock semantics and PEP 563. - - - -Performance Comparison ----------------------- - -Performance with this PEP should be favorable, when compared with either -stock behavior or PEP 563. In general, resources are only consumed -on demand—"you only pay for what you use". - -There are three scenarios to consider: - -* the runtime cost when annotations aren't defined, -* the runtime cost when annotations are defined but *not* referenced, and -* the runtime cost when annotations are defined *and* referenced. - -We'll examine each of these scenarios in the context of all three -semantics for annotations: stock, PEP 563, and this PEP. - -When there are no annotations, all three semantics have the same -runtime cost: zero. No annotations dict is created and no code is -generated for it. This requires no runtime processor time and -consumes no memory. - -When annotations are defined but not referenced, the runtime cost -of Python with this PEP should be roughly equal to or slightly better -than PEP 563 semantics, and slightly better than "stock" Python -semantics. The specifics depend on the object being annotated: - -* With stock semantics, the annotations dict is always built, and - set as an attribute of the object being annotated. -* In PEP 563 semantics, for function objects, a single constant - (a tuple) is set as an attribute of the function. For class and - module objects, the annotations dict is always built and set as - an attribute of the class or module. -* With this PEP, a single object is set as an attribute of the - object being annotated. Most often, this object is a constant - (a code object). In cases where the annotation refers to local - variables or class variables, the code object will be bound to - a function object, and the function object is set as the attribute - of the object being annotated. - -When annotations are both defined and referenced, code using -this PEP should be much faster than code using PEP 563 semantics, -and equivalent to or slightly improved over original Python -semantics. PEP 563 semantics requires invoking ``eval()`` for -every value inside an annotations dict, which is enormously slow. -And, as already mentioned, this PEP generates measurably more -efficient bytecode for class and module annotations than stock -semantics; for function annotations, this PEP and stock semantics -should be roughly equivalent. - -Memory use should also be comparable in all three scenarios across -all three semantic contexts. In the first and third scenarios, -memory usage should be roughly equivalent in all cases. -In the second scenario, when annotations are defined but not -referenced, using this PEP's semantics will mean the -function/class/module will store one unused code object (possibly -bound to an unused function object); with the other two semantics, -they'll store one unused dictionary (or constant tuple). - -Bytecode Comparison -------------------- - -The bytecode generated for annotations functions with -this PEP uses the efficient ``BUILD_CONST_KEY_MAP`` opcode -to build the dict for all annotatable objects: -functions, classes, and modules. - -Stock semantics also uses ``BUILD_CONST_KEY_MAP`` bytecode -for function annotations. PEP 563 has an even more efficient -method for building annotations dicts on functions, leveraging -the fact that its annotations dicts only contain strings for -both keys and values. At compile-time it constructs a tuple -containing pairs of keys and values at compile-time, then -at runtime it converts that tuple into a dict on demand. -This is a faster technique than either stock semantics -or this PEP can employ, because in those two cases -annotations dicts can contain Python values of any type. -Of course, this performance win is negated if the -annotations are examined, due to the overhead of ``eval()``. - -For class and module annotations, both stock semantics -and PEP 563 generate a longer and slightly-less-efficient -stanza of bytecode, creating the dict and setting the -annotations individually. - - -For Future Discussion -===================== - -Circular Imports ----------------- - -There is one unfortunately-common scenario where PEP 563 -currently provides a better experience, and it has to do -with large code bases, with circular dependencies and -imports, that examine their annotations at run-time. - -PEP 563 permitted defining *and examining* invalid -expressions as annotations. Its implementation requires -annotations to be legal Python expressions, which it then -converts into strings at compile-time. But legal Python -expressions may not be computable at runtime, if for -example the expression references a name that isn't defined. -This is a problem for stringized annotations if they're -evaluated, e.g. with ``typing.get_type_hints()``. But -any stringized annotation may be examined harmlessly at -any time--as long as you don't evaluate it, and only -examine it as a string. - -Some large organizations have code bases that unfortunately -have circular dependency problems with their annotations--class -A has methods annotated with class B, but class B has methods -annotated with class A--that can be difficult to resolve. -Since PEP 563 stringizes their annotations, it allows them -to leave these circular dependencies in place, and they can -sidestep the circular import problem by never importing the -module that defines the types used in the annotations. Their -annotations can no longer be evaluated, but this appears not -to be a concern in practice. They can then examine the -stringized form of the annotations at runtime and this seems -to be sufficient for their needs. - -This PEP allows for many of the same behaviors. -Annotations must be legal Python expressions, which -are compiled into a function at compile-time. -And if the code never examines an annotation, it won't -have any runtime effect, so here too annotations can -harmlessly refer to undefined names. (It's exactly -like defining a function that refers to undefined -names--then never calling that function. Until you -call the function, nothing bad will happen.) - -But examining an annotation when this PEP is active -means evaluating it, which means the names evaluated -in that expression must be defined. An undefined name -will throw a ``NameError`` in an annotation function, -just as it would with a stringized annotation passed -in to ``typing.get_type_hints()``, and just like any -other context in Python where an expression is evaluated. - -In discussions we have yet to find a solution to this -problem that makes all the participants in the -conversation happy. There are various avenues to explore -here: - -* One workaround is to continue to stringize one's - annotations, either by hand or done automatically - by the Python compiler (as it does today with - ``from __future__ import annotations``). This might - mean preserving Python's current stringizing annotations - going forward, although leaving it turned off by default, - only available by explicit request (though likely with - a different mechanism than - ``from __future__ import annotations``). -* Another possible workaround involves importing - the circularly-dependent modules separately, then - externally adding ("monkey-patching") their dependencies - to each other after the modules are loaded. As long - as the modules don't examine their annotations until - after they are completely loaded, this should work fine - and be maintainable with a minimum of effort. -* A third and more radical approach would be to change the - semantics of annotations so that they don't raise a - ``NameError`` when an unknown name is evaluated, - but instead create some sort of proxy "reference" object. -* Of course, even if we do deprecate PEP 563, it will be - several releases before the before the functionality - is removed, giving us several years in which to to - research and innovate new solutions for this problem. - -In any case, the participants of the discussion agree that -this PEP should still move forward, even as this issue remains -currently unresolved [1]_. - -.. [1] https://github.com/larryhastings/co_annotations/issues/1 - - -cls.__globals__ and fn.__locals__ ---------------------------------- - -Is it permissible to add the ``__globals__`` reference to class -objects as proposed here? It's not clear why this hasn't already -been done; PEP 563 could have made use of class globals, but instead -made do with looking up classes inside ``sys.modules``. Python -seems strangely allergic to adding a ``__globals__`` reference to -class objects. - -If adding ``__globals__`` to class objects is indeed a bad idea -(for reasons I don't know), here are two alternatives as to -how classes could get a reference to their globals for the -implementation of this PEP: - -* The generate code for a class could bind its annotations code - object to a function at the time the class is bound, rather than - waiting for ``__annotations__`` to be referenced, making them an - exception to the rule (even though "special cases aren't special - enough to break the rules"). This would result in a small - additional runtime cost when annotations were defined but not - referenced on class objects. Honestly I'm more worried about - the lack of symmetry in semantics. (But I wouldn't want to - pre-bind all annotations code objects, as that would become - much more costly for function objects, even as annotations are - rarely used at runtime.) -* Use the class's ``__module__`` attribute to look up its module - by name in ``sys.modules``. This is what PEP 563 advises. - While this is passable for userspace or library code, it seems - like a little bit of a code smell for this to be defined semantics - baked into the language itself. - -Also, the prototype gets globals for class objects by calling -``globals()`` then storing the result. I'm sure there's a much -faster way to do this, I just didn't know what it was when I was -prototyping. I'm sure we can revise this to something much faster -and much more sanitary. I'd prefer to make it completely internal -anyway, and not make it visible to the user (via this new -__globals__ attribute). There's possibly already a good place to -put it anyway--``ht_module``. - -Similarly, this PEP adds one new dunder member to functions, -classes, and modules (``__co_annotations__``), and a second new -dunder member to functions (``__locals__``). This might be -considered excessive. - - -Bikeshedding the name ---------------------- - -During most of the development of this PEP, user code actually -could see the raw annotation code objects. ``__co_annotations__`` -could only be set to a code object; functions and other callables -weren't permitted. In that context the name ``co_annotations`` -makes a lot of sense. But with this last-minute pivot where -``__co_annotations__`` now presents itself as a callable, -perhaps the name of the attribute and the name of the -``from __future__ import`` needs a re-think. - - -Acknowledgements -================ - -Thanks to Barry Warsaw, Eric V. Smith, Mark Shannon, -and Guido van Rossum for feedback and encouragement. -Thanks in particular to Mark Shannon for two key -suggestions—build the entire annotations dict inside -a single code object, and only bind it to a function -on demand—that quickly became among the best aspects -of this proposal. Also, thanks in particular to Guido -van Rossum for suggesting that ``__co_annotations__`` -functions should duplicate the name visibility rules of -annotations under "stock" semantics--this resulted in -a sizeable improvement to the second draft. Finally, -special thanks to Jelle Zijlstra, who contributed not -just feedback--but code! - - -Copyright -========= - -This document is placed in the public domain or under the -CC0-1.0-Universal license, whichever is more permissive. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0659.rst b/pep-0659.rst deleted file mode 100644 index 74728a69a..000000000 --- a/pep-0659.rst +++ /dev/null @@ -1,308 +0,0 @@ -PEP: 659 -Title: Specializing Adaptive Interpreter -Author: Mark Shannon <mark@hotpy.org> -Status: Draft -Type: Informational -Content-Type: text/x-rst -Created: 13-Apr-2021 -Post-History: 11-May-2021 - - -Abstract -======== - -In order to perform well, virtual machines for dynamic languages must specialize the code that they execute -to the types and values in the program being run. -This specialization is often associated with "JIT" compilers, but is beneficial even without machine code generation. - -A specializing, adaptive interpreter is one that speculatively specializes on the types or values it is currently operating on, -and adapts to changes in those types and values. - -Specialization gives us improved performance, and adaptation allows the interpreter to rapidly change when the pattern of usage in a program alters, -limiting the amount of additional work caused by mis-specialization. - -This PEP proposes using a specializing, adaptive interpreter that specializes code aggressively, but over a very small region, -and is able to adjust to mis-specialization rapidly and at low cost. - -Adding a specializing, adaptive interpreter to CPython will bring significant performance improvements. -It is hard to come up with meaningful numbers, as it depends very much on the benchmarks and on work that has not yet happened. -Extensive experimentation suggests speedups of up to 50%. -Even if the speedup were only 25%, this would still be a worthwhile enhancement. - -Motivation -========== - -Python is widely acknowledged as slow. -Whilst Python will never attain the performance of low-level languages like C, Fortran, or even Java, -we would like it to be competitive with fast implementations of scripting languages, like V8 for Javascript or luajit for lua. -Specifically, we want to achieve these performance goals with CPython to benefit all users of Python -including those unable to use PyPy or other alternative virtual machines. - -Achieving these performance goals is a long way off, and will require a lot of engineering effort, -but we can make a significant step towards those goals by speeding up the interpreter. -Both academic research and practical implementations have shown that a fast interpreter is a key part of a fast virtual machine. - -Typical optimizations for virtual machines are expensive, so a long "warm up" time is required -to gain confidence that the cost of optimization is justified. -In order to get speed-ups rapidly, without noticeable warmup times, -the VM should speculate that specialization is justified even after a few executions of a function. -To do that effectively, the interpreter must be able to optimize and deoptimize continually and very cheaply. - -By using adaptive and speculative specialization at the granularity of individual virtual machine instructions, we get a faster -interpreter that also generates profiling information for more sophisticated optimizations in the future. - -Rationale -========= - -There are many practical ways to speed-up a virtual machine for a dynamic language. -However, specialization is the most important, both in itself and as an enabler of other optimizations. -Therefore it makes sense to focus our efforts on specialization first, if we want to improve the performance of CPython. - -Specialization is typically done in the context of a JIT compiler, but research shows specialization in an interpreter -can boost performance significantly, even outperforming a naive compiler [1]_. - -There have been several ways of doing this proposed in the academic literature, -but most attempt to optimize regions larger than a single bytecode [1]_ [2]_. -Using larger regions than a single instruction, requires code to handle deoptimization in the middle of a region. -Specialization at the level of individual bytecodes makes deoptimization trivial, as it cannot occur in the middle of a region. - -By speculatively specializing individual bytecodes, we can gain significant performance improvements without anything but the most local, -and trivial to implement, deoptimizations. - -The closest approach to this PEP in the literature is "Inline Caching meets Quickening" [3]_. -This PEP has the advantages of inline caching, but adds the ability to quickly deoptimize making the performance -more robust in cases where specialization fails or is not stable. - -Performance ------------ - -The expected speedup of 50% can be broken roughly down as follows: - -* In the region of 30% from specialization. Much of that is from specialization of calls, - with improvements in instructions that are already specialized such as ``LOAD_ATTR`` and ``LOAD_GLOBAL`` - contributing much of the remainder. Specialization of operations adds a small amount. -* About 10% from improved dispatch such as super-instructions and other optimizations enabled by quickening. -* Further increases in the benefits of other optimizations, as they can exploit, or be exploited by specialization. - -Implementation -============== - -Overview --------- - -Once any instruction in a code object has executed a few times, that code object will be "quickened" by allocating a new array -for the bytecode that can be modified at runtime, and is not constrained as the ``code.co_code`` object is. -From that point onwards, whenever any instruction in that code object is executed, it will use the quickened form. - -Any instruction that would benefit from specialization will be replaced by an "adaptive" form of that instruction. -When executed, the adaptive instructions will specialize themselves in response to the types and values that they see. - -Quickening ----------- - -Quickening is the process of replacing slow instructions with faster variants. - -Quickened code has number of advantages over the normal bytecode: - -* It can be changed at runtime -* It can use super-instructions that span lines and take multiple operands. -* It does not need to handle tracing as it can fallback to the normal bytecode for that. - -In order that tracing can be supported, and quickening performed quickly, the quickened instruction format should match the normal -bytecode format: 16-bit instructions of 8-bit opcode followed by 8-bit operand. - -Adaptive instructions ---------------------- - -Each instruction that would benefit from specialization is replaced by an adaptive version during quickening. -For example, the ``LOAD_ATTR`` instruction would be replaced with ``LOAD_ATTR_ADAPTIVE``. - -Each adaptive instruction maintains a counter, and periodically attempts to specialize itself. - -Specialization --------------- - -CPython bytecode contains many bytecodes that represent high-level operations, and would benefit from specialization. -Examples include ``CALL_FUNCTION``, ``LOAD_ATTR``, ``LOAD_GLOBAL`` and ``BINARY_ADD``. - -By introducing a "family" of specialized instructions for each of these instructions allows effective specialization, -since each new instruction is specialized to a single task. -Each family will include an "adaptive" instruction, that maintains a counter and periodically attempts to specialize itself. -Each family will also include one or more specialized instructions that perform the equivalent -of the generic operation much faster provided their inputs are as expected. -Each specialized instruction will maintain a saturating counter which will be incremented whenever the inputs are as expected. -Should the inputs not be as expected, the counter will be decremented and the generic operation will be performed. -If the counter reaches the minimum value, the instruction is deoptimized by simply replacing its opcode with the adaptive version. - -Ancillary data --------------- - -Most families of specialized instructions will require more information than can fit in an 8-bit operand. -To do this, an array of specialization data entries will be maintained alongside the new instruction array. -For instructions that need specialization data, the operand in the quickened array will serve as a partial index, -along with the offset of the instruction, to find the first specialization data entry for that instruction. -Each entry will be 8 bytes (for a 64 bit machine). The data in an entry, and the number of entries needed, will vary from instruction to instruction. - -Data layout ------------ - -Quickened instructions will be stored in an array (it is neither necessary not desirable to store them in a Python object) with the same -format as the original bytecode. Ancillary data will be stored in a separate array. - -Each instruction will use 0 or more data entries. Each instruction within a family must have the same amount of data allocated, although some -instructions may not use all of it. Instructions that cannot be specialized, e.g. ``POP_TOP``, do not need any entries. -Experiments show that 25% to 30% of instructions can be usefully specialized. -Different families will need different amounts of data, but most need 2 entries (16 bytes on a 64 bit machine). - -In order to support larger functions than 256 instructions, we compute the offset of the first data entry for instructions -as ``(instruction offset)//2 + (quickened operand)``. - -Compared to the opcache in Python 3.10, this design: - -* is faster; it requires no memory reads to compute the offset. 3.10 requires two reads, which are dependent. -* uses much less memory, as the data can be different sizes for different instruction families, and doesn't need an additional array of offsets. -* can support much larger functions, up to about 5000 instructions per function. 3.10 can support about 1000. - - -Example families of instructions --------------------------------- - -CALL_FUNCTION -''''''''''''' - -The ``CALL_FUNCTION`` instruction calls the (N+1)th item on the stack with top N items on the stack as arguments. - -This is an obvious candidate for specialization. For example, the call in ``len(x)`` is represented as the bytecode ``CALL_FUNCTION 1``. -In this case we would always expect the object ``len`` to be the function. We probably don't want to specialize for ``len`` -(although we might for ``type`` and ``isinstance``), but it would be beneficial to specialize for builtin functions taking a single argument. -A fast check that the underlying function is a builtin function taking a single argument (``METHOD_O``) would allow us to avoid a -sequence of checks for number of parameters and keyword arguments. - -``CALL_FUNCTION_ADAPTIVE`` would track how often it is executed, and call the ``call_function_optimize`` when executed enough times, or jump -to ``CALL_FUNCTION`` otherwise. -When optimizing, the kind of the function would be checked and if a suitable specialized instruction was found, -it would replace ``CALL_FUNCTION_ADAPTIVE`` in place. - -Specializations might include: - -* ``CALL_FUNCTION_PY_SIMPLE``: Calls to Python functions with exactly matching parameters. -* ``CALL_FUNCTION_PY_DEFAULTS``: Calls to Python functions with more parameters and default values. - Since the exact number of defaults needed is known, the instruction needs to do no additional checking or computation; just copy some defaults. -* ``CALL_BUILTIN_O``: The example given above for calling builtin methods taking exactly one argument. -* ``CALL_BUILTIN_VECTOR``: For calling builtin function taking vector arguments. - -Note how this allows optimizations that complement other optimizations. -For example, if the Python and C call stacks were decoupled and the data stack were contiguous, -then Python-to-Python calls could be made very fast. - -LOAD_GLOBAL -''''''''''' - -The ``LOAD_GLOBAL`` instruction looks up a name in the global namespace and then, if not present in the global namespace, -looks it up in the builtins namespace. -In 3.9 the C code for the ``LOAD_GLOBAL`` includes code to check to see whether the whole code object should be modified to add a cache, -whether either the global or builtins namespace, code to lookup the value in a cache, and fallback code. -This makes it complicated and bulky. It also performs many redundant operations even when supposedly optimized. - -Using a family of instructions makes the code more maintainable and faster, as each instruction only needs to handle one concern. - -Specializations would include: - -* ``LOAD_GLOBAL_ADAPTIVE`` would operate like ``CALL_FUNCTION_ADAPTIVE`` above. -* ``LOAD_GLOBAL_MODULE`` can be specialized for the case where the value is in the globals namespace. - After checking that the keys of the namespace have not changed, it can load the value from the stored index. -* ``LOAD_GLOBAL_BUILTIN`` can be specialized for the case where the value is in the builtins namespace. - It needs to check that the keys of the global namespace have not been added to, and that the builtins namespace has not changed. - Note that we don't care if the values of the global namespace have changed, just the keys. - -See [4]_ for a full implementation. - -.. note:: - - This PEP outlines the mechanisms for managing specialization, and does not specify the particular optimizations to be applied. - The above scheme is just one possible scheme. Many others are possible and may well be better. - -Compatibility -============= - -There will be no change to the language, library or API. - -The only way that users will be able to detect the presence of the new interpreter is through timing execution, the use of debugging tools, -or measuring memory use. - -Costs -===== - -Memory use ----------- - -An obvious concern with any scheme that performs any sort of caching is "how much more memory does it use?". -The short answer is "none". - -Comparing memory use to 3.10 -'''''''''''''''''''''''''''' -The following table shows the additional bytes per instruction to support the 3.10 opcache -or the proposed adaptive interpreter, on a 64 bit machine. - -================ ===== ======== ===== ===== - Version 3.10 3.10 opt 3.11 3.11 - Specialised 20% 20% 25% 33% ----------------- ----- -------- ----- ----- - quickened code 0 0 2 2 - opcache_map 1 1 0 0 - opcache/data 6.4 4.8 4 5.3 ----------------- ----- -------- ----- ----- - Total 7.4 5.8 6 7.3 -================ ===== ======== ===== ===== - -``3.10`` is the current version of 3.10 which uses 32 bytes per entry. -``3.10 opt`` is a hypothetical improved version of 3.10 that uses 24 bytes per entry. - -Even if one third of all instructions were specialized (a high proportion), then the memory use is still less than -that of 3.10. With a more realistic 25%, then memory use is basically the same as the hypothetical improved version of 3.10. - - -Security Implications -===================== - -None - - -Rejected Ideas -============== - -Too many to list. - - -References -========== - -.. [1] The construction of high-performance virtual machines for dynamic languages, Mark Shannon 2010. - http://theses.gla.ac.uk/2975/1/2011shannonphd.pdf - -.. [2] Dynamic Interpretation for Dynamic Scripting Languages - https://www.scss.tcd.ie/publications/tech-reports/reports.09/TCD-CS-2009-37.pdf - -.. [3] Inline Caching meets Quickening - http://www.complang.tuwien.ac.at/kps09/pdfs/brunthaler.pdf - -.. [4] Adaptive specializing examples (This will be moved to a more permanent location, once this PEP is accepted) - https://gist.github.com/markshannon/556ccc0e99517c25a70e2fe551917c03 - - -Copyright -========= - -This document is placed in the public domain or under the -CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0661.rst b/pep-0661.rst deleted file mode 100644 index 24237c3d7..000000000 --- a/pep-0661.rst +++ /dev/null @@ -1,343 +0,0 @@ -PEP: 661 -Title: Sentinel Values -Author: Tal Einat <tal@python.org> -Discussions-To: https://discuss.python.org/t/pep-661-sentinel-values/9126 -Status: Draft -Type: Standards Track -Content-Type: text/x-rst -Created: 06-Jun-2021 -Post-History: 06-Jun-2021 - - -TL;DR: See the `Specification`_ and `Reference Implementation`_. - - -Abstract -======== - -Unique placeholder values, commonly known as "sentinel values", are useful in -Python programs for several things, such as default values for function -arguments where ``None`` is a valid input value. These cases are common -enough for several idioms for implementing such "sentinels" to have arisen -over the years, but uncommon enough that there hasn't been a clear need for -standardization. However, the common implementations, including some in the -stdlib, suffer from several significant drawbacks. - -This PEP suggests adding a utility for defining sentinel values, to be used -in the stdlib and made publicly available as part of the stdlib. - -Note: Changing all existing sentinels in the stdlib to be implemented this -way is not deemed necessary, and whether to do so is left to the discretion -of each maintainer. - - -Motivation -========== - -In May 2021, a question was brought up on the python-dev mailing list -[#python-dev-thread]_ about how to better implement a sentinel value for -``traceback.print_exception``. The existing implementation used the -following common idiom:: - - _sentinel = object() - -However, this object has an uninformative and overly verbose repr, causing the -function's signature to be overly long and hard to read:: - - >>> help(traceback.print_exception) - Help on function print_exception in module traceback: - - print_exception(exc, /, value=<object object at - 0x000002825DF09650>, tb=<object object at 0x000002825DF09650>, - limit=None, file=None, chain=True) - -Additionally, two other drawbacks of many existing sentinels were brought up -in the discussion: - -1. Not having a distinct type, hence it being impossible to define strict - type signatures functions with sentinels as default values -2. Incorrect behavior after being copied or unpickled, due to a separate - instance being created and thus comparisons using ``is`` failing - -In the ensuing discussion, Victor Stinner supplied a list of currently used -sentinel values in the Python standard library [#list-of-sentinels-in-stdlib]_. -This showed that the need for sentinels is fairly common, that there are -various implementation methods used even within the stdlib, and that many of -these suffer from at least one of the aforementioned drawbacks. - -The discussion did not lead to any clear consensus on whether a standard -implementation method is needed or desirable, whether the drawbacks mentioned -are significant, nor which kind of implementation would be good. - -A poll was created on discuss.python.org [#poll]_ to get a clearer sense of -the community's opinions. The poll's results were not conclusive, with 40% -voting for "The status-quo is fine / there’s no need for consistency in -this", but most voters voting for one or more standardized solutions. -Specifically, 37% of the voters chose "Consistent use of a new, dedicated -sentinel factory / class / meta-class, also made publicly available in the -stdlib". - -With such mixed opinions, this PEP was created to facilitate making a decision -on the subject. - - -Rationale -========= - -The criteria guiding the chosen implementation were: - -1. The sentinel objects should behave as expected by a sentinel object: When - compared using the ``is`` operator, it should always be considered identical - to itself but never to any other object. -2. It should be simple to define as many distinct sentinel values as needed. -3. The sentinel objects should have a clear and short repr. -4. The sentinel objects should each have a *distinct* type, usable in type - annotations to define *strict* type signatures. -5. The sentinel objects should behave correctly after copying and/or - unpickling. -6. Creating a sentinel object should be a simple, straightforward one-liner. -7. Works using CPython and PyPy3. Will hopefully also work with other - implementations. - -After researching existing idioms and implementations, and going through many -different possible implementations, an implementation was written which meets -all of these criteria (see `Reference Implementation`_). - - -Specification -============= - -A new ``sentinel`` function will be added to a new ``sentinels`` module. -It will accept a single required argument, the name of the sentinel object, -and a single optional argument, the repr of the object. - -:: - - >>> NotGiven = sentinel('NotGiven') - >>> NotGiven - <NotGiven> - >>> MISSING = sentinel('MISSING', repr='mymodule.MISSING') - >>> MISSING - mymodule.MISSING - -Checking if a value is such a sentinel *should* be done using the ``is`` -operator, as is recommended for ``None``. Equality checks using ``==`` will -also work as expected, returning ``True`` only when the object is compared -with itself. - -The name should be set to the name of the variable used to reference the -object, as in the examples above. Otherwise, the sentinel object won't be -able to survive copying or pickling+unpickling while retaining the above -described behavior. Note, that when defined in a class scope, the name must -be the fully-qualified name of the variable in the module, for example:: - - class MyClass: - NotGiven = sentinel('MyClass.NotGiven') - -Type annotations for sentinel values will use `typing.Literal`_. -For example:: - - def foo(value: int | Literal[NotGiven]) -> None: - ... - -.. _typing.Literal: https://docs.python.org/3/library/typing.html#typing.Literal - - -Reference Implementation -======================== - -The reference implementation is found in a dedicated GitHub repo -[#reference-github-repo]_. A simplified version follows:: - - def sentinel(name, repr=None): - """Create a unique sentinel object.""" - repr = repr or f'<{name}>' - - module = _get_parent_frame().f_globals.get('__name__', '__main__') - class_name = _get_class_name(name, module) - class_namespace = { - '__repr__': lambda self: repr, - } - cls = type(class_name, (), class_namespace) - cls.__module__ = module - _get_parent_frame().f_globals[class_name] = cls - - sentinel = cls() - cls.__new__ = lambda cls_: sentinel - - return sentinel - - def _get_class_name(sentinel_qualname, module_name): - return '__'.join(['_sentinel_type', - module_name.replace('.', '_'), - sentinel_qualname.replace('.', '_')]) - - -Note that a dedicated class is created automatically for each sentinel object. -This class is assigned to the namespace of the module from which the -``sentinel()`` call was made, or to that of the ``sentinels`` module itself as -a fallback. These classes have long names comprised of several parts to -ensure their uniqueness. However, these names usually wouldn't be used, since -type annotations should use ``Literal[]`` as described above, and identity -checks should be preferred over type checks. - - -Rejected Ideas -============== - - -Use ``NotGiven = object()`` ---------------------------- - -This suffers from all of the drawbacks mentioned in the `Rationale`_ section. - - -Add a single new sentinel value, e.g. ``MISSING`` or ``Sentinel`` ------------------------------------------------------------------ - -Since such a value could be used for various things in various places, one -could not always be confident that it would never be a valid value in some use -cases. On the other hand, a dedicated and distinct sentinel value can be used -with confidence without needing to consider potential edge-cases. - -Additionally, it is useful to be able to provide a meaningful name and repr -for a sentinel value, specific to the context where it is used. - -Finally, this was a very unpopular option in the poll [#poll]_, with only 12% -of the votes voting for it. - - -Use the existing ``Ellipsis`` sentinel value --------------------------------------------- - -This is not the original intended use of Ellipsis, though it has become -increasingly common to use it to define empty class or function blocks instead -of using ``pass``. - -Also, similar to a potential new single sentinel value, ``Ellipsis`` can't be -as confidently used in all cases, unlike a dedicated, distinct value. - - -Use a single-valued enum ------------------------- - -The suggested idiom is: - -:: - - class NotGivenType(Enum): - NotGiven = 'NotGiven' - NotGiven = NotGivenType.NotGiven - -Besides the excessive repetition, the repr is overly long: -``<NotGivenType.NotGiven: 'NotGiven'>``. A shorter repr can be defined, at -the expense of a bit more code and yet more repetition. - -Finally, this option was the least popular among the nine options in the poll -[#poll]_, being the only option to receive no votes. - - -A sentinel class decorator --------------------------- - -The suggested interface: - -:: - - @sentinel(repr='<NotGiven>') - class NotGivenType: pass - NotGiven = NotGivenType() - -While this allowed for a very simple and clear implementation, the interface -is too verbose, repetitive, and difficult to remember. - - -Using class objects -------------------- - -Since classes are inherently singletons, using a class as a sentinel value -makes sense and allows for a simple implementation. - -The simplest version of this idiom is: - -:: - - class NotGiven: pass - -To have a clear repr, one could define ``__repr__``: - -:: - - class NotGiven: - def __repr__(self): - return '<NotGiven>' - -... or use a meta-class: - -:: - - class NotGiven(metaclass=SentinelMeta): pass - -However, all such implementations don't have a dedicated type for the -sentinel, which is considered desirable for strict typing. A dedicated type -could be created by a meta-class or class decorator, but at that point the -implementation would become much more complex and loses its advantages over -the chosen implementation. - -Additionally, using classes this way is unusual and could be confusing. - - -Define a recommended "standard" idiom, without supplying an implementation --------------------------------------------------------------------------- - -Most common exiting idioms have significant drawbacks. So far, no idiom -has been found that is clear and concise while avoiding these drawbacks. - -Also, in the poll on this subject [#poll]_, the options for recommending an -idiom were unpopular, with the highest-voted option being voted for by only -25% of the voters. - - -Additional Notes -================ - -* This PEP and the initial implementation are drafted in a dedicated GitHub - repo [#reference-github-repo]_. - -* The support for copying/unpickling works when defined in a module's scope or - a (possibly nested) class's scope. Note that in the latter case, the name - provided as the first parameter must be the fully-qualified name of the - variable in the module:: - - class MyClass: - NotGiven = sentinel('MyClass.NotGiven', repr='<NotGiven>') - - -References -========== - -.. [#python-dev-thread] Python-Dev mailing list: `The repr of a sentinel <https://mail.python.org/archives/list/python-dev@python.org/thread/ZLVPD2OISI7M4POMTR2FCQTE6TPMPTO3/>`_ -.. [#list-of-sentinels-in-stdlib] Python-Dev mailing list: `"The stdlib contains tons of sentinels" <https://mail.python.org/archives/list/python-dev@python.org/message/JBYXQH3NV3YBF7P2HLHB5CD6V3GVTY55/>`_ -.. [#poll] discuss.python.org Poll: `Sentinel Values in the Stdlib <https://discuss.python.org/t/sentinel-values-in-the-stdlib/8810/>`_ -.. [#reference-github-repo] `Reference implementation at the taleinat/python-stdlib-sentinels GitHub repo <https://github.com/taleinat/python-stdlib-sentinels>`_ -.. [5] `bpo-44123: Make function parameter sentinel values true singletons <https://bugs.python.org/issue44123>`_ -.. [6] `The "sentinels" package on PyPI <https://pypi.org/project/sentinels/>`_ -.. [7] `The "sentinel" package on PyPI <https://pypi.org/project/sentinel/>`_ -.. [8] `Discussion thread about type signatures for these sentinels on the typing-sig mailing list <https://mail.python.org/archives/list/typing-sig@python.org/thread/NDEJ7UCDPINP634GXWDARVMTGDVSNBKV/#LVCPTY26JQJW7NKGKGAZXHQKWVW7GOGL>`_ - - -Copyright -========= - -This document is placed in the public domain or under the -CC0-1.0-Universal license, whichever is more permissive. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0664.rst b/pep-0664.rst deleted file mode 100644 index 4857b30f6..000000000 --- a/pep-0664.rst +++ /dev/null @@ -1,101 +0,0 @@ -PEP: 664 -Title: Python 3.11 Release Schedule -Version: $Revision$ -Last-Modified: $Date$ -Author: Pablo Galindo Salgado <pablogsal@python.org> -Status: Draft -Type: Informational -Content-Type: text/x-rst -Created: 12-Jul-2021 -Python-Version: 3.11 - - -Abstract -======== - -This document describes the development and release schedule for -Python 3.11. The schedule primarily concerns itself with PEP-sized -items. - -.. Small features may be added up to the first beta - release. Bugs may be fixed until the final release, - which is planned for end of October 2021. - -Release Manager and Crew -======================== - -- 3.11 Release Manager: Pablo Galindo Salgado -- Windows installers: Steve Dower -- Mac installers: Ned Deily -- Documentation: Julien Palard - - -Release Schedule -================ - -3.11.0 schedule ---------------- - -Note: the dates below use a 17-month development period that results -in a 12-month release cadence between major versions, as defined by -PEP 602. - -Actual: - -- 3.11 development begins: Monday, 2021-05-03 -- 3.11.0 alpha 1: Monday, 2021-10-05 - -Expected: - -- 3.11.0 alpha 2: Tuesday, 2021-11-02 -- 3.11.0 alpha 3: Monday, 2021-12-06 -- 3.11.0 alpha 4: Monday, 2022-01-03 -- 3.11.0 alpha 5: Wednesday, 2022-02-02 -- 3.11.0 alpha 6: Monday, 2022-02-28 -- 3.11.0 alpha 7: Tuesday, 2022-04-05 -- 3.11.0 beta 1: Friday, 2022-05-06 - -(No new features beyond this point.) - -- 3.11.0 beta 2: Monday, 2022-05-30 -- 3.11.0 beta 3: Thursday, 2022-06-16 -- 3.11.0 beta 4: Saturday, 2022-07-09 -- 3.11.0 candidate 1: Monday, 2022-08-01 -- 3.11.0 candidate 2: Monday, 2022-09-05 -- 3.11.0 final: Monday, 2022-10-03 - -Subsequent bugfix releases every two months. - - -3.11 Lifespan -------------- - -3.11 will receive bugfix updates approximately every 2 months for -approximately 18 months. Some time after the release of 3.12.0 final, -the ninth and final 3.11 bugfix update will be released. After that, -it is expected that security updates (source only) will be released -until 5 years after the release of 3.11.0 final, so until approximately -October 2027. - - -Features for 3.11 -================= - -Some of the notable features of Python 3.11 include: - - ** Watch this space :) ** - -Copyright -========= - -This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 72 - coding: utf-8 - End: diff --git a/pep-0670.rst b/pep-0670.rst deleted file mode 100644 index 0549dcd7e..000000000 --- a/pep-0670.rst +++ /dev/null @@ -1,367 +0,0 @@ -PEP: 670 -Title: Convert macros to functions in the Python C API -Author: Erlend Egeberg Aasland <erlend.aasland@protonmail.com>, - Victor Stinner <vstinner@python.org> -Status: Draft -Type: Standards Track -Content-Type: text/x-rst -Created: 19-Oct-2021 -Python-Version: 3.11 - - -Abstract -======== - -Convert macros to static inline functions or regular functions. - -Remove the return value of macros having a return value, whereas they -should not, to aid detecting bugs in C extensions when the C API is -misused. - -Some function arguments are still cast to ``PyObject*`` to prevent -emitting new compiler warnings. - - -Rationale -========= - -The use of macros may have unintended adverse effects that are hard to -avoid, even for experienced C developers. Some issues have been known -for years, while others have been discovered recently in Python. -Working around macro pitfalls makes the macro coder harder to read and -to maintain. - -Converting macros to functions has multiple advantages: - -* By design, functions don't have macro pitfalls. -* Arguments type and return type are well defined. -* Debuggers and profilers can retrieve the name of inlined functions. -* Debuggers can put breakpoints on inlined functions. -* Variables have a well defined scope. -* Code is usually easier to read and to maintain than similar macro - code. Functions don't need the following workarounds for macro - pitfalls: - - * Add parentheses around arguments. - * Use line continuation characters if the function is written on - multiple lines. - * Add commas to execute multiple expressions. - * Use ``do { ... } while (0)`` to write multiple statements. - -Converting macros and static inline functions to regular functions makes -these regular functions accessible to projects which use Python but -cannot use macros and static inline functions. - - -Macro Pitfalls -============== - -The `GCC documentation -<https://gcc.gnu.org/onlinedocs/cpp/Macro-Pitfalls.html>`_ lists several -common macro pitfalls: - -- Misnesting -- Operator precedence problems -- Swallowing the semicolon -- Duplication of side effects -- Self-referential macros -- Argument prescan -- Newlines in arguments - - -Performance and inlining -======================== - -Static inline functions is a feature added to the C99 standard. Modern C -compilers have efficient heuristics to decide if a function should be -inlined or not. - -When a C compiler decides to not inline, there is likely a good reason. -For example, inlining would reuse a register which require to -save/restore the register value on the stack and so increase the stack -memory usage or be less efficient. - - -Debug build ------------ - -When Python is built in debug mode, most compiler optimizations are -disabled. For example, Visual Studio disables inlining. Benchmarks must -not be run on a Python debug build, only on release build: using LTO and -PGO is recommended for reliable benchmarks. PGO helps the compiler to -decide if function should be inlined or not. - - -Force inlining --------------- - -The ``Py_ALWAYS_INLINE`` macro can be used to force inlining. This macro -uses ``__attribute__((always_inline))`` with GCC and Clang, and -``__forceinline`` with MSC. - -So far, previous attempts to use ``Py_ALWAYS_INLINE`` didn't show any -benefit and were abandoned. See for example: `bpo-45094 -<https://bugs.python.org/issue45094>`_: "Consider using -``__forceinline`` and ``__attribute__((always_inline))`` on static -inline functions (``Py_INCREF``, ``Py_TYPE``) for debug build". - -When the ``Py_INCREF()`` macro was converted to a static inline -functions in 2018 (`commit -<https://github.com/python/cpython/commit/2aaf0c12041bcaadd7f2cc5a54450eefd7a6ff12>`__), -it was decided not to force inlining. The machine code was analyzed with -multiple C compilers and compiler options: ``Py_INCREF()`` was always -inlined without having to force inlining. The only case where it was not -inlined was the debug build. See discussion in the `bpo-35059 -<https://bugs.python.org/issue35059>`_: "Convert ``Py_INCREF()`` and -``PyObject_INIT()`` to inlined functions". - - -Disable inlining ----------------- - -On the other side, the ``Py_NO_INLINE`` macro can be used to disable -inlining. It is useful to reduce the stack memory usage. It is -especially useful on a LTO+PGO build which is more aggressive to inline -code: see `bpo-33720 <https://bugs.python.org/issue33720>`_. The -``Py_NO_INLINE`` macro uses ``__attribute__ ((noinline))`` with GCC and -Clang, and ``__declspec(noinline)`` with MSC. - - -Specification -============= - -Convert macros to static inline functions ------------------------------------------ - -Most macros should be converted to static inline functions to prevent -`macro pitfalls`_. - -The following macros should not be converted: - -* Empty macros. Example: ``#define Py_HAVE_CONDVAR``. -* Macros only defining a number, even if a constant with a well defined - type can better. Example: ``#define METH_VARARGS 0x0001``. -* Compatibility layer for different C compilers, C language extensions, - or recent C features. - Example: ``#define Py_ALWAYS_INLINE __attribute__((always_inline))``. - - -Convert static inline functions to regular functions ----------------------------------------------------- - -The performance impact of converting static inline functions to regular -functions should be measured with benchmarks. If there is a significant -slowdown, there should be a good reason to do the conversion. One reason -can be hiding implementation details. - -Using static inline functions in the internal C API is fine: the -internal C API exposes implemenation details by design and should not be -used outside Python. - -Cast to PyObject* ------------------ - -When a macro is converted to a function and the macro casts its -arguments to ``PyObject*``, the new function comes with a new macro -which cast arguments to ``PyObject*`` to prevent emitting new compiler -warnings. So the converted functions still accept pointers to structures -inheriting from ``PyObject`` (ex: ``PyTupleObject``). - -For example, the ``Py_TYPE(obj)`` macro casts its ``obj`` argument to -``PyObject*``:: - - #define _PyObject_CAST_CONST(op) ((const PyObject*)(op)) - - static inline PyTypeObject* _Py_TYPE(const PyObject *ob) { - return ob->ob_type; - } - #define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST_CONST(ob)) - -The undocumented private ``_Py_TYPE()`` function must not be called -directly. Only the documented public ``Py_TYPE()`` macro must be used. - -Later, the cast can be removed on a case by case basis, but that is out -of scope for this PEP. - -Remove the return value ------------------------ - -When a macro is implemented as an expression, it has an implicit return -value. In some cases, the macro must not have a return value and can be -misused in third party C extensions. See `bpo-30459 -<https://bugs.python.org/issue30459>`_ for the example of -``PyList_SET_ITEM()`` and ``PyCell_SET()`` macros. It is not easy to -notice this issue while reviewing macro code. - -These macros are converted to functions using the ``void`` return type -to remove their return value. Removing the return value aids detecting -bugs in C extensions when the C API is misused. - - -Backwards Compatibility -======================= - -Removing the return value of macros is an incompatible API change made -on purpose: see the `Remove the return value`_ section. - - -Rejected Ideas -============== - -Keep macros, but fix some macro issues --------------------------------------- - -Converting macros to functions is not needed to `remove the return -value`_: casting a macro return value to ``void`` also fix the issue. -For example, the ``PyList_SET_ITEM()`` macro was already fixed like -that. - -Macros are always "inlined" with any C compiler. - -The duplication of side effects can be worked around in the caller of -the macro. - -People using macros should be considered "consenting adults". People who -feel unsafe with macros should simply not use them. - -Examples of hard to read macros -=============================== - -_Py_NewReference() ------------------- - -Example showing the usage of an ``#ifdef`` inside a macro. - -Python 3.7 macro (simplified code):: - - #ifdef COUNT_ALLOCS - # define _Py_INC_TPALLOCS(OP) inc_count(Py_TYPE(OP)) - # define _Py_COUNT_ALLOCS_COMMA , - #else - # define _Py_INC_TPALLOCS(OP) - # define _Py_COUNT_ALLOCS_COMMA - #endif /* COUNT_ALLOCS */ - - #define _Py_NewReference(op) ( \ - _Py_INC_TPALLOCS(op) _Py_COUNT_ALLOCS_COMMA \ - Py_REFCNT(op) = 1) - -Python 3.8 function (simplified code):: - - static inline void _Py_NewReference(PyObject *op) - { - _Py_INC_TPALLOCS(op); - Py_REFCNT(op) = 1; - } - -PyObject_INIT() ---------------- - -Example showing the usage of commas in a macro. - -Python 3.7 macro:: - - #define PyObject_INIT(op, typeobj) \ - ( Py_TYPE(op) = (typeobj), _Py_NewReference((PyObject *)(op)), (op) ) - -Python 3.8 function (simplified code):: - - static inline PyObject* - _PyObject_INIT(PyObject *op, PyTypeObject *typeobj) - { - Py_TYPE(op) = typeobj; - _Py_NewReference(op); - return op; - } - - #define PyObject_INIT(op, typeobj) \ - _PyObject_INIT(_PyObject_CAST(op), (typeobj)) - -The function doesn't need the line continuation character. It has an -explicit ``"return op;"`` rather than a surprising ``", (op)"`` at the -end of the macro. It uses one short statement per line, rather than a -single long line. Inside the function, the *op* argument has a well -defined type: ``PyObject*``. - - -Macros converted to functions since Python 3.8 -============================================== - -Macros converted to static inline functions -------------------------------------------- - -Python 3.8: - -* ``Py_DECREF()`` -* ``Py_INCREF()`` -* ``Py_XDECREF()`` -* ``Py_XINCREF()`` -* ``PyObject_INIT()`` -* ``PyObject_INIT_VAR()`` -* ``_PyObject_GC_UNTRACK()`` -* ``_Py_Dealloc()`` - -Python 3.10: - -* ``Py_REFCNT()`` - -Python 3.11: - -* ``Py_TYPE()`` -* ``Py_SIZE()`` - -Macros converted to regular functions -------------------------------------- - -Python 3.9: - -* ``PyIndex_Check()`` -* ``PyObject_CheckBuffer()`` -* ``PyObject_GET_WEAKREFS_LISTPTR()`` -* ``PyObject_IS_GC()`` -* ``PyObject_NEW()``: alias to ``PyObject_New()`` -* ``PyObject_NEW_VAR()``: alias to ``PyObjectVar_New()`` - -To avoid any risk of performance slowdown on Python built without LTO, -private static inline functions have been added to the internal C API: - -* ``_PyIndex_Check()`` -* ``_PyObject_IS_GC()`` -* ``_PyType_HasFeature()`` -* ``_PyType_IS_GC()`` - -Static inline functions converted to regular functions -------------------------------------------------------- - -Python 3.11: - -* ``PyObject_CallOneArg()`` -* ``PyObject_Vectorcall()`` -* ``PyVectorcall_Function()`` -* ``_PyObject_FastCall()`` - -To avoid any risk of performance slowdown on Python built without LTO, a -private static inline function has been added to the internal C API: - -* ``_PyVectorcall_FunctionInline()`` - - -References -========== - -* `bpo-45490 <https://bugs.python.org/issue45490>`_: - [meta][C API] Avoid C macro pitfalls and usage of static inline - functions (October 2021). -* `What to do with unsafe macros - <https://discuss.python.org/t/what-to-do-with-unsafe-macros/7771>`_ - (March 2021). -* `bpo-43502 <https://bugs.python.org/issue43502>`_: - [C-API] Convert obvious unsafe macros to static inline functions - (March 2021). - - -Copyright -========= - -This document is placed in the public domain or under the -CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-3147-1.png b/pep-3147-1.png deleted file mode 100644 index 930910adf..000000000 Binary files a/pep-3147-1.png and /dev/null differ diff --git a/pep-8103.rst b/pep-8103.rst deleted file mode 100644 index 5700b0b29..000000000 --- a/pep-8103.rst +++ /dev/null @@ -1,166 +0,0 @@ -PEP: 8103 -Title: 2022 Term steering council election -Version: $Revision$ -Last-Modified: $Date$ -Author: Ewa Jodlowska <ewa@python.org>, Ee W. Durbin III <ee@python.org>, Joe Carey <joe@python.org> -Sponsor: Barry Warsaw <barry@python.org> -Status: Draft -Type: Informational -Content-Type: text/x-rst -Created: 04-Oct-2021 - - -Abstract -======== - -This document describes the schedule and other details of the December -2021 election for the Python steering council, as specified in -PEP 13. This is the steering council election for the 2022 term -(i.e. Python 3.11). - - -Election Administration -======================= - -TBD: Determine election administrators - - -Schedule -======== - -There will be a two-week nomination period, followed by a two-week -vote. - -The nomination period shall be: November 1, 2021 through November 16, -2021 12:00 UTC (The end of November 15, 2021 `Anywhere on Earth -<https://www.ieee802.org/16/aoe.html>`_). - -The voting period shall be: December 1, 2021 12:00 UTC through -December 16, 2021 12:00 UTC (The end of December 15, 2021 `Anywhere on -Earth <https://www.ieee802.org/16/aoe.html>`_). - - -Candidates -========== - -Candidates must be nominated by a core team member. If the candidate -is a core team member, they may nominate themselves. - -Nominees (in alphabetical order): - -- TBD - -Withdrawn nominations: - -- TBD - - -Voter Roll -========== - -All active Python core team members are eligible to vote. Active status -is determined as described in `PEP 13 <https://www.python.org/dev/peps/pep-0013/#membership>`_ -and implemented via the software at `python/voters <https://github.com/python/voters>`_ [1]_. - -Ballots will be distributed based on the `The Python Voter Roll for this -election -<https://github.com/python/voters/blob/master/voter-files/>`_ -[1]_. - -While this file is not public as it contains private email addresses, the -`Complete Voter Roll`_ by name will be made available when the roll is -created. - - -Election Implementation -======================= - -The election will be conducted using the `Helios Voting Service -<https://heliosvoting.org>`__. - - -Configuration -------------- - -.. note:: - - These details are subject to change. - - -Short name: ``2022-python-steering-council`` - -Name: ``2022 Python Steering Council Election`` - -Description: ``Election for the Python steering council, as specified -in PEP 13. This is steering council election for the 2022 term.`` - -type: ``Election`` - -Use voter aliases: ``[X]`` - -Randomize answer order: ``[X]`` - -Private: ``[X]`` - -Help Email Address: ``psf-election@python.org`` - -Voting starts at: ``December 1, 2021 00:00 UTC`` - -Voting ends at: ``December 16, 2021 12:00 UTC`` - -This will create an election in which: - -* Voting is not open to the public, only those on the `Voter Roll`_ may - participate. Ballots will be emailed when voting starts. -* Candidates are presented in random order, to help avoid bias. -* Voter identities and ballots are protected against cryptographic advances. - -Questions ---------- - -Question 1 -~~~~~~~~~~ - -Select between ``0`` and ``- (approval)`` answers. Result Type: ``absolute`` - -Question: ``Select candidates for the Python Steering Council`` - -Answer #1 - #N: ``Candidates from Candidates_ Section`` - - - -Results -======= - -- TBD - - -Copyright -========= - -This document has been placed in the public domain. - - -Complete Voter Roll -=================== - -Active Python core developers ------------------------------ - -- TBD - - -.. [1] This repository is private and accessible only to Python Core - Developers, administrators, and Python Software Foundation Staff as it - contains personal email addresses. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep.css b/pep.css deleted file mode 100644 index d75dff1d8..000000000 --- a/pep.css +++ /dev/null @@ -1,344 +0,0 @@ -/* -:Author: David Goodger -:Contact: goodger@python.org -:date: $Date$ -:version: $Revision$ -:copyright: This stylesheet has been placed in the public domain. - -Default cascading style sheet for the PEP HTML output of Docutils. -*/ - -/* "! important" is used here to override other ``margin-top`` and - ``margin-bottom`` styles that are later in the stylesheet or - more specific. See http://www.w3.org/TR/CSS1#the-cascade */ -.first { - margin-top: 0 ! important } - -.last, .with-subtitle { - margin-bottom: 0 ! important } - -.hidden { - display: none } - -.navigation { - width: 100% ; - background: #99ccff ; - margin-top: 0px ; - margin-bottom: 0px } - -.navigation .navicon { - width: 150px ; - height: 35px } - -.navigation .textlinks { - padding-left: 1em ; - text-align: left } - -.navigation td, .navigation th { - padding-left: 0em ; - padding-right: 0em ; - vertical-align: middle } - -.rfc2822 { - margin-top: 0.5em ; - margin-left: 0.5em ; - margin-right: 0.5em ; - margin-bottom: 0em } - -.rfc2822 td { - text-align: left } - -.rfc2822 th.field-name { - text-align: right ; - font-family: sans-serif ; - padding-right: 0.5em ; - font-weight: bold ; - margin-bottom: 0em } - -a.toc-backref { - text-decoration: none ; - color: black } - -blockquote.epigraph { - margin: 2em 5em ; } - -body { - margin: 0px ; - margin-bottom: 1em ; - padding: 0px } - -dl.docutils dd { - margin-bottom: 0.5em } - -div.section { - margin-left: 1em ; - margin-right: 1em ; - margin-bottom: 1.5em } - -div.section div.section { - margin-left: 0em ; - margin-right: 0em ; - margin-top: 1.5em } - -div.abstract { - margin: 2em 5em } - -div.abstract p.topic-title { - font-weight: bold ; - text-align: center } - -div.admonition, div.attention, div.caution, div.danger, div.error, -div.hint, div.important, div.note, div.tip, div.warning { - margin: 2em ; - border: medium outset ; - padding: 1em } - -div.admonition p.admonition-title, div.hint p.admonition-title, -div.important p.admonition-title, div.note p.admonition-title, -div.tip p.admonition-title { - font-weight: bold ; - font-family: sans-serif } - -div.attention p.admonition-title, div.caution p.admonition-title, -div.danger p.admonition-title, div.error p.admonition-title, -div.warning p.admonition-title { - color: red ; - font-weight: bold ; - font-family: sans-serif } - -/* Uncomment (and remove this text!) to get reduced vertical space in - compound paragraphs. -div.compound .compound-first, div.compound .compound-middle { - margin-bottom: 0.5em } - -div.compound .compound-last, div.compound .compound-middle { - margin-top: 0.5em } -*/ - -div.dedication { - margin: 2em 5em ; - text-align: center ; - font-style: italic } - -div.dedication p.topic-title { - font-weight: bold ; - font-style: normal } - -div.figure { - margin-left: 2em ; - margin-right: 2em } - -div.footer, div.header { - clear: both; - font-size: smaller } - -div.footer { - margin-left: 1em ; - margin-right: 1em } - -div.line-block { - display: block ; - margin-top: 1em ; - margin-bottom: 1em } - -div.line-block div.line-block { - margin-top: 0 ; - margin-bottom: 0 ; - margin-left: 1.5em } - -div.sidebar { - margin-left: 1em ; - border: medium outset ; - padding: 1em ; - background-color: #ffffee ; - width: 40% ; - float: right ; - clear: right } - -div.sidebar p.rubric { - font-family: sans-serif ; - font-size: medium } - -div.system-messages { - margin: 5em } - -div.system-messages h1 { - color: red } - -div.system-message { - border: medium outset ; - padding: 1em } - -div.system-message p.system-message-title { - color: red ; - font-weight: bold } - -div.topic { - margin: 2em } - -h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, -h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { - margin-top: 0.4em } - -h1 { - font-family: sans-serif ; - font-size: large } - -h2 { - font-family: sans-serif ; - font-size: medium } - -h3 { - font-family: sans-serif ; - font-size: small } - -h4 { - font-family: sans-serif ; - font-style: italic ; - font-size: small } - -h5 { - font-family: sans-serif; - font-size: x-small } - -h6 { - font-family: sans-serif; - font-style: italic ; - font-size: x-small } - -hr.docutils { - width: 75% } - -img.align-left { - clear: left } - -img.align-right { - clear: right } - -img.borderless { - border: 0 } - -ol.simple, ul.simple { - margin-bottom: 1em } - -ol.arabic { - list-style: decimal } - -ol.loweralpha { - list-style: lower-alpha } - -ol.upperalpha { - list-style: upper-alpha } - -ol.lowerroman { - list-style: lower-roman } - -ol.upperroman { - list-style: upper-roman } - -p.attribution { - text-align: right ; - margin-left: 50% } - -p.caption { - font-style: italic } - -p.credits { - font-style: italic ; - font-size: smaller } - -p.label { - white-space: nowrap } - -p.rubric { - font-weight: bold ; - font-size: larger ; - color: maroon ; - text-align: center } - -p.sidebar-title { - font-family: sans-serif ; - font-weight: bold ; - font-size: larger } - -p.sidebar-subtitle { - font-family: sans-serif ; - font-weight: bold } - -p.topic-title { - font-family: sans-serif ; - font-weight: bold } - -pre.address { - margin-bottom: 0 ; - margin-top: 0 ; - font-family: serif ; - font-size: 100% } - -pre.literal-block, pre.doctest-block { - margin-left: 2em ; - margin-right: 2em } - -span.classifier { - font-family: sans-serif ; - font-style: oblique } - -span.classifier-delimiter { - font-family: sans-serif ; - font-weight: bold } - -span.interpreted { - font-family: sans-serif } - -span.option { - white-space: nowrap } - -span.option-argument { - font-style: italic } - -span.pre { - white-space: pre } - -span.problematic { - color: red } - -span.section-subtitle { - /* font-size relative to parent (h1..h6 element) */ - font-size: 80% } - -table.citation { - border-left: solid 1px gray; - margin-left: 1px } - -table.docinfo { - margin: 2em 4em } - -table.docutils { - margin-top: 0.5em ; - margin-bottom: 0.5em } - -table.footnote { - border-left: solid 1px black; - margin-left: 1px } - -table.docutils td, table.docutils th, -table.docinfo td, table.docinfo th { - padding-left: 0.5em ; - padding-right: 0.5em ; - vertical-align: top } - -td.num { - text-align: right } - -th.field-name { - font-weight: bold ; - text-align: left ; - white-space: nowrap ; - padding-left: 0 } - -h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, -h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { - font-size: 100% } - -ul.auto-toc { - list-style-type: none } diff --git a/pep0/__init__.py b/pep0/__init__.py deleted file mode 100644 index b7db25411..000000000 --- a/pep0/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Empty diff --git a/pep0/constants.py b/pep0/constants.py deleted file mode 100644 index 108710d09..000000000 --- a/pep0/constants.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -text_type = str -title_length = 55 -author_length = 40 -table_separator = "== ==== " + "="*title_length + " " + "="*author_length -column_format = ( - '%(type)1s%(status)1s %(number)4s %(title)-{title_length}s %(authors)-s' -).format(title_length=title_length) - -header = """\ -PEP: 0 -Title: Index of Python Enhancement Proposals (PEPs) -Version: N/A -Last-Modified: %s -Author: python-dev <python-dev@python.org> -Status: Active -Type: Informational -Content-Type: text/x-rst -Created: 13-Jul-2000 -""" - -intro = """\ -This PEP contains the index of all Python Enhancement Proposals, -known as PEPs. PEP numbers are assigned by the PEP editors[1_], and -once assigned are never changed. The version control history [2_] of -the PEP texts represent their historical record. -""" - -references = """\ -.. [1] PEP 1: PEP Purpose and Guidelines -.. [2] View PEP history online: https://github.com/python/peps -""" - -footer = """ \ -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End:\ -""" diff --git a/pep0/output.py b/pep0/output.py deleted file mode 100644 index 10024c221..000000000 --- a/pep0/output.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Code to handle the output of PEP 0.""" -from __future__ import absolute_import -from __future__ import print_function -import datetime -import sys -import unicodedata - -from operator import attrgetter - -from . import constants -from .pep import PEP, PEPError - -# This is a list of reserved PEP numbers. Reservations are not to be used for -# the normal PEP number allocation process - just give out the next available -# PEP number. These are for "special" numbers that may be used for semantic, -# humorous, or other such reasons, e.g. 401, 666, 754. -# -# PEP numbers may only be reserved with the approval of a PEP editor. Fields -# here are the PEP number being reserved and the claimants for the PEP. -# Although the output is sorted when PEP 0 is generated, please keep this list -# sorted as well. -RESERVED = [ - (801, 'Warsaw'), - ] - - -indent = u' ' - -def emit_column_headers(output): - """Output the column headers for the PEP indices.""" - column_headers = {'status': '.', 'type': '.', 'number': 'PEP', - 'title': 'PEP Title', 'authors': 'PEP Author(s)'} - print(constants.table_separator, file=output) - print(constants.column_format % column_headers, file=output) - print(constants.table_separator, file=output) - - -def sort_peps(peps): - """Sort PEPs into meta, informational, accepted, open, finished, - and essentially dead.""" - meta = [] - info = [] - provisional = [] - accepted = [] - open_ = [] - finished = [] - historical = [] - deferred = [] - dead = [] - for pep in peps: - # Order of 'if' statement important. Key Status values take precedence - # over Type value, and vice-versa. - if pep.status == 'Draft': - open_.append(pep) - elif pep.status == 'Deferred': - deferred.append(pep) - elif pep.type_ == 'Process': - if pep.status == "Active": - meta.append(pep) - elif pep.status in ("Withdrawn", "Rejected"): - dead.append(pep) - else: - historical.append(pep) - elif pep.status in ('Rejected', 'Withdrawn', - 'Incomplete', 'Superseded'): - dead.append(pep) - elif pep.type_ == 'Informational': - # Hack until the conflict between the use of "Final" - # for both API definition PEPs and other (actually - # obsolete) PEPs is addressed - if (pep.status == "Active" or - "Release Schedule" not in pep.title): - info.append(pep) - else: - historical.append(pep) - elif pep.status == 'Provisional': - provisional.append(pep) - elif pep.status in ('Accepted', 'Active'): - accepted.append(pep) - elif pep.status == 'Final': - finished.append(pep) - else: - raise PEPError("unsorted (%s/%s)" % - (pep.type_, pep.status), - pep.filename, pep.number) - return (meta, info, provisional, accepted, open_, - finished, historical, deferred, dead) - - -def verify_email_addresses(peps): - authors_dict = {} - for pep in peps: - for author in pep.authors: - # If this is the first time we have come across an author, add them. - if author not in authors_dict: - authors_dict[author] = [author.email] - else: - found_emails = authors_dict[author] - # If no email exists for the author, use the new value. - if not found_emails[0]: - authors_dict[author] = [author.email] - # If the new email is an empty string, move on. - elif not author.email: - continue - # If the email has not been seen, add it to the list. - elif author.email not in found_emails: - authors_dict[author].append(author.email) - - valid_authors_dict = {} - too_many_emails = [] - for author, emails in authors_dict.items(): - if len(emails) > 1: - too_many_emails.append((author.first_last, emails)) - else: - valid_authors_dict[author] = emails[0] - if too_many_emails: - err_output = [] - for author, emails in too_many_emails: - err_output.append(" %s: %r" % (author, emails)) - raise ValueError("some authors have more than one email address " - "listed:\n" + '\n'.join(err_output)) - - return valid_authors_dict - - -def sort_authors(authors_dict): - authors_list = list(authors_dict.keys()) - authors_list.sort(key=attrgetter('sort_by')) - return authors_list - -def normalized_last_first(name): - return len(unicodedata.normalize('NFC', name.last_first)) - -def emit_title(text, anchor, output, *, symbol="="): - print(".. _{anchor}:\n".format(anchor=anchor), file=output) - print(text, file=output) - print(symbol*len(text), file=output) - print(file=output) - -def emit_subtitle(text, anchor, output): - emit_title(text, anchor, output, symbol="-") - -def emit_pep_category(output, category, anchor, peps): - emit_subtitle(category, anchor, output) - emit_column_headers(output) - for pep in peps: - print(pep, file=output) - print(constants.table_separator, file=output) - print(file=output) - -def write_pep0(peps, output=sys.stdout): - # PEP metadata - today = datetime.date.today().strftime("%Y-%m-%d") - print(constants.header % today, file=output) - print(file=output) - # Introduction - emit_title("Introduction", "intro", output) - print(constants.intro, file=output) - print(file=output) - # PEPs by category - (meta, info, provisional, accepted, open_, - finished, historical, deferred, dead) = sort_peps(peps) - emit_title("Index by Category", "by-category", output) - emit_pep_category( - category="Meta-PEPs (PEPs about PEPs or Processes)", - anchor="by-category-meta", - peps=meta, - output=output, - ) - emit_pep_category( - category="Other Informational PEPs", - anchor="by-category-other-info", - peps=info, - output=output, - ) - emit_pep_category( - category="Provisional PEPs (provisionally accepted; interface may still change)", - anchor="by-category-provisional", - peps=provisional, - output=output, - ) - emit_pep_category( - category="Accepted PEPs (accepted; may not be implemented yet)", - anchor="by-category-accepted", - peps=accepted, - output=output, - ) - emit_pep_category( - category="Open PEPs (under consideration)", - anchor="by-category-open", - peps=open_, - output=output, - ) - emit_pep_category( - category="Finished PEPs (done, with a stable interface)", - anchor="by-category-finished", - peps=finished, - output=output, - ) - emit_pep_category( - category="Historical Meta-PEPs and Informational PEPs", - anchor="by-category-historical", - peps=historical, - output=output, - ) - emit_pep_category( - category="Deferred PEPs (postponed pending further research or updates)", - anchor="by-category-deferred", - peps=deferred, - output=output, - ) - emit_pep_category( - category="Abandoned, Withdrawn, and Rejected PEPs", - anchor="by-category-abandoned", - peps=dead, - output=output, - ) - print(file=output) - # PEPs by number - emit_title("Numerical Index", "by-pep-number", output) - emit_column_headers(output) - prev_pep = 0 - for pep in peps: - if pep.number - prev_pep > 1: - print(file=output) - print(constants.text_type(pep), file=output) - prev_pep = pep.number - print(constants.table_separator, file=output) - print(file=output) - # Reserved PEP numbers - emit_title('Reserved PEP Numbers', "reserved", output) - emit_column_headers(output) - for number, claimants in sorted(RESERVED): - print(constants.column_format % { - 'type': '.', - 'status': '.', - 'number': number, - 'title': 'RESERVED', - 'authors': claimants, - }, file=output) - print(constants.table_separator, file=output) - print(file=output) - # PEP types key - emit_title("PEP Types Key", "type-key", output) - for type_ in sorted(PEP.type_values): - print(u" %s - %s PEP" % (type_[0], type_), file=output) - print(file=output) - print(file=output) - # PEP status key - emit_title("PEP Status Key", "status-key", output) - for status in sorted(PEP.status_values): - # Draft PEPs have no status displayed, Active shares a key with Accepted - if status in ("Active", "Draft"): - continue - if status == "Accepted": - msg = " A - Accepted (Standards Track only) or Active proposal" - else: - msg = " {status[0]} - {status} proposal".format(status=status) - print(msg, file=output) - print(file=output) - - print(file=output) - # PEP owners - emit_title("Authors/Owners", "authors", output) - authors_dict = verify_email_addresses(peps) - max_name = max(authors_dict.keys(), key=normalized_last_first) - max_name_len = len(max_name.last_first) - author_table_separator = "="*max_name_len + " " + "="*len("email address") - print(author_table_separator, file=output) - _author_header_fmt = "{name:{max_name_len}} Email Address" - print(_author_header_fmt.format(name="Name", max_name_len=max_name_len), file=output) - print(author_table_separator, file=output) - sorted_authors = sort_authors(authors_dict) - _author_fmt = "{author.last_first:{max_name_len}} {author_email}" - for author in sorted_authors: - # Use the email from authors_dict instead of the one from 'author' as - # the author instance may have an empty email. - _entry = _author_fmt.format( - author=author, - author_email=authors_dict[author], - max_name_len=max_name_len, - ) - print(_entry, file=output) - print(author_table_separator, file=output) - print(file=output) - print(file=output) - # References for introduction footnotes - emit_title("References", "references", output) - print(constants.references, file=output) - print(constants.footer, file=output) diff --git a/pep0/pep.py b/pep0/pep.py deleted file mode 100644 index 1a4d83213..000000000 --- a/pep0/pep.py +++ /dev/null @@ -1,316 +0,0 @@ -# -*- coding: utf-8 -*- -"""Code for handling object representation of a PEP.""" -from __future__ import absolute_import -import re -import sys -import textwrap -import unicodedata - -from email.parser import HeaderParser - -from . import constants - - -class PEPError(Exception): - - def __init__(self, error, pep_file, pep_number=None): - super(PEPError, self).__init__(error) - self.filename = pep_file - self.number = pep_number - - def __str__(self): - error_msg = super(PEPError, self).__str__() - if self.number is not None: - return "PEP %d: %r" % (self.number, error_msg) - else: - return "(%s): %r" % (self.filename, error_msg) - - -class PEPParseError(PEPError): - - pass - - -class Author(object): - - """Represent PEP authors. - - Attributes: - - + first_last : str - The author's full name. - - + last_first : str - Output the author's name in Last, First, Suffix order. - - + first : str - The author's first name. A middle initial may be included. - - + last : str - The author's last name. - - + suffix : str - A person's suffix (can be the empty string). - - + sort_by : str - Modification of the author's last name that should be used for - sorting. - - + email : str - The author's email address. - """ - - def __init__(self, author_and_email_tuple): - """Parse the name and email address of an author.""" - name, email = author_and_email_tuple - self.first_last = name.strip() - self.email = email.lower() - last_name_fragment, suffix = self._last_name(name) - name_sep = name.index(last_name_fragment) - self.first = name[:name_sep].rstrip() - self.last = last_name_fragment - if self.last[1] == u'.': - # Add an escape to avoid docutils turning `v.` into `22.`. - self.last = u'\\' + self.last - self.suffix = suffix - if not self.first: - self.last_first = self.last - else: - self.last_first = u', '.join([self.last, self.first]) - if self.suffix: - self.last_first += u', ' + self.suffix - if self.last == "van Rossum": - # Special case for our beloved BDFL. :) - if self.first == "Guido": - self.nick = "GvR" - elif self.first == "Just": - self.nick = "JvR" - else: - raise ValueError("unknown van Rossum %r!" % self) - self.last_first += " (%s)" % (self.nick,) - else: - self.nick = self.last - - def __hash__(self): - return hash(self.first_last) - - def __eq__(self, other): - return self.first_last == other.first_last - - @property - def sort_by(self): - name_parts = self.last.split() - for index, part in enumerate(name_parts): - if part[0].isupper(): - base = u' '.join(name_parts[index:]).lower() - break - else: - # If no capitals, use the whole string - base = self.last.lower() - return unicodedata.normalize('NFKD', base).encode('ASCII', 'ignore') - - def _last_name(self, full_name): - """Find the last name (or nickname) of a full name. - - If no last name (e.g, 'Aahz') then return the full name. If there is - a leading, lowercase portion to the last name (e.g., 'van' or 'von') - then include it. If there is a suffix (e.g., 'Jr.') that is appended - through a comma, then drop the suffix. - - """ - name_partition = full_name.partition(u',') - no_suffix = name_partition[0].strip() - suffix = name_partition[2].strip() - name_parts = no_suffix.split() - part_count = len(name_parts) - if part_count == 1 or part_count == 2: - return name_parts[-1], suffix - else: - assert part_count > 2 - if name_parts[-2].islower(): - return u' '.join(name_parts[-2:]), suffix - else: - return name_parts[-1], suffix - - -class PEP(object): - - """Representation of PEPs. - - Attributes: - - + number : int - PEP number. - - + title : str - PEP title. - - + type_ : str - The type of PEP. Can only be one of the values from - PEP.type_values. - - + status : str - The PEP's status. Value must be found in PEP.status_values. - - + authors : Sequence(Author) - A list of the authors. - """ - - # The various RFC 822 headers that are supported. - # The second item in the nested tuples represents if the header is - # required or not. - headers = (('PEP', True), ('Title', True), ('Version', False), - ('Last-Modified', False), ('Author', True), - ('Sponsor', False), ('BDFL-Delegate', False), - ('PEP-Delegate', False), - ('Discussions-To', False), ('Status', True), ('Type', True), - ('Content-Type', False), ('Requires', False), - ('Created', True), ('Python-Version', False), - ('Post-History', False), ('Replaces', False), - ('Superseded-By', False), ('Resolution', False), - ) - # Valid values for the Type header. - type_values = (u"Standards Track", u"Informational", u"Process") - # Valid values for the Status header. - # Active PEPs can only be for Informational or Process PEPs. - status_values = (u"Accepted", u"Provisional", - u"Rejected", u"Withdrawn", u"Deferred", - u"Final", u"Active", u"Draft", u"Superseded") - - def __init__(self, pep_file): - """Init object from an open PEP file object.""" - # Parse the headers. - self.filename = pep_file - pep_parser = HeaderParser() - metadata = pep_parser.parse(pep_file) - header_order = iter(self.headers) - try: - for header_name in metadata.keys(): - current_header, required = next(header_order) - while header_name != current_header and not required: - current_header, required = next(header_order) - if header_name != current_header: - raise PEPError("did not deal with " - "%r before having to handle %r" % - (header_name, current_header), - pep_file.name) - except StopIteration: - raise PEPError("headers missing or out of order", - pep_file.name) - required = False - try: - while not required: - current_header, required = next(header_order) - else: - raise PEPError("PEP is missing its %r" % (current_header,), - pep_file.name) - except StopIteration: - pass - # 'PEP'. - try: - self.number = int(metadata['PEP']) - except ValueError: - raise PEPParseError("PEP number isn't an integer", pep_file.name) - # 'Title'. - self.title = metadata['Title'] - # 'Type'. - type_ = metadata['Type'] - if type_ not in self.type_values: - raise PEPError('%r is not a valid Type value' % (type_,), - pep_file.name, self.number) - self.type_ = type_ - # 'Status'. - status = metadata['Status'] - if status not in self.status_values: - if status == "April Fool!": - # See PEP 401 :) - status = "Rejected" - else: - raise PEPError("%r is not a valid Status value" % - (status,), pep_file.name, self.number) - # Special case for Active PEPs. - if (status == u"Active" and - self.type_ not in ("Process", "Informational")): - raise PEPError("Only Process and Informational PEPs may " - "have an Active status", pep_file.name, - self.number) - # Special case for Provisional PEPs. - if (status == u"Provisional" and self.type_ != "Standards Track"): - raise PEPError("Only Standards Track PEPs may " - "have a Provisional status", pep_file.name, - self.number) - self.status = status - # 'Author'. - authors_and_emails = self._parse_author(metadata['Author']) - if len(authors_and_emails) < 1: - raise PEPError("no authors found", pep_file.name, - self.number) - self.authors = list(map(Author, authors_and_emails)) - - def _parse_author(self, data): - """Return a list of author names and emails.""" - # XXX Consider using email.utils.parseaddr (doesn't work with names - # lacking an email address. - angled = constants.text_type(r'(?P<author>.+?) <(?P<email>.+?)>') - paren = constants.text_type(r'(?P<email>.+?) \((?P<author>.+?)\)') - simple = constants.text_type(r'(?P<author>[^,]+)') - author_list = [] - for regex in (angled, paren, simple): - # Watch out for commas separating multiple names. - regex += r'(,\s*)?' - for match in re.finditer(regex, data): - # Watch out for suffixes like 'Jr.' when they are comma-separated - # from the name and thus cause issues when *all* names are only - # separated by commas. - match_dict = match.groupdict() - author = match_dict['author'] - if not author.partition(' ')[1] and author.endswith('.'): - prev_author = author_list.pop() - author = ', '.join([prev_author, author]) - if u'email' not in match_dict: - email = '' - else: - email = match_dict['email'] - author_list.append((author, email)) - else: - # If authors were found then stop searching as only expect one - # style of author citation. - if author_list: - break - return author_list - - @property - def type_abbr(self): - """Return the how the type is to be represented in the index.""" - return self.type_[0].upper() - - @property - def status_abbr(self): - """Return how the status should be represented in the index.""" - if self.status in ('Draft', 'Active'): - return u' ' - else: - return self.status[0].upper() - - @property - def author_abbr(self): - """Return the author list as a comma-separated with only last names.""" - return u', '.join(x.nick for x in self.authors) - - @property - def title_abbr(self): - """Shorten the title to be no longer than the max title length.""" - if len(self.title) <= constants.title_length: - return self.title - wrapped_title = textwrap.wrap(self.title, constants.title_length - 4) - return wrapped_title[0] + u' ...' - - def __unicode__(self): - """Return the line entry for the PEP.""" - pep_info = {'type': self.type_abbr, 'number': str(self.number), - 'title': self.title_abbr, 'status': self.status_abbr, - 'authors': self.author_abbr} - return constants.column_format % pep_info - - if sys.version_info[0] > 2: - __str__ = __unicode__ diff --git a/pep2html.py b/pep2html.py deleted file mode 100755 index 9a2b666f0..000000000 --- a/pep2html.py +++ /dev/null @@ -1,710 +0,0 @@ -#!/usr/bin/env python3.9 -"""Convert PEPs to (X)HTML - courtesy of /F - -Usage: %(PROGRAM)s [options] [<peps> ...] - -Options: - --u, --user - python.org username - --b, --browse - After generating the HTML, direct your web browser to view it - (using the Python webbrowser module). If both -i and -b are - given, this will browse the on-line HTML; otherwise it will - browse the local HTML. If no pep arguments are given, this - will browse PEP 0. - --i, --install - After generating the HTML, install it and the plaintext source file - (.txt) on python.org. In that case the user's name is used in the scp - and ssh commands, unless "-u username" is given (in which case, it is - used instead). Without -i, -u is ignored. - --l, --local - Same as -i/--install, except install on the local machine. Use this - when logged in to the python.org machine (dinsdale). - --q, --quiet - Turn off verbose messages. - --h, --help - Print this help message and exit. - -The optional arguments ``peps`` are either pep numbers, .rst or .txt files. -""" - -from __future__ import print_function, unicode_literals - -import sys -import os -import re -import glob -import getopt -import errno -import random -import time -from io import open -try: - from html import escape -except ImportError: - from cgi import escape - -from docutils import core, nodes, utils -from docutils.readers import standalone -from docutils.transforms import peps, frontmatter, Transform -from docutils.parsers import rst - -class DataError(Exception): - pass - -REQUIRES = {'python': '2.6', - 'docutils': '0.2.7'} -PROGRAM = sys.argv[0] -RFCURL = 'http://www.faqs.org/rfcs/rfc%d.html' -PEPURL = 'pep-%04d.html' -PEPCVSURL = ('https://hg.python.org/peps/file/tip/pep-%04d.txt') -PEPDIRRUL = 'http://www.python.org/peps/' - - -HOST = "dinsdale.python.org" # host for update -HDIR = "/data/ftp.python.org/pub/www.python.org/peps" # target host directory -LOCALVARS = "Local Variables:" - -COMMENT = """<!-- -This HTML is auto-generated. DO NOT EDIT THIS FILE! If you are writing a new -PEP, see http://www.python.org/peps/pep-0001.html for instructions and links -to templates. DO NOT USE THIS HTML FILE AS YOUR TEMPLATE! --->""" - -# The generated HTML doesn't validate -- you cannot use <hr> and <h3> inside -# <pre> tags. But if I change that, the result doesn't look very nice... -DTD = ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"\n' - ' "http://www.w3.org/TR/REC-html40/loose.dtd">') - -fixpat = re.compile(r"((https?|ftp):[-_a-zA-Z0-9/.+~:?#$=&,]+)|(pep-\d+(.txt|.rst)?)|" - r"(RFC[- ]?(?P<rfcnum>\d+))|" - r"(PEP\s+(?P<pepnum>\d+))|" - r".") - -EMPTYSTRING = '' -SPACE = ' ' -COMMASPACE = ', ' - - - -def usage(code, msg=''): - """Print usage message and exit. Uses stderr if code != 0.""" - if code == 0: - out = sys.stdout - else: - out = sys.stderr - print(__doc__ % globals(), file=out) - if msg: - print(msg, file=out) - sys.exit(code) - - - -def fixanchor(current, match): - text = match.group(0) - link = None - if (text.startswith('http:') or text.startswith('https:') - or text.startswith('ftp:')): - # Strip off trailing punctuation. Pattern taken from faqwiz. - ltext = list(text) - while ltext: - c = ltext.pop() - if c not in '''();:,.?'"<>''': - ltext.append(c) - break - link = EMPTYSTRING.join(ltext) - elif text.startswith('pep-') and text != current: - link = os.path.splitext(text)[0] + ".html" - elif text.startswith('PEP'): - pepnum = int(match.group('pepnum')) - link = PEPURL % pepnum - elif text.startswith('RFC'): - rfcnum = int(match.group('rfcnum')) - link = RFCURL % rfcnum - if link: - return '<a href="%s">%s</a>' % (escape(link), escape(text)) - return escape(match.group(0)) # really slow, but it works... - - - -NON_MASKED_EMAILS = [ - 'peps@python.org', - 'python-list@python.org', - 'python-dev@python.org', - ] - -def fixemail(address, pepno): - if address.lower() in NON_MASKED_EMAILS: - # return hyperlinked version of email address - return linkemail(address, pepno) - else: - # return masked version of email address - parts = address.split('@', 1) - return '%s at %s' % (parts[0], parts[1]) - - -def linkemail(address, pepno): - parts = address.split('@', 1) - return ('<a href="mailto:%s@%s?subject=PEP%%20%s">' - '%s at %s</a>' - % (parts[0], parts[1], pepno, parts[0], parts[1])) - - -def fixfile(inpath, input_lines, outfile): - try: - from email.Utils import parseaddr - except ImportError: - from email.utils import parseaddr - basename = os.path.basename(inpath) - infile = iter(input_lines) - # convert plaintext pep to minimal XHTML markup - print(DTD, file=outfile) - print('<html>', file=outfile) - print(COMMENT, file=outfile) - print('<head>', file=outfile) - # head - header = [] - pep = "" - title = "" - for line in infile: - if not line.strip(): - break - if line[0].strip(): - if ":" not in line: - break - key, value = line.split(":", 1) - value = value.strip() - header.append((key, value)) - else: - # continuation line - key, value = header[-1] - value = value + line - header[-1] = key, value - if key.lower() == "title": - title = value - elif key.lower() == "pep": - pep = value - if pep: - title = "PEP " + pep + " -- " + title - if title: - print(' <title>%s' % escape(title), file=outfile) - r = random.choice(list(range(64))) - print(( - ' \n' - '\n' - '\n' - '\n' - '\n' - '
\n' - '\n' - '[Python]\n' - '[Python Home]' % r), file=outfile) - if basename != 'pep-0000.txt': - print('[PEP Index]', file=outfile) - if pep: - try: - print(('[PEP Source' - ']' % int(pep)), file=outfile) - except ValueError as error: - print(('ValueError (invalid PEP number): %s' - % error), file=sys.stderr) - print('
', file=outfile) - print('
\n', file=outfile) - for k, v in header: - if k.lower() in ('author', 'pep-delegate', 'bdfl-delegate', 'discussions-to', - 'sponsor'): - mailtos = [] - for part in re.split(r',\s*', v): - if '@' in part: - realname, addr = parseaddr(part) - if k.lower() == 'discussions-to': - m = linkemail(addr, pep) - else: - m = fixemail(addr, pep) - mailtos.append('%s <%s>' % (realname, m)) - elif part.startswith('http:'): - mailtos.append( - '%s' % (part, part)) - else: - mailtos.append(part) - v = COMMASPACE.join(mailtos) - elif k.lower() in ('replaces', 'superseded-by', 'requires'): - otherpeps = '' - for otherpep in re.split(r',?\s+', v): - otherpep = int(otherpep) - otherpeps += '%i ' % (otherpep, - otherpep) - v = otherpeps - elif k.lower() in ('last-modified',): - date = v or time.strftime('%d-%b-%Y', - time.localtime(os.stat(inpath)[8])) - if date.startswith('$' 'Date: ') and date.endswith(' $'): - date = date[6:-2] - if basename == 'pep-0000.txt': - v = date - else: - try: - url = PEPCVSURL % int(pep) - v = '%s ' % (url, escape(date)) - except ValueError as error: - v = date - elif k.lower() in ('content-type',): - url = PEPURL % 9 - pep_type = v or 'text/plain' - v = '%s ' % (url, escape(pep_type)) - elif k.lower() == 'version': - if v.startswith('$' 'Revision: ') and v.endswith(' $'): - v = escape(v[11:-2]) - else: - v = escape(v) - print(' ' \ - % (escape(k), v), file=outfile) - print('
%s: %s
', file=outfile) - print('
', file=outfile) - print('
', file=outfile) - print('
', file=outfile) - need_pre = 1 - for line in infile: - if line[0] == '\f': - continue - if line.strip() == LOCALVARS: - break - if line[0].strip(): - if not need_pre: - print('', file=outfile) - print('

%s

' % line.strip(), file=outfile) - need_pre = 1 - elif not line.strip() and need_pre: - continue - else: - # PEP 0 has some special treatment - if basename == 'pep-0000.txt': - parts = line.split() - if len(parts) > 1 and re.match(r'\s*\d{1,4}', parts[1]): - # This is a PEP summary line, which we need to hyperlink - url = PEPURL % int(parts[1]) - if need_pre: - print('
', file=outfile)
-                        need_pre = 0
-                    print(re.sub(
-                        parts[1],
-                        '%s' % (url, parts[1]),
-                        line, 1), end='', file=outfile)
-                    continue
-                elif parts and '@' in parts[-1]:
-                    # This is a pep email address line, so filter it.
-                    url = fixemail(parts[-1], pep)
-                    if need_pre:
-                        print('
', file=outfile)
-                        need_pre = 0
-                    print(re.sub(
-                        parts[-1], url, line, 1), end='', file=outfile)
-                    continue
-            line = fixpat.sub(lambda x, c=inpath: fixanchor(c, x), line)
-            if need_pre:
-                print('
', file=outfile)
-                need_pre = 0
-            outfile.write(line)
-    if not need_pre:
-        print('
', file=outfile)
-    print('
', file=outfile) - print('', file=outfile) - print('', file=outfile) - - -docutils_settings = None -"""Runtime settings object used by Docutils. Can be set by the client -application when this module is imported.""" - -class PEPHeaders(Transform): - - """ - Process fields in a PEP's initial RFC-2822 header. - """ - - default_priority = 360 - - pep_url = 'pep-%04d' - pep_cvs_url = PEPCVSURL - rcs_keyword_substitutions = ( - (re.compile(r'\$' r'RCSfile: (.+),v \$$', re.IGNORECASE), r'\1'), - (re.compile(r'\$[a-zA-Z]+: (.+) \$$'), r'\1'),) - - def apply(self): - if not len(self.document): - # @@@ replace these DataErrors with proper system messages - raise DataError('Document tree is empty.') - header = self.document[0] - if not isinstance(header, nodes.field_list) or \ - 'rfc2822' not in header['classes']: - raise DataError('Document does not begin with an RFC-2822 ' - 'header; it is not a PEP.') - pep = None - for field in header: - if field[0].astext().lower() == 'pep': # should be the first field - value = field[1].astext() - try: - pep = int(value) - cvs_url = self.pep_cvs_url % pep - except ValueError: - pep = value - cvs_url = None - msg = self.document.reporter.warning( - '"PEP" header must contain an integer; "%s" is an ' - 'invalid value.' % pep, base_node=field) - msgid = self.document.set_id(msg) - prb = nodes.problematic(value, value or '(none)', - refid=msgid) - prbid = self.document.set_id(prb) - msg.add_backref(prbid) - if len(field[1]): - field[1][0][:] = [prb] - else: - field[1] += nodes.paragraph('', '', prb) - break - if pep is None: - raise DataError('Document does not contain an RFC-2822 "PEP" ' - 'header.') - if pep == 0: - # Special processing for PEP 0. - pending = nodes.pending(peps.PEPZero) - self.document.insert(1, pending) - self.document.note_pending(pending) - if len(header) < 2 or header[1][0].astext().lower() != 'title': - raise DataError('No title!') - for field in header: - name = field[0].astext().lower() - body = field[1] - if len(body) > 1: - raise DataError('PEP header field body contains multiple ' - 'elements:\n%s' % field.pformat(level=1)) - elif len(body) == 1: - if not isinstance(body[0], nodes.paragraph): - raise DataError('PEP header field body may only contain ' - 'a single paragraph:\n%s' - % field.pformat(level=1)) - elif name == 'last-modified': - date = time.strftime( - '%d-%b-%Y', - time.localtime(os.stat(self.document['source'])[8])) - if cvs_url: - body += nodes.paragraph( - '', '', nodes.reference('', date, refuri=cvs_url)) - else: - # empty - continue - para = body[0] - if name in ('author', 'bdfl-delegate', 'pep-delegate', 'sponsor'): - for node in para: - if isinstance(node, nodes.reference): - node.replace_self(peps.mask_email(node)) - elif name == 'discussions-to': - for node in para: - if isinstance(node, nodes.reference): - node.replace_self(peps.mask_email(node, pep)) - elif name in ('replaces', 'superseded-by', 'requires'): - newbody = [] - space = nodes.Text(' ') - for refpep in re.split(r',?\s+', body.astext()): - pepno = int(refpep) - newbody.append(nodes.reference( - refpep, refpep, - refuri=(self.document.settings.pep_base_url - + self.pep_url % pepno))) - newbody.append(space) - para[:] = newbody[:-1] # drop trailing space - elif name == 'last-modified': - utils.clean_rcs_keywords(para, self.rcs_keyword_substitutions) - if cvs_url: - date = para.astext() - para[:] = [nodes.reference('', date, refuri=cvs_url)] - elif name == 'content-type': - pep_type = para.astext() - uri = self.document.settings.pep_base_url + self.pep_url % 12 - para[:] = [nodes.reference('', pep_type, refuri=uri)] - elif name == 'version' and len(body): - utils.clean_rcs_keywords(para, self.rcs_keyword_substitutions) - -class PEPReader(standalone.Reader): - - supported = ('pep',) - """Contexts this reader supports.""" - - settings_spec = ( - 'PEP Reader Option Defaults', - 'The --pep-references and --rfc-references options (for the ' - 'reStructuredText parser) are on by default.', - ()) - - config_section = 'pep reader' - config_section_dependencies = ('readers', 'standalone reader') - - def get_transforms(self): - transforms = standalone.Reader.get_transforms(self) - # We have PEP-specific frontmatter handling. - transforms.remove(frontmatter.DocTitle) - transforms.remove(frontmatter.SectionSubTitle) - transforms.remove(frontmatter.DocInfo) - transforms.extend([PEPHeaders, peps.Contents, peps.TargetNotes]) - return transforms - - settings_default_overrides = {'pep_references': 1, 'rfc_references': 1} - - inliner_class = rst.states.Inliner - - def __init__(self, parser=None, parser_name=None): - """`parser` should be ``None``.""" - if parser is None: - parser = rst.Parser(rfc2822=True, inliner=self.inliner_class()) - standalone.Reader.__init__(self, parser, '') - - -def fix_rst_pep(inpath, input_lines, outfile): - output = core.publish_string( - source=''.join(input_lines), - source_path=inpath, - destination_path=outfile.name, - reader=PEPReader(), - parser_name='restructuredtext', - writer_name='pep_html', - settings=docutils_settings, - # Allow Docutils traceback if there's an exception: - settings_overrides={'traceback': 1, 'halt_level': 2}) - outfile.write(output.decode('utf-8')) - - -def get_pep_type(input_lines): - """ - Return the Content-Type of the input. "text/plain" is the default. - Return ``None`` if the input is not a PEP. - """ - pep_type = None - for line in input_lines: - line = line.rstrip().lower() - if not line: - # End of the RFC 2822 header (first blank line). - break - elif line.startswith('content-type: '): - pep_type = line.split()[1] or 'text/plain' - break - elif line.startswith('pep: '): - # Default PEP type, used if no explicit content-type specified: - pep_type = 'text/plain' - return pep_type - - -def get_input_lines(inpath): - try: - infile = open(inpath, encoding='utf-8') - except IOError as e: - if e.errno != errno.ENOENT: raise - print('Error: Skipping missing PEP file:', e.filename, file=sys.stderr) - sys.stderr.flush() - return None - lines = infile.read().splitlines(1) # handles x-platform line endings - infile.close() - return lines - - -def find_pep(pep_str): - """Find the .rst or .txt file indicated by a cmd line argument""" - if os.path.exists(pep_str): - return pep_str - num = int(pep_str) - rstpath = "pep-%04d.rst" % num - if os.path.exists(rstpath): - return rstpath - return "pep-%04d.txt" % num - -def make_html(inpath, verbose=0): - input_lines = get_input_lines(inpath) - if input_lines is None: - return None - pep_type = get_pep_type(input_lines) - if pep_type is None: - print('Error: Input file %s is not a PEP.' % inpath, file=sys.stderr) - sys.stdout.flush() - return None - elif pep_type not in PEP_TYPE_DISPATCH: - print(('Error: Unknown PEP type for input file %s: %s' - % (inpath, pep_type)), file=sys.stderr) - sys.stdout.flush() - return None - elif PEP_TYPE_DISPATCH[pep_type] is None: - pep_type_error(inpath, pep_type) - return None - outpath = os.path.splitext(inpath)[0] + ".html" - if verbose: - print(inpath, "(%s)" % pep_type, "->", outpath) - sys.stdout.flush() - outfile = open(outpath, "w", encoding='utf-8') - PEP_TYPE_DISPATCH[pep_type](inpath, input_lines, outfile) - outfile.close() - os.chmod(outfile.name, 0o664) - return outpath - -def push_pep(htmlfiles, txtfiles, username, verbose, local=0): - quiet = "" - if local: - if verbose: - quiet = "-v" - target = HDIR - copy_cmd = "cp" - chmod_cmd = "chmod" - else: - if not verbose: - quiet = "-q" - if username: - username = username + "@" - target = username + HOST + ":" + HDIR - copy_cmd = "scp" - chmod_cmd = "ssh %s%s chmod" % (username, HOST) - files = htmlfiles[:] - files.extend(txtfiles) - files.append("style.css") - files.append("pep.css") - filelist = SPACE.join(files) - rc = os.system("%s %s %s %s" % (copy_cmd, quiet, filelist, target)) - if rc: - sys.exit(rc) -## rc = os.system("%s 664 %s/*" % (chmod_cmd, HDIR)) -## if rc: -## sys.exit(rc) - - -PEP_TYPE_DISPATCH = {'text/plain': fixfile, - 'text/x-rst': fix_rst_pep} -PEP_TYPE_MESSAGES = {} - -def check_requirements(): - # Check Python: - # This is pretty much covered by the __future__ imports... - if sys.version_info < (2, 6, 0): - PEP_TYPE_DISPATCH['text/plain'] = None - PEP_TYPE_MESSAGES['text/plain'] = ( - 'Python %s or better required for "%%(pep_type)s" PEP ' - 'processing; %s present (%%(inpath)s).' - % (REQUIRES['python'], sys.version.split()[0])) - # Check Docutils: - try: - import docutils - except ImportError: - PEP_TYPE_DISPATCH['text/x-rst'] = None - PEP_TYPE_MESSAGES['text/x-rst'] = ( - 'Docutils not present for "%(pep_type)s" PEP file %(inpath)s. ' - 'See README.rst for installation.') - else: - installed = [int(part) for part in docutils.__version__.split('.')] - required = [int(part) for part in REQUIRES['docutils'].split('.')] - if installed < required: - PEP_TYPE_DISPATCH['text/x-rst'] = None - PEP_TYPE_MESSAGES['text/x-rst'] = ( - 'Docutils must be reinstalled for "%%(pep_type)s" PEP ' - 'processing (%%(inpath)s). Version %s or better required; ' - '%s present. See README.rst for installation.' - % (REQUIRES['docutils'], docutils.__version__)) - -def pep_type_error(inpath, pep_type): - print('Error: ' + PEP_TYPE_MESSAGES[pep_type] % locals(), file=sys.stderr) - sys.stdout.flush() - - -def browse_file(pep): - import webbrowser - file = find_pep(pep) - if file.startswith('pep-') and file.endswith((".txt", '.rst')): - file = file[:-3] + "html" - file = os.path.abspath(file) - url = "file:" + file - webbrowser.open(url) - -def browse_remote(pep): - import webbrowser - file = find_pep(pep) - if file.startswith('pep-') and file.endswith((".txt", '.rst')): - file = file[:-3] + "html" - url = PEPDIRRUL + file - webbrowser.open(url) - - -def main(argv=None): - # defaults - update = 0 - local = 0 - username = '' - verbose = 1 - browse = 0 - - check_requirements() - - if argv is None: - argv = sys.argv[1:] - - try: - opts, args = getopt.getopt( - argv, 'bilhqu:', - ['browse', 'install', 'local', 'help', 'quiet', 'user=']) - except getopt.error as msg: - usage(1, msg) - - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-i', '--install'): - update = 1 - elif opt in ('-l', '--local'): - update = 1 - local = 1 - elif opt in ('-u', '--user'): - username = arg - elif opt in ('-q', '--quiet'): - verbose = 0 - elif opt in ('-b', '--browse'): - browse = 1 - - if args: - pep_list = [] - html = [] - for pep in args: - file = find_pep(pep) - pep_list.append(file) - newfile = make_html(file, verbose=verbose) - if newfile: - html.append(newfile) - if browse and not update: - browse_file(pep) - else: - # do them all - pep_list = [] - html = [] - files = glob.glob("pep-*.txt") + glob.glob("pep-*.rst") - files.sort() - for file in files: - pep_list.append(file) - newfile = make_html(file, verbose=verbose) - if newfile: - html.append(newfile) - if browse and not update: - browse_file("0") - - if update: - push_pep(html, pep_list, username, verbose, local=local) - if browse: - if args: - for pep in args: - browse_remote(pep) - else: - browse_remote("0") - - - -if __name__ == "__main__": - main() diff --git a/pep2rss.py b/pep2rss.py deleted file mode 100755 index 52b532f51..000000000 --- a/pep2rss.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 - -# usage: python3 pep2rss.py . - -import datetime -import glob -import os -import re -import sys -import time -import PyRSS2Gen as rssgen -import docutils.frontend -import docutils.nodes -import docutils.parsers.rst -import docutils.utils - -RSS_PATH = os.path.join(sys.argv[1], 'peps.rss') - - -def remove_prefix(text: str, prefix: str) -> str: - try: - # Python 3.9+ - return text.removeprefix(prefix) - except AttributeError: - if text.startswith(prefix): - return text[len(prefix):] - return text - - -def parse_rst(text: str) -> docutils.nodes.document: - parser = docutils.parsers.rst.Parser() - components = (docutils.parsers.rst.Parser,) - settings = docutils.frontend.OptionParser(components=components).get_default_values() - document = docutils.utils.new_document('', settings=settings) - parser.parse(text, document) - return document - - -def pep_abstract(full_path: str) -> str: - """Return the first paragraph of the PEP abstract""" - abstract = None - with open(full_path, encoding="utf-8") as f: - text = f.read() - document = parse_rst(text) - nodes = list(document) - for node in nodes: - if "Abstract" in str(node): - for child in node: - if child.tagname == "paragraph": - abstract = child.astext() - # Just fetch the first paragraph - break - return abstract - - -def firstline_startingwith(full_path, text): - for line in open(full_path, encoding="utf-8"): - if line.startswith(text): - return line[len(text):].strip() - return None - - -# get list of peps with creation time -# (from "Created:" string in pep .rst or .txt) -peps = glob.glob('pep-*.txt') -peps.extend(glob.glob('pep-*.rst')) - - -def pep_creation_dt(full_path): - created_str = firstline_startingwith(full_path, 'Created:') - # bleh, I was hoping to avoid re but some PEPs editorialize - # on the Created line - m = re.search(r'''(\d+-\w+-\d{4})''', created_str) - if not m: - # some older ones have an empty line, that's okay, if it's old - # we ipso facto don't care about it. - # "return None" would make the most sense but datetime objects - # refuse to compare with that. :-| - return datetime.datetime(*time.localtime(0)[:6]) - created_str = m.group(1) - try: - t = time.strptime(created_str, '%d-%b-%Y') - except ValueError: - t = time.strptime(created_str, '%d-%B-%Y') - return datetime.datetime(*t[:6]) - - -peps_with_dt = [(pep_creation_dt(full_path), full_path) for full_path in peps] -# sort peps by date, newest first -peps_with_dt.sort(reverse=True) - -# generate rss items for 10 most recent peps -items = [] -for dt, full_path in peps_with_dt[:10]: - try: - n = int(full_path.split('-')[-1].split('.')[0]) - except ValueError: - pass - title = firstline_startingwith(full_path, 'Title:') - author = firstline_startingwith(full_path, 'Author:') - abstract = pep_abstract(full_path) - url = 'https://www.python.org/dev/peps/pep-%0.4d/' % n - item = rssgen.RSSItem( - title='PEP %d: %s' % (n, title), - link=url, - description=abstract, - author=author, - guid=rssgen.Guid(url), - pubDate=dt) - items.append(item) - -# the rss envelope -desc = """ -Newest Python Enhancement Proposals (PEPs) - Information on new -language features, and some meta-information like release -procedure and schedules -""".strip() -rss = rssgen.RSS2( - title='Newest Python PEPs', - link = 'https://www.python.org/dev/peps/', - description=desc, - lastBuildDate=datetime.datetime.now(), - items=items) - -with open(RSS_PATH, 'w', encoding="utf-8") as fp: - fp.write(rss.to_xml(encoding="utf-8")) diff --git a/pep_rss_gen.py b/pep_rss_gen.py deleted file mode 100644 index a06ffd20c..000000000 --- a/pep_rss_gen.py +++ /dev/null @@ -1,145 +0,0 @@ -import datetime -import email.utils -from pathlib import Path -import re - -from dateutil import parser -import docutils.frontend -import docutils.nodes -import docutils.parsers.rst -import docutils.utils -from feedgen import entry -from feedgen import feed - - -# Monkeypatch feedgen.util.formatRFC2822 -def _format_rfc_2822(dt: datetime.datetime) -> str: - return email.utils.format_datetime(dt, usegmt=True) - - -entry.formatRFC2822 = feed.formatRFC2822 = _format_rfc_2822 -line_cache: dict[Path, dict[str, str]] = {} - - -def first_line_starting_with(full_path: Path, text: str) -> str: - # Try and retrieve from cache - if full_path in line_cache: - return line_cache[full_path].get(text, "") - - # Else read source - line_cache[full_path] = path_cache = {} - for line in full_path.open(encoding="utf-8"): - if line.startswith("Created:"): - path_cache["Created:"] = line.removeprefix("Created:").strip() - elif line.startswith("Title:"): - path_cache["Title:"] = line.removeprefix("Title:").strip() - elif line.startswith("Author:"): - path_cache["Author:"] = line.removeprefix("Author:").strip() - - # Once all have been found, exit loop - if path_cache.keys == {"Created:", "Title:", "Author:"}: - break - return path_cache.get(text, "") - - -def pep_creation(full_path: Path) -> datetime.datetime: - created_str = first_line_starting_with(full_path, "Created:") - # bleh, I was hoping to avoid re but some PEPs editorialize on the Created line - # (note as of Aug 2020 only PEP 102 has additional content on the Created line) - m = re.search(r"(\d+[- ][\w\d]+[- ]\d{2,4})", created_str) - if not m: - # some older ones have an empty line, that's okay, if it's old we ipso facto don't care about it. - # "return None" would make the most sense but datetime objects refuse to compare with that. :-| - return datetime.datetime(1900, 1, 1) - created_str = m.group(1) - try: - return parser.parse(created_str, dayfirst=True) - except (ValueError, OverflowError): - return datetime.datetime(1900, 1, 1) - - -def parse_rst(text: str) -> docutils.nodes.document: - rst_parser = docutils.parsers.rst.Parser() - components = (docutils.parsers.rst.Parser,) - settings = docutils.frontend.OptionParser(components=components).get_default_values() - document = docutils.utils.new_document('', settings=settings) - rst_parser.parse(text, document) - return document - - -def pep_abstract(full_path: Path) -> str: - """Return the first paragraph of the PEP abstract""" - text = full_path.read_text(encoding="utf-8") - for node in parse_rst(text): - if "Abstract" in str(node): - for child in node: - if child.tagname == "paragraph": - return child.astext().strip().replace("\n", " ") - return "" - - -def main(): - # get the directory with the PEP sources - pep_dir = Path(__file__).parent - - # get list of peps with creation time (from "Created:" string in pep source) - peps_with_dt = sorted((pep_creation(path), path) for path in pep_dir.glob("pep-????.*")) - - # generate rss items for 10 most recent peps - items = [] - for dt, full_path in peps_with_dt[-10:]: - try: - pep_num = int(full_path.stem.split("-")[-1]) - except ValueError: - continue - - title = first_line_starting_with(full_path, "Title:") - author = first_line_starting_with(full_path, "Author:") - if "@" in author or " at " in author: - parsed_authors = email.utils.getaddresses([author]) - # ideal would be to pass as a list of dicts with names and emails to - # item.author, but FeedGen's RSS output doesn't pass W3C - # validation (as of 12/06/2021) - joined_authors = ", ".join(f"{name} ({email_address})" for name, email_address in parsed_authors) - else: - joined_authors = author - url = f"https://www.python.org/dev/peps/pep-{pep_num:0>4}" - - item = entry.FeedEntry() - item.title(f"PEP {pep_num}: {title}") - item.link(href=url) - item.description(pep_abstract(full_path)) - item.guid(url, permalink=True) - item.published(dt.replace(tzinfo=datetime.timezone.utc)) # ensure datetime has a timezone - item.author(email=joined_authors) - items.append(item) - - # The rss envelope - desc = """ - Newest Python Enhancement Proposals (PEPs) - Information on new - language features, and some meta-information like release - procedure and schedules. - """.replace("\n ", " ").strip() - - # Setup feed generator - fg = feed.FeedGenerator() - fg.language("en") - fg.generator("") - fg.docs("https://cyber.harvard.edu/rss/rss.html") - - # Add metadata - fg.title("Newest Python PEPs") - fg.link(href="https://www.python.org/dev/peps") - fg.link(href="https://www.python.org/dev/peps/peps.rss", rel="self") - fg.description(desc) - fg.lastBuildDate(datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)) - - # Add PEP information (ordered by newest first) - for item in items: - fg.add_entry(item) - - pep_dir.joinpath("peps.rss").write_bytes(fg.rss_str(pretty=True)) - - -if __name__ == "__main__": - main() diff --git a/pep_sphinx_extensions/LICENCE.rst b/pep_sphinx_extensions/LICENCE.rst new file mode 100644 index 000000000..c4665c1ad --- /dev/null +++ b/pep_sphinx_extensions/LICENCE.rst @@ -0,0 +1,2 @@ +This files in this directory are placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep_sphinx_extensions/__init__.py b/pep_sphinx_extensions/__init__.py index 37ffde408..8538b838a 100644 --- a/pep_sphinx_extensions/__init__.py +++ b/pep_sphinx_extensions/__init__.py @@ -5,54 +5,78 @@ from __future__ import annotations from typing import TYPE_CHECKING from docutils.writers.html5_polyglot import HTMLTranslator -from sphinx.environment import BuildEnvironment -from sphinx.environment import default_settings +from sphinx import environment -from pep_sphinx_extensions import config +from pep_sphinx_extensions.generate_rss import create_rss_feed +from pep_sphinx_extensions.pep_processor.html import pep_html_builder from pep_sphinx_extensions.pep_processor.html import pep_html_translator +from pep_sphinx_extensions.pep_processor.parsing import pep_banner_directive from pep_sphinx_extensions.pep_processor.parsing import pep_parser from pep_sphinx_extensions.pep_processor.parsing import pep_role +from pep_sphinx_extensions.pep_processor.transforms import pep_references from pep_sphinx_extensions.pep_zero_generator.pep_index_generator import create_pep_zero if TYPE_CHECKING: from sphinx.application import Sphinx -# Monkeypatch sphinx.environment.default_settings as Sphinx doesn't allow custom settings or Readers -# These settings should go in docutils.conf, but are overridden here for now so as not to affect -# pep2html.py -default_settings |= { - "pep_references": True, - "rfc_references": True, - "pep_base_url": "", - "pep_file_url_template": "pep-%04d.html", - "_disable_config": True, # disable using docutils.conf whilst running both PEP generators -} - -# Monkeypatch sphinx.environment.BuildEnvironment.collect_relations, as it takes a long time -# and we don't use the parent/next/prev functionality -BuildEnvironment.collect_relations = lambda self: {} - def _depart_maths(): pass # No-op callable for the type checker -def _update_config_for_builder(app: Sphinx): +def _update_config_for_builder(app: Sphinx) -> None: + app.env.document_ids = {} # For PEPReferenceRoleTitleText + app.env.settings["builder"] = app.builder.name if app.builder.name == "dirhtml": - config.pep_url = f"../{config.pep_stem}" - app.env.settings["pep_file_url_template"] = "../pep-%04d" + app.env.settings["pep_url"] = "pep-{:0>4}/" + + app.connect("build-finished", _post_build) # Post-build tasks + + +def _post_build(app: Sphinx, exception: Exception | None) -> None: + from pathlib import Path + + from build import create_index_file + + if exception is not None: + return + + # internal_builder exists if Sphinx is run by build.py + if "internal_builder" not in app.tags: + create_index_file(Path(app.outdir), app.builder.name) + create_rss_feed(app.doctreedir, app.outdir) def setup(app: Sphinx) -> dict[str, bool]: """Initialize Sphinx extension.""" + environment.default_settings["pep_url"] = "pep-{:0>4}.html" + environment.default_settings["halt_level"] = 2 # Fail on Docutils warning + # Register plugin logic + app.add_builder(pep_html_builder.FileBuilder, override=True) + app.add_builder(pep_html_builder.DirectoryBuilder, override=True) + app.add_source_parser(pep_parser.PEPParser) # Add PEP transforms - app.add_role("pep", pep_role.PEPRole(), override=True) # Transform PEP references to links + app.set_translator("html", pep_html_translator.PEPTranslator) # Docutils Node Visitor overrides (html builder) app.set_translator("dirhtml", pep_html_translator.PEPTranslator) # Docutils Node Visitor overrides (dirhtml builder) - app.connect("env-before-read-docs", create_pep_zero) # PEP 0 hook + + app.add_role("pep", pep_role.PEPRole(), override=True) # Transform PEP references to links + + app.add_post_transform(pep_references.PEPReferenceRoleTitleText) + + # Register custom directives + app.add_directive( + "pep-banner", pep_banner_directive.PEPBanner) + app.add_directive( + "canonical-doc", pep_banner_directive.CanonicalDocBanner) + app.add_directive( + "canonical-pypa-spec", pep_banner_directive.CanonicalPyPASpecBanner) + + # Register event callbacks app.connect("builder-inited", _update_config_for_builder) # Update configuration values for builder used + app.connect("env-before-read-docs", create_pep_zero) # PEP 0 hook # Mathematics rendering inline_maths = HTMLTranslator.visit_math, _depart_maths diff --git a/pep_sphinx_extensions/config.py b/pep_sphinx_extensions/config.py deleted file mode 100644 index 3ea56aad4..000000000 --- a/pep_sphinx_extensions/config.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Miscellaneous configuration variables for the PEP Sphinx extensions.""" - -pep_stem = "pep-{:0>4}" -pep_url = f"{pep_stem}.html" -pep_vcs_url = "https://github.com/python/peps/blob/master/" -pep_commits_url = "https://github.com/python/peps/commits/master/" diff --git a/pep_sphinx_extensions/generate_rss.py b/pep_sphinx_extensions/generate_rss.py new file mode 100644 index 000000000..5e5e0b8bc --- /dev/null +++ b/pep_sphinx_extensions/generate_rss.py @@ -0,0 +1,117 @@ +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +from __future__ import annotations + +import datetime as dt +import pickle +from email.utils import format_datetime, getaddresses +from html import escape +from pathlib import Path + +from docutils import nodes + +RSS_DESCRIPTION = ( + "Newest Python Enhancement Proposals (PEPs): " + "Information on new language features " + "and some meta-information like release procedure and schedules." +) + + +def _format_rfc_2822(datetime: dt.datetime) -> str: + datetime = datetime.replace(tzinfo=dt.timezone.utc) + return format_datetime(datetime, usegmt=True) + + +document_cache: dict[Path, dict[str, str]] = {} + + +def get_from_doctree(full_path: Path, text: str) -> str: + # Try and retrieve from cache + if full_path in document_cache: + return document_cache[full_path].get(text, "") + + # Else load doctree + document = pickle.loads(full_path.read_bytes()) + # Store the headers (populated in the PEPHeaders transform) + document_cache[full_path] = path_cache = document.get("headers", {}) + # Store the Abstract + path_cache["Abstract"] = pep_abstract(document) + # Return the requested key + return path_cache.get(text, "") + + +def pep_creation(full_path: Path) -> dt.datetime: + created_str = get_from_doctree(full_path, "Created") + try: + return dt.datetime.strptime(created_str, "%d-%b-%Y") + except ValueError: + return dt.datetime.min + + +def pep_abstract(document: nodes.document) -> str: + """Return the first paragraph of the PEP abstract""" + for node in document.findall(nodes.section): + title_node = node.next_node(nodes.title) + if title_node is None: + continue + if title_node.astext() == "Abstract": + return node.next_node(nodes.paragraph).astext().strip().replace("\n", " ") + return "" + + +def _generate_items(doctree_dir: Path): + # get list of peps with creation time (from "Created:" string in pep source) + peps_with_dt = sorted((pep_creation(path), path) for path in doctree_dir.glob("pep-????.doctree")) + + # generate rss items for 10 most recent peps (in reverse order) + for datetime, full_path in reversed(peps_with_dt[-10:]): + try: + pep_num = int(get_from_doctree(full_path, "PEP")) + except ValueError: + continue + + title = get_from_doctree(full_path, "Title") + url = f"https://peps.python.org/pep-{pep_num:0>4}/" + abstract = get_from_doctree(full_path, "Abstract") + author = get_from_doctree(full_path, "Author") + if "@" in author or " at " in author: + parsed_authors = getaddresses([author]) + joined_authors = ", ".join(f"{name} ({email_address})" for name, email_address in parsed_authors) + else: + joined_authors = author + + item = f"""\ + + PEP {pep_num}: {escape(title, quote=False)} + {escape(url, quote=False)} + {escape(abstract, quote=False)} + {escape(joined_authors, quote=False)} + {url} + {_format_rfc_2822(datetime)} + """ + yield item + + +def create_rss_feed(doctree_dir: Path, output_dir: Path): + # The rss envelope + last_build_date = _format_rfc_2822(dt.datetime.now(dt.timezone.utc)) + items = "\n".join(_generate_items(Path(doctree_dir))) + output = f"""\ + + + + Newest Python PEPs + https://peps.python.org/peps.rss + {RSS_DESCRIPTION} + + https://cyber.harvard.edu/rss/rss.html + en + {last_build_date} +{items} + + +""" + + # output directory for target HTML files + Path(output_dir, "peps.rss").write_text(output, encoding="utf-8") diff --git a/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py b/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py new file mode 100644 index 000000000..7349f712f --- /dev/null +++ b/pep_sphinx_extensions/pep_processor/html/pep_html_builder.py @@ -0,0 +1,50 @@ +from docutils import nodes +from docutils.frontend import OptionParser +from sphinx.builders.html import StandaloneHTMLBuilder +from sphinx.writers.html import HTMLWriter + +from sphinx.builders.dirhtml import DirectoryHTMLBuilder + + +class FileBuilder(StandaloneHTMLBuilder): + copysource = False # Prevent unneeded source copying - we link direct to GitHub + search = False # Disable search + + # Things we don't use but that need to exist: + indexer = None + relations = {} + _script_files = _css_files = [] + globalcontext = {"script_files": [], "css_files": []} + + def prepare_writing(self, _doc_names: set[str]) -> None: + self.docwriter = HTMLWriter(self) + _opt_parser = OptionParser([self.docwriter], defaults=self.env.settings, read_config_files=True) + self.docsettings = _opt_parser.get_default_values() + self._orig_css_files = self._orig_js_files = [] + + def get_doc_context(self, docname: str, body: str, _metatags: str) -> dict: + """Collect items for the template context of a page.""" + try: + title = self.env.longtitles[docname].astext() + except KeyError: + title = "" + + # local table of contents + toc_tree = self.env.tocs[docname].deepcopy() + if len(toc_tree) and len(toc_tree[0]) > 1: + toc_tree = toc_tree[0][1] # don't include document title + del toc_tree[0] # remove contents node + for node in toc_tree.findall(nodes.reference): + node["refuri"] = node["anchorname"] or '#' # fix targets + toc = self.render_partial(toc_tree)["fragment"] + else: + toc = "" # PEPs with no sections -- 9, 210 + + return {"title": title, "toc": toc, "body": body} + + +class DirectoryBuilder(FileBuilder): + # sync all overwritten things from DirectoryHTMLBuilder + name = DirectoryHTMLBuilder.name + get_target_uri = DirectoryHTMLBuilder.get_target_uri + get_outfilename = DirectoryHTMLBuilder.get_outfilename diff --git a/pep_sphinx_extensions/pep_processor/html/pep_html_translator.py b/pep_sphinx_extensions/pep_processor/html/pep_html_translator.py index 9e87daf89..e8893790e 100644 --- a/pep_sphinx_extensions/pep_processor/html/pep_html_translator.py +++ b/pep_sphinx_extensions/pep_processor/html/pep_html_translator.py @@ -57,30 +57,49 @@ class PEPTranslator(html5.HTML5Translator): """Add corresponding end tag from `visit_paragraph`.""" self.body.append(self.context.pop()) + def visit_footnote_reference(self, node): + self.body.append(self.starttag(node, "a", suffix="[", + CLASS=f"footnote-reference {self.settings.footnote_references}", + href=f"#{node['refid']}" + )) + + def depart_footnote_reference(self, node): + self.body.append(']') + + def visit_label(self, node): + # pass parent node to get id into starttag: + self.body.append(self.starttag(node.parent, "dt", suffix="[", CLASS="label")) + + # footnote/citation backrefs: + back_refs = node.parent["backrefs"] + if self.settings.footnote_backlinks and len(back_refs) == 1: + self.body.append(f'') + self.context.append("]") + else: + self.context.append("]") + def depart_label(self, node) -> None: """PEP link/citation block cleanup with italicised backlinks.""" - if not self.settings.footnote_backlinks: - self.body.append("") - self.body.append("\n
") - return - - # If only one reference to this footnote - back_references = node.parent["backrefs"] - if len(back_references) == 1: - self.body.append("") - - # Close the tag - self.body.append("") - - # If more than one reference - if len(back_references) > 1: - back_links = [f"{i}" for i, ref in enumerate(back_references, start=1)] - back_links_str = ", ".join(back_links) - self.body.append(f" ({back_links_str}) ") + self.body.append(self.context.pop()) + back_refs = node.parent["backrefs"] + if self.settings.footnote_backlinks and len(back_refs) > 1: + back_links = ", ".join(f"{i}" for i, ref in enumerate(back_refs, start=1)) + self.body.append(f" ({back_links}) ") # Close the def tags self.body.append("\n
") + def visit_bullet_list(self, node): + if isinstance(node.parent, nodes.section) and "contents" in node.parent["names"]: + self.body.append("
Table of Contents") + self.context.append("
") + super().visit_bullet_list(node) + + def depart_bullet_list(self, node): + super().depart_bullet_list(node) + if isinstance(node.parent, nodes.section) and "contents" in node.parent["names"]: + self.body.append(self.context.pop()) + def unknown_visit(self, node: nodes.Node) -> None: """No processing for unknown node types.""" pass diff --git a/pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py b/pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py new file mode 100644 index 000000000..d0f1cbee2 --- /dev/null +++ b/pep_sphinx_extensions/pep_processor/parsing/pep_banner_directive.py @@ -0,0 +1,101 @@ +"""Roles to insert custom admonitions pointing readers to canonical content.""" + +from __future__ import annotations + +from docutils import nodes +from docutils.parsers import rst + +PYPA_SPEC_BASE_URL = "https://packaging.python.org/en/latest/specifications/" + + +class PEPBanner(rst.Directive): + """Insert a special banner admonition in a PEP document.""" + + has_content = True + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + option_spec = {} + + admonition_pre_template = "" + admonition_pre_text = "" + admonition_post_text = "" + + admonition_class = nodes.important + css_classes = [] + + + def run(self) -> list[nodes.admonition]: + + if self.arguments: + link_content = self.arguments[0] + pre_text = self.admonition_pre_template.format( + link_content=link_content) + else: + pre_text = self.admonition_pre_text + + pre_text_node = nodes.paragraph(pre_text) + pre_text_node.line = self.lineno + pre_node, pre_msg = self.state.inline_text(pre_text, self.lineno) + pre_text_node.extend(pre_node + pre_msg) + + post_text = self.admonition_post_text + post_text_node = nodes.paragraph(post_text) + post_text_node.line = self.lineno + post_node, post_msg = self.state.inline_text(post_text, self.lineno) + post_text_node.extend(post_node + post_msg) + + source_lines = [pre_text] + list(self.content or []) + [post_text] + admonition_node = self.admonition_class( + "\n".join(source_lines), classes=["pep-banner"] + self.css_classes) + + admonition_node.append(pre_text_node) + if self.content: + self.state.nested_parse( + self.content, self.content_offset, admonition_node) + admonition_node.append(post_text_node) + + return [admonition_node] + + +class CanonicalDocBanner(PEPBanner): + """Insert an admonition pointing readers to a PEP's canonical docs.""" + + admonition_pre_template = ( + "This PEP is a historical document. " + "The up-to-date, canonical documentation can now be found " + "at {link_content}." + ) + admonition_pre_text = ( + "This PEP is a historical document. " + "The up-to-date, canonical documentation can now be found elsewhere." + ) + admonition_post_text = ( + "See :pep:`1` for how to propose changes." + ) + + css_classes = ["canonical-doc", "sticky-banner"] + + + +class CanonicalPyPASpecBanner(PEPBanner): + """Insert a specialized admonition for PyPA packaging specifications.""" + + admonition_pre_template = ( + "This PEP is a historical document. " + "The up-to-date, canonical spec, {link_content}, is maintained on " + f"the `PyPA specs page <{PYPA_SPEC_BASE_URL}>`__." + ) + admonition_pre_text = ( + "This PEP is a historical document. " + "The up-to-date, canonical specifications are maintained on " + f"the `PyPA specs page <{PYPA_SPEC_BASE_URL}>`__." + ) + admonition_post_text = ( + "See the `PyPA specification update process " + "`__ " + "for how to propose changes." + ) + admonition_class = nodes.attention + + css_classes = ["canonical-pypa-spec", "sticky-banner"] diff --git a/pep_sphinx_extensions/pep_processor/parsing/pep_role.py b/pep_sphinx_extensions/pep_processor/parsing/pep_role.py index 6260c3422..be1683a33 100644 --- a/pep_sphinx_extensions/pep_processor/parsing/pep_role.py +++ b/pep_sphinx_extensions/pep_processor/parsing/pep_role.py @@ -1,15 +1,39 @@ +from docutils import nodes from sphinx import roles -from pep_sphinx_extensions import config - -class PEPRole(roles.PEP): +class PEPRole(roles.ReferenceRole): """Override the :pep: role""" - def build_uri(self) -> str: - """Get PEP URI from role text.""" + def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: + # Get PEP URI from role text. pep_str, _, fragment = self.target.partition("#") - pep_base = config.pep_url.format(int(pep_str)) + try: + pep_num = int(pep_str) + except ValueError: + msg = self.inliner.reporter.error(f'invalid PEP number {self.target}', line=self.lineno) + prb = self.inliner.problematic(self.rawtext, self.rawtext, msg) + return [prb], [msg] + pep_base = self.inliner.document.settings.pep_url.format(pep_num) + if self.inliner.document.settings.builder == "dirhtml": + pep_base = "../" + pep_base + if "topic" in self.get_location(): + pep_base = "../" + pep_base if fragment: - return f"{pep_base}#{fragment}" - return pep_base + ref_uri = f"{pep_base}#{fragment}" + else: + ref_uri = pep_base + if self.has_explicit_title: + title = self.title + else: + title = f"PEP {pep_num}" + + return [ + nodes.reference( + "", title, + internal=True, + refuri=ref_uri, + classes=["pep"], + _title_tuple=(pep_num, fragment) + ) + ], [] diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_contents.py b/pep_sphinx_extensions/pep_processor/transforms/pep_contents.py index db975549f..3810429ac 100644 --- a/pep_sphinx_extensions/pep_processor/transforms/pep_contents.py +++ b/pep_sphinx_extensions/pep_processor/transforms/pep_contents.py @@ -17,8 +17,7 @@ class PEPContents(transforms.Transform): if not Path(self.document["source"]).match("pep-*"): return # not a PEP file, exit early # Create the contents placeholder section - title = nodes.title("", "", nodes.Text("Contents")) - contents_section = nodes.section("", title) + contents_section = nodes.section("") if not self.document.has_name("contents"): contents_section["names"].append("contents") self.document.note_implicit_target(contents_section) diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py b/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py index 5fa1b3844..c49355fd1 100644 --- a/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py +++ b/pep_sphinx_extensions/pep_processor/transforms/pep_footer.py @@ -1,25 +1,16 @@ -import datetime +import time from pathlib import Path import subprocess from docutils import nodes from docutils import transforms -from docutils.transforms import misc -from docutils.transforms import references - -from pep_sphinx_extensions import config class PEPFooter(transforms.Transform): """Footer transforms for PEPs. - - Appends external links to footnotes. - - Creates a link to the (GitHub) source text. - - TargetNotes: - Locate the `References` section, insert a placeholder at the end - for an external target footnote insertion transform, and schedule - the transform to run immediately. + - Remove the References/Footnotes section if it is empty when rendered. + - Create a link to the (GitHub) source text. Source Link: Create the link to the source file from the document source path, @@ -32,80 +23,86 @@ class PEPFooter(transforms.Transform): def apply(self) -> None: pep_source_path = Path(self.document["source"]) - if not pep_source_path.match("pep-*"): + if not pep_source_path.match("pep-????.???"): return # not a PEP file, exit early - doc = self.document[0] - reference_section = copyright_section = None - # Iterate through sections from the end of the document - num_sections = len(doc) - for i, section in enumerate(reversed(doc)): + for section in reversed(self.document[0]): if not isinstance(section, nodes.section): continue - title_words = section[0].astext().lower().split() - if "references" in title_words: - reference_section = section - break - elif "copyright" in title_words: - copyright_section = num_sections - i - 1 - - # Add a references section if we didn't find one - if not reference_section: - reference_section = nodes.section() - reference_section += nodes.title("", "References") - self.document.set_id(reference_section) - if copyright_section: - # Put the new "References" section before "Copyright": - doc.insert(copyright_section, reference_section) - else: - # Put the new "References" section at end of doc: - doc.append(reference_section) - - # Add and schedule execution of the TargetNotes transform - pending = nodes.pending(references.TargetNotes) - reference_section.append(pending) - self.document.note_pending(pending, priority=0) - - # If there are no references after TargetNotes has finished, remove the - # references section - pending = nodes.pending(misc.CallBack, details={"callback": _cleanup_callback}) - reference_section.append(pending) - self.document.note_pending(pending, priority=1) + title_words = {*section[0].astext().lower().split()} + if {"references", "footnotes"} & title_words: + # Remove references/footnotes sections if there is no displayed + # content (i.e. they only have title & link target nodes) + to_hoist = [] + types = set() + for node in section: + types.add(type(node)) + if isinstance(node, nodes.target): + to_hoist.append(node) + if types <= {nodes.title, nodes.target, nodes.system_message}: + section.parent.extend(to_hoist) + section.parent.remove(section) # Add link to source text and last modified date if pep_source_path.stem != "pep-0000": + if pep_source_path.stem != "pep-0210": # 210 is entirely empty, skip + self.document += nodes.transition() self.document += _add_source_link(pep_source_path) self.document += _add_commit_history_info(pep_source_path) -def _cleanup_callback(pending: nodes.pending) -> None: - """Remove an empty "References" section. - - Called after the `references.TargetNotes` transform is complete. - - """ - if len(pending.parent) == 2: # and <pending> - pending.parent.parent.remove(pending.parent) - - def _add_source_link(pep_source_path: Path) -> nodes.paragraph: """Add link to source text on VCS (GitHub)""" - source_link = config.pep_vcs_url + pep_source_path.name + source_link = f"https://github.com/python/peps/blob/main/peps/{pep_source_path.name}" link_node = nodes.reference("", source_link, refuri=source_link) return nodes.paragraph("", "Source: ", link_node) def _add_commit_history_info(pep_source_path: Path) -> nodes.paragraph: """Use local git history to find last modified date.""" - args = ["git", "--no-pager", "log", "-1", "--format=%at", pep_source_path.name] try: - file_modified = subprocess.check_output(args) - since_epoch = file_modified.decode("utf-8").strip() - dt = datetime.datetime.utcfromtimestamp(float(since_epoch)) - except (subprocess.CalledProcessError, ValueError): + iso_time = _LAST_MODIFIED_TIMES[pep_source_path.stem] + except KeyError: return nodes.paragraph() - commit_link = config.pep_commits_url + pep_source_path.name - link_node = nodes.reference("", f"{dt.isoformat(sep=' ')} GMT", refuri=commit_link) + commit_link = f"https://github.com/python/peps/commits/main/{pep_source_path.name}" + link_node = nodes.reference("", f"{iso_time} GMT", refuri=commit_link) return nodes.paragraph("", "Last modified: ", link_node) + + +def _get_last_modified_timestamps(): + # get timestamps and changed files from all commits (without paging results) + args = ("git", "--no-pager", "log", "--format=#%at", "--name-only") + ret = subprocess.run(args, stdout=subprocess.PIPE, text=True, encoding="utf-8") + if ret.returncode: # non-zero return code + return {} + all_modified = ret.stdout + + # remove "peps/" prefix from file names + all_modified = all_modified.replace("\npeps/", "\n") + + # set up the dictionary with the *current* files + peps_dir = Path(__file__, "..", "..", "..", "..", "peps").resolve() + last_modified = {path.stem: "" for path in peps_dir.glob("pep-????.rst")} + + # iterate through newest to oldest, updating per file timestamps + change_sets = all_modified.removeprefix("#").split("#") + for change_set in change_sets: + timestamp, files = change_set.split("\n", 1) + for file in files.strip().split("\n"): + if not file.startswith("pep-") or not file.endswith((".rst", ".txt")): + continue # not a PEP + file = file[:-4] + if last_modified.get(file) != "": + continue # most recent modified date already found + try: + y, m, d, hh, mm, ss, *_ = time.gmtime(float(timestamp)) + except ValueError: + continue # if float conversion fails + last_modified[file] = f"{y:04}-{m:02}-{d:02} {hh:02}:{mm:02}:{ss:02}" + + return last_modified + + +_LAST_MODIFIED_TIMES = _get_last_modified_timestamps() diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_headers.py b/pep_sphinx_extensions/pep_processor/transforms/pep_headers.py index 12805db49..a7cd0c303 100644 --- a/pep_sphinx_extensions/pep_processor/transforms/pep_headers.py +++ b/pep_sphinx_extensions/pep_processor/transforms/pep_headers.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from pathlib import Path import re @@ -7,9 +5,44 @@ from docutils import nodes from docutils import transforms from sphinx import errors -from pep_sphinx_extensions import config from pep_sphinx_extensions.pep_processor.transforms import pep_zero +from pep_sphinx_extensions.pep_processor.transforms.pep_zero import _mask_email +from pep_sphinx_extensions.pep_zero_generator.constants import ( + SPECIAL_STATUSES, + STATUS_ACCEPTED, + STATUS_ACTIVE, + STATUS_DEFERRED, + STATUS_DRAFT, + STATUS_FINAL, + STATUS_PROVISIONAL, + STATUS_REJECTED, + STATUS_SUPERSEDED, + STATUS_WITHDRAWN, + TYPE_INFO, + TYPE_PROCESS, + TYPE_STANDARDS, +) +ABBREVIATED_STATUSES = { + STATUS_DRAFT: "Proposal under active discussion and revision", + STATUS_DEFERRED: "Inactive draft that may be taken up again at a later time", + STATUS_ACCEPTED: "Normative proposal accepted for implementation", + STATUS_ACTIVE: "Currently valid informational guidance, or an in-use process", + STATUS_FINAL: "Accepted and implementation complete, or no longer active", + STATUS_WITHDRAWN: "Removed from consideration by sponsor or authors", + STATUS_REJECTED: "Formally declined and will not be accepted", + STATUS_SUPERSEDED: "Replaced by another succeeding PEP", + STATUS_PROVISIONAL: "Provisionally accepted but additional feedback needed", +} + +ABBREVIATED_TYPES = { + TYPE_STANDARDS: "Normative PEP with a new feature for Python, implementation " + "change for CPython or interoperability standard for the ecosystem", + TYPE_INFO: "Non-normative PEP containing background, guidelines or other " + "information relevant to the Python ecosystem", + TYPE_PROCESS: "Normative PEP describing or proposing a change to a Python " + "community process, workflow or governance", +} class PEPParsingError(errors.SphinxError): pass @@ -39,14 +72,14 @@ class PEPHeaders(transforms.Transform): raise PEPParsingError("Document does not contain an RFC-2822 'PEP' header!") # Extract PEP number - value = pep_field[1].astext() + pep_num_str = pep_field[1].astext() try: - pep = int(value) + pep_num = int(pep_num_str) except ValueError: - raise PEPParsingError(f"'PEP' header must contain an integer. '{value}' is invalid!") + raise PEPParsingError(f"PEP header must contain an integer. '{pep_num_str}' is invalid!") # Special processing for PEP 0. - if pep == 0: + if pep_num == 0: pending = nodes.pending(pep_zero.PEPZero) self.document.insert(1, pending) self.document.note_pending(pending) @@ -56,7 +89,11 @@ class PEPHeaders(transforms.Transform): raise PEPParsingError("No title!") fields_to_remove = [] + self.document["headers"] = headers = {} for field in header: + row_attributes = {sub.tagname: sub.rawsource for sub in field} + headers[row_attributes["field_name"]] = row_attributes["field_body"] + name = field[0].astext().lower() body = field[1] if len(body) == 0: @@ -70,45 +107,197 @@ class PEPHeaders(transforms.Transform): raise PEPParsingError(msg) para = body[0] - if name in {"author", "bdfl-delegate", "pep-delegate", "discussions-to", "sponsor"}: + if name in {"author", "bdfl-delegate", "pep-delegate", "sponsor"}: # mask emails for node in para: - if isinstance(node, nodes.reference): - pep_num = pep if name == "discussions-to" else None - node.replace_self(_mask_email(node, pep_num)) + if not isinstance(node, nodes.reference): + continue + node.replace_self(_mask_email(node)) + elif name in {"discussions-to", "resolution", "post-history"}: + # Prettify mailing list and Discourse links + for node in para: + if (not isinstance(node, nodes.reference) + or not node["refuri"]): + continue + # Have known mailto links link to their main list pages + if node["refuri"].lower().startswith("mailto:"): + node["refuri"] = _generate_list_url(node["refuri"]) + parts = node["refuri"].lower().split("/") + if len(parts) <= 2 or parts[2] not in LINK_PRETTIFIERS: + continue + pretty_title = _make_link_pretty(str(node["refuri"])) + if name == "post-history": + node["reftitle"] = pretty_title + else: + node[0] = nodes.Text(pretty_title) elif name in {"replaces", "superseded-by", "requires"}: # replace PEP numbers with normalised list of links to PEPs new_body = [] - for ref_pep in re.split(r",?\s+", body.astext()): - new_body += [nodes.reference("", ref_pep, refuri=config.pep_url.format(int(ref_pep)))] - new_body += [nodes.Text(", ")] + for pep_str in re.split(r",?\s+", body.astext()): + target = self.document.settings.pep_url.format(int(pep_str)) + if self.document.settings.builder == "dirhtml": + target = f"../{target}" + new_body += [nodes.reference("", pep_str, refuri=target), nodes.Text(", ")] para[:] = new_body[:-1] # drop trailing space + elif name == "topic": + new_body = [] + for topic_name in body.astext().split(","): + if topic_name: + target = f"topic/{topic_name.lower().strip()}" + if self.document.settings.builder == "html": + target = f"{target}.html" + else: + target = f"../{target}/" + new_body += [ + nodes.reference("", topic_name, refuri=target), + nodes.Text(", "), + ] + if new_body: + para[:] = new_body[:-1] # Drop trailing space/comma + elif name == "status": + para[:] = [ + nodes.abbreviation( + body.astext(), + body.astext(), + explanation=_abbreviate_status(body.astext()), + ) + ] + elif name == "type": + para[:] = [ + nodes.abbreviation( + body.astext(), + body.astext(), + explanation=_abbreviate_type(body.astext()), + ) + ] elif name in {"last-modified", "content-type", "version"}: # Mark unneeded fields fields_to_remove.append(field) + # Remove any trailing commas and whitespace in the headers + if para and isinstance(para[-1], nodes.Text): + last_node = para[-1] + if last_node.astext().strip() == ",": + last_node.parent.remove(last_node) + else: + para[-1] = last_node.rstrip().rstrip(",") + # Remove unneeded fields for field in fields_to_remove: field.parent.remove(field) -def _mask_email(ref: nodes.reference, pep_num: int | None = None) -> nodes.reference: - """Mask the email address in `ref` and return a replacement node. +def _generate_list_url(mailto: str) -> str: + list_name_domain = mailto.lower().removeprefix("mailto:").strip() + list_name = list_name_domain.split("@")[0] - `ref` is returned unchanged if it contains no email address. + if list_name_domain.endswith("@googlegroups.com"): + return f"https://groups.google.com/g/{list_name}" - If given an email not explicitly whitelisted, process it such that - `user@host` -> `user at host`. + if not list_name_domain.endswith("@python.org"): + return mailto - If given a PEP number `pep_num`, add a default email subject. + # Active lists not yet on Mailman3; this URL will redirect if/when they are + if list_name in {"csv", "db-sig", "doc-sig", "python-list", "web-sig"}: + return f"https://mail.python.org/mailman/listinfo/{list_name}" + # Retired lists that are closed for posting, so only the archive matters + if list_name in {"import-sig", "python-3000"}: + return f"https://mail.python.org/pipermail/{list_name}/" + # The remaining lists (and any new ones) are all on Mailman3/Hyperkitty + return f"https://mail.python.org/archives/list/{list_name}@python.org/" - """ - if "refuri" not in ref or not ref["refuri"].startswith("mailto:"): - return ref - non_masked_addresses = {"peps@python.org", "python-list@python.org", "python-dev@python.org"} - if ref["refuri"].removeprefix("mailto:").strip() not in non_masked_addresses: - ref[0] = nodes.raw("", ref[0].replace("@", " at "), format="html") - if pep_num is None: - return ref[0] # return email text without mailto link - ref["refuri"] += f"?subject=PEP%20{pep_num}" - return ref + +def _process_list_url(parts: list[str]) -> tuple[str, str]: + item_type = "list" + + # HyperKitty (Mailman3) archive structure is + # https://mail.python.org/archives/list/<list_name>/thread/<id> + if "archives" in parts: + list_name = ( + parts[parts.index("archives") + 2].removesuffix("@python.org")) + if len(parts) > 6 and parts[6] in {"message", "thread"}: + item_type = parts[6] + + # Mailman3 list info structure is + # https://mail.python.org/mailman3/lists/<list_name>.python.org/ + elif "mailman3" in parts: + list_name = ( + parts[parts.index("mailman3") + 2].removesuffix(".python.org")) + + # Pipermail (Mailman) archive structure is + # https://mail.python.org/pipermail/<list_name>/<month>-<year>/<id> + elif "pipermail" in parts: + list_name = parts[parts.index("pipermail") + 1] + item_type = "message" if len(parts) > 6 else "list" + + # Mailman listinfo structure is + # https://mail.python.org/mailman/listinfo/<list_name> + elif "listinfo" in parts: + list_name = parts[parts.index("listinfo") + 1] + + # Not a link to a mailing list, message or thread + else: + raise ValueError( + f"{'/'.join(parts)} not a link to a list, message or thread") + + return list_name, item_type + + +def _process_discourse_url(parts: list[str]) -> tuple[str, str]: + item_name = "discourse" + + if len(parts) < 5 or ("t" not in parts and "c" not in parts): + raise ValueError( + f"{'/'.join(parts)} not a link to a Discourse thread or category") + + first_subpart = parts[4] + has_title = not first_subpart.isnumeric() + + if "t" in parts: + item_type = "message" if len(parts) > (5 + has_title) else "thread" + elif "c" in parts: + item_type = "category" + if has_title: + item_name = f"{first_subpart.replace('-', ' ')} {item_name}" + + return item_name, item_type + + +# Domains supported for pretty URL parsing +LINK_PRETTIFIERS = { + "mail.python.org": _process_list_url, + "discuss.python.org": _process_discourse_url, +} + + +def _process_pretty_url(url: str) -> tuple[str, str]: + parts = url.lower().strip().strip("/").split("/") + try: + item_name, item_type = LINK_PRETTIFIERS[parts[2]](parts) + except KeyError as error: + raise ValueError( + f"{url} not a link to a recognized domain to prettify") from error + item_name = item_name.title().replace("Sig", "SIG").replace("Pep", "PEP") + return item_name, item_type + + +def _make_link_pretty(url: str) -> str: + item_name, item_type = _process_pretty_url(url) + return f"{item_name} {item_type}" + + +def _abbreviate_status(status: str) -> str: + if status in SPECIAL_STATUSES: + status = SPECIAL_STATUSES[status] + + try: + return ABBREVIATED_STATUSES[status] + except KeyError: + raise PEPParsingError(f"Unknown status: {status}") + + +def _abbreviate_type(type_: str) -> str: + try: + return ABBREVIATED_TYPES[type_] + except KeyError: + raise PEPParsingError(f"Unknown type: {type_}") diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_references.py b/pep_sphinx_extensions/pep_processor/transforms/pep_references.py new file mode 100644 index 000000000..1e00e84cb --- /dev/null +++ b/pep_sphinx_extensions/pep_processor/transforms/pep_references.py @@ -0,0 +1,36 @@ +from pathlib import Path + +from docutils import nodes +from docutils import transforms + + +class PEPReferenceRoleTitleText(transforms.Transform): + """Add title text of document titles to reference role references.""" + + default_priority = 730 + + def apply(self) -> None: + if not Path(self.document["source"]).match("pep-*"): + return # not a PEP file, exit early + for node in self.document.findall(nodes.reference): + if "_title_tuple" not in node: + continue + + # get pep number and section target (fragment) + pep_num, fragment = node.attributes.pop("_title_tuple") + filename = f"pep-{pep_num:0>4}" + + # Cache target_ids + env = self.document.settings.env + try: + target_ids = env.document_ids[filename] + except KeyError: + env.document_ids[filename] = target_ids = env.get_doctree(filename).ids + + # Create title text string. We hijack the 'reftitle' attribute so + # that we don't have to change things in the HTML translator + node["reftitle"] = env.titles[filename].astext() + try: + node["reftitle"] += f" § {target_ids[fragment][0].astext()}" + except KeyError: + pass diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_title.py b/pep_sphinx_extensions/pep_processor/transforms/pep_title.py index 14e1190aa..de3ce4466 100644 --- a/pep_sphinx_extensions/pep_processor/transforms/pep_title.py +++ b/pep_sphinx_extensions/pep_processor/transforms/pep_title.py @@ -22,13 +22,19 @@ class PEPTitle(transforms.Transform): pep_header_details = {} # Iterate through the header fields, which are the first section of the document + desired_fields = {"PEP", "Title"} + fields_to_remove = [] for field in self.document[0]: # Hold details of the attribute's tag against its details row_attributes = {sub.tagname: sub.rawsource for sub in field} pep_header_details[row_attributes["field_name"]] = row_attributes["field_body"] + # Store the redundant fields in the table for removal + if row_attributes["field_name"] in desired_fields: + fields_to_remove.append(field) + # We only need the PEP number and title - if pep_header_details.keys() >= {"PEP", "Title"}: + if pep_header_details.keys() >= desired_fields: break # Create the title string for the PEP @@ -46,6 +52,10 @@ class PEPTitle(transforms.Transform): pep_title_node.extend(document_children) self.document.note_implicit_target(pep_title_node, pep_title_node) + # Remove the now-redundant fields + for field in fields_to_remove: + field.parent.remove(field) + def _line_to_nodes(text: str) -> list[nodes.Node]: """Parse RST string to nodes.""" diff --git a/pep_sphinx_extensions/pep_processor/transforms/pep_zero.py b/pep_sphinx_extensions/pep_processor/transforms/pep_zero.py index b638dbbb8..bebe50621 100644 --- a/pep_sphinx_extensions/pep_processor/transforms/pep_zero.py +++ b/pep_sphinx_extensions/pep_processor/transforms/pep_zero.py @@ -1,74 +1,34 @@ +from __future__ import annotations + from docutils import nodes from docutils import transforms -from docutils.transforms import peps - -from pep_sphinx_extensions import config class PEPZero(transforms.Transform): """Schedule PEP 0 processing.""" - # Run during sphinx post processing + # Run during sphinx post-processing default_priority = 760 def apply(self) -> None: - # Walk document and then remove this node - visitor = PEPZeroSpecial(self.document) - self.document.walk(visitor) + # Walk document and mask email addresses if present. + for reference_node in self.document.findall(nodes.reference): + reference_node.replace_self(_mask_email(reference_node)) + # Remove this node self.startnode.parent.remove(self.startnode) -class PEPZeroSpecial(nodes.SparseNodeVisitor): - """Perform the special processing needed by PEP 0: +def _mask_email(ref: nodes.reference) -> nodes.reference: + """Mask the email address in `ref` and return a replacement node. - - Mask email addresses. - - Link PEP numbers in the second column of 4-column tables to the PEPs themselves. + `ref` is returned unchanged if it contains no email address. + + If given an email not explicitly whitelisted, process it such that + `user@host` -> `user at host`. + + The returned node has no refuri link attribute. """ - - def __init__(self, document: nodes.document): - super().__init__(document) - self.pep_table: int = 0 - self.entry: int = 0 - - def unknown_visit(self, node: nodes.Node) -> None: - """No processing for undefined node types.""" - pass - - @staticmethod - def visit_reference(node: nodes.reference) -> None: - """Mask email addresses if present.""" - node.replace_self(peps.mask_email(node)) - - @staticmethod - def visit_field_list(node: nodes.field_list) -> None: - """Skip PEP headers.""" - if "rfc2822" in node["classes"]: - raise nodes.SkipNode - - def visit_tgroup(self, node: nodes.tgroup) -> None: - """Set column counter and PEP table marker.""" - self.pep_table = node["cols"] == 4 - self.entry = 0 # reset column number - - def visit_colspec(self, node: nodes.colspec) -> None: - self.entry += 1 - if self.pep_table and self.entry == 2: - node["classes"].append("num") - - def visit_row(self, _node: nodes.row) -> None: - self.entry = 0 # reset column number - - def visit_entry(self, node: nodes.entry) -> None: - self.entry += 1 - if self.pep_table and self.entry == 2 and len(node) == 1: - node["classes"].append("num") - # if this is the PEP number column, replace the number with a link to the PEP - para = node[0] - if isinstance(para, nodes.paragraph) and len(para) == 1: - pep_str = para.astext() - try: - ref = config.pep_url.format(int(pep_str)) - para[0] = nodes.reference(pep_str, pep_str, refuri=ref) - except ValueError: - pass + if not ref.get("refuri", "").startswith("mailto:"): + return ref + return nodes.raw("", ref[0].replace("@", " at "), format="html") diff --git a/pep_sphinx_extensions/pep_theme/static/colour_scheme.js b/pep_sphinx_extensions/pep_theme/static/colour_scheme.js new file mode 100644 index 000000000..ee94274d2 --- /dev/null +++ b/pep_sphinx_extensions/pep_theme/static/colour_scheme.js @@ -0,0 +1,35 @@ +// Handle setting and changing the site's color scheme (light/dark) + +"use strict"; + +const prefersDark = window.matchMedia("(prefers-color-scheme: dark)") + +const getColourScheme = () => document.documentElement.dataset.colour_scheme +const setColourScheme = (colourScheme = getColourScheme()) => { + document.documentElement.dataset.colour_scheme = colourScheme + localStorage.setItem("colour_scheme", colourScheme) + setPygments(colourScheme) +} + +// Map system theme to a cycle of steps +const cycles = { + dark: ["auto", "light", "dark"], // auto (dark) → light → dark + light: ["auto", "dark", "light"], // auto (light) → dark → light +} + +const nextColourScheme = (colourScheme = getColourScheme()) => { + const cycle = cycles[prefersDark.matches ? "dark" : "light"] + return cycle[(cycle.indexOf(colourScheme) + 1) % cycle.length] +} + +const setPygments = (colourScheme = getColourScheme()) => { + const pygmentsDark = document.getElementById("pyg-dark") + const pygmentsLight = document.getElementById("pyg-light") + pygmentsDark.disabled = colourScheme === "light" + pygmentsLight.disabled = colourScheme === "dark" + pygmentsDark.media = colourScheme === "auto" ? "(prefers-color-scheme: dark)" : "" + pygmentsLight.media = colourScheme === "auto" ? "(prefers-color-scheme: light)" : "" +} + +// Update Pygments state (the page theme is initialised inline, see page.html) +document.addEventListener("DOMContentLoaded", () => setColourScheme()) diff --git a/pep_sphinx_extensions/pep_theme/static/doctools.js b/pep_sphinx_extensions/pep_theme/static/doctools.js deleted file mode 100644 index 5676a8ff3..000000000 --- a/pep_sphinx_extensions/pep_theme/static/doctools.js +++ /dev/null @@ -1,5 +0,0 @@ -/* JavaScript utilities for all documentation. */ - -// Footnote fixer -document.querySelectorAll("span.brackets").forEach(el => el.innerHTML = "[" + el.innerHTML + "]") -document.querySelectorAll("a.brackets").forEach(el => el.innerHTML = "[" + el.innerHTML + "]") diff --git a/pep_sphinx_extensions/pep_theme/static/mq.css b/pep_sphinx_extensions/pep_theme/static/mq.css index c800e5d92..7977585e6 100644 --- a/pep_sphinx_extensions/pep_theme/static/mq.css +++ b/pep_sphinx_extensions/pep_theme/static/mq.css @@ -1,7 +1,12 @@ @charset "UTF-8"; + /* Media Queries */ -@media (max-width: 32.5em) { - /* Reduce padding & margins for the smallest screens */ + +/* Reduce padding & margins for smaller screens */ +@media (max-width: 40em) { + section#pep-page-section { + padding: 1rem; + } section#pep-page-section > header > h1 { padding-right: 0; border-right: none; @@ -12,25 +17,25 @@ nav#pep-sidebar { display: none; } + pre { + font-size: 0.8175rem; + } table th, table td { padding: 0 0.1rem; } } -@media (min-width: 32.5em) { + +@media (min-width: 40em) { section#pep-page-section { - max-width: 40em; - width: 100%; + display: table; margin: 0 auto; - padding: .5rem 1rem 0; - } -} -@media (min-width: 54em) { - section#pep-page-section { max-width: 75em; + padding: 0.5rem 1rem 0; + width: 100%; } section#pep-page-section > article { - max-width: 40em; + max-width: 37em; width: 74%; float: right; margin-right: 0; @@ -41,10 +46,15 @@ float: left; margin-right: 2%; } + /* Make less prominent when sidebar ToC is available */ + details > summary { + font-size: 1rem; + width: max-content; + } } @media (min-width: 60em) { section#pep-page-section > article { - max-width: none; + max-width: 56em; padding-left: 3.2%; padding-right: 3.2%; } diff --git a/pep_sphinx_extensions/pep_theme/static/sticky_banner.js b/pep_sphinx_extensions/pep_theme/static/sticky_banner.js new file mode 100644 index 000000000..cc2ceb3dd --- /dev/null +++ b/pep_sphinx_extensions/pep_theme/static/sticky_banner.js @@ -0,0 +1,28 @@ +"use strict"; + +// Inject a style element into the document head that adds scroll-margin-top to +// all elements with an id attribute. This is used to offset the scroll position +// when clicking on a link to an element with an id attribute. The offset is +// equal to the height of the sticky banner. +document.addEventListener("DOMContentLoaded", () => { + const stickyBanners = document.getElementsByClassName("sticky-banner"); + if (!stickyBanners.length) { + return; + } + + const stickyBanner = stickyBanners[0]; + const node = document.createElement("style"); + node.id = "sticky-banner-style"; + document.head.appendChild(node); + + function adjustBannerMargin() { + const text = document.createTextNode( + ":target { scroll-margin-top: " + stickyBanner.offsetHeight + "px; }" + ); + node.replaceChildren(text); + } + + adjustBannerMargin(); + document.addEventListener("resize", adjustBannerMargin); + document.addEventListener("load", adjustBannerMargin); +}); diff --git a/pep_sphinx_extensions/pep_theme/static/style.css b/pep_sphinx_extensions/pep_theme/static/style.css index 29d90b350..70e1cbf08 100644 --- a/pep_sphinx_extensions/pep_theme/static/style.css +++ b/pep_sphinx_extensions/pep_theme/static/style.css @@ -1,114 +1,120 @@ @charset "UTF-8"; -/* Styles for PEPs -Colours: -white: - background - footnotes/references vertical border -#333 - body text -#888 - blockquote left line - header breadcrumbs separator - link underline (hovered/focused) -#ccc: - scrollbar -#ddd - header bottom border - horizontal rule - table vertical border -#eee: - link underline - table rows & top/bottom border - PEP header rows - footnotes/references rows - admonition note background -#f8f8f8: - inline code background +/* Styles for PEPs */ -#0072aa: - links -# fee: - admonition warning background +/* + * `initial` works like undefined variables, so `var(initial, x)` will resolve to `x`. + * A space means an empty value, so `var( , x) y` will resolve to `y`. + */ +@media (prefers-color-scheme: dark) { + :root { + --light: ; + --dark: initial; + } +} -*/ +@media (prefers-color-scheme: light) { + :root { + --dark: ; + --light: initial; + } +} + +:root[data-colour_scheme="dark"] { + --light: ; + --dark: initial; +} + +:root[data-colour_scheme="light"] { + --dark: ; + --light: initial; +} + +/* Set master colours */ +:root { + --colour-background: var(--light, white) var(--dark, #111); + --colour-background-accent-strong: var(--light, #ccc) var(--dark, #444); + --colour-background-accent-medium: var(--light, #ddd) var(--dark, #333); + --colour-background-accent-light: var(--light, #eee) var(--dark, #222); + --colour-text: var(--light, #333) var(--dark, #ccc); + --colour-text-strong: var(--light, #222) var(--dark, #ddd); + --colour-links: var(--light, #069) var(--dark, #8bf); + --colour-links-light: var(--light, #057) var(--dark, #acf); + --colour-scrollbar: var(--light, #ccc) var(--dark, #333); + --colour-rule-strong: var(--light, #888) var(--dark, #777); + --colour-rule-light: var(--light, #ddd) var(--dark, #222); + --colour-inline-code-bg: var(--light, #eee) var(--dark, #333); + --colour-inline-code-text: var(--light, #222) var(--dark, #ccc); + --colour-error: var(--light, #faa) var(--dark, #800); + --colour-warning: var(--light, #fca) var(--dark, #840); + --colour-caution: var(--light, #ffa) var(--dark, #550); + --colour-attention: var(--light, #bdf) var(--dark, #045); + --colour-tip: var(--light, #bfc) var(--dark, #041); +} + +img.invert-in-dark-mode { + filter: var(--dark, invert(1) hue-rotate(.5turn)); +} /* Set master rules */ * {box-sizing: border-box} +:root {color-scheme: light dark} html { overflow-y: scroll; - -webkit-font-smoothing: antialiased; - margin: 0; - line-height: 1.4rem; - font-weight: normal; + line-height: 1.5; font-size: 1rem; - font-family: "Source Sans Pro", Arial, sans-serif; + font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, noto, arial, sans-serif; } body { margin: 0; - color: #333; - background-color: white; + color: var(--colour-text); + background-color: var(--colour-background); } section#pep-page-section { - padding: 0.25rem 0.25rem 0; - display: table; + padding: 0.25rem; } -/* Reduce margin sizes for body text */ -p {margin: .5rem 0} - /* Header rules */ -h1.page-title { - line-height: 2.3rem; +h1 { font-size: 2rem; font-weight: bold; - margin-top: 2rem; - margin-bottom: 1.5rem; } h2 { font-size: 1.6rem; font-weight: bold; - margin-top: 1rem; - margin-bottom: .5rem; } h3 { font-size: 1.4rem; font-weight: normal; - margin-top: 1rem; - margin-bottom: 0.5rem; } h4 { font-size: 1.2rem; font-weight: normal; - margin-top: .5rem; - margin-bottom: 0; } h5, h6 { font-size: 1rem; font-weight: bold; - margin-top: 0; - margin-bottom: 0; } /* Anchor link rules */ a, a:active, a:visited { - color: #0072aa; - text-decoration-color: #eee; + color: var(--colour-links); display: inline; + overflow-wrap: anywhere; + text-decoration-color: var(--colour-background-accent-strong); } a:hover, a:focus { - text-decoration-color: #888; + text-decoration-color: var(--colour-rule-strong); } /* Blockquote rules */ blockquote { font-style: italic; - border-left: 1px solid #888; - margin: .5rem; + border-left: 1px solid var(--colour-rule-strong); padding: .5rem 1rem; } blockquote em { @@ -120,20 +126,37 @@ cite { } /* Code rules (code literals and Pygments highlighting blocks) */ -pre, -code { - font-family: ui-monospace, "Cascadia Mono", "Segoe UI Mono", "DejaVu Sans Mono", Consolas, monospace; - white-space: pre-wrap; - word-wrap: break-word; +code, +pre { + font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; + font-size: 0.875rem; -webkit-hyphens: none; hyphens: none; } +code { + overflow-wrap: anywhere; +} code.literal { + background-color: var(--colour-inline-code-bg); + color: var(--colour-inline-code-text); font-size: .8em; - background-color: #f8f8f8; + padding: 1px 2px 1px; } pre { + overflow-x: auto; padding: .5rem .75rem; + white-space: pre; +} + +/* Contents rules */ +details > summary { + cursor: pointer; + font-size: 1.6rem; + font-weight: bold; + margin-bottom: 1em; +} +details > summary:hover { + text-decoration: underline; } /* Definition list rules */ @@ -141,16 +164,15 @@ dl dt { font-weight: bold; } dl dd { - margin: 0; + margin-bottom: 0.5rem; } /* Horizontal rule rule */ hr { border: 0; - border-top: 1px solid #ddd; - margin: 1.75rem 0; + border-top: 1px solid var(--colour-rule-light); } -/*Image rules */ +/* Image rules */ img { max-width: 100%; } @@ -160,13 +182,6 @@ a img { } /* List rules */ -ul, -ol { - padding: 0; - margin: 0 0 0 1.5rem; -} -ul {list-style: square} -ol.arabic {list-style: decimal} ol.loweralpha {list-style: lower-alpha} ol.upperalpha {list-style: upper-alpha} ol.lowerroman {list-style: lower-roman} @@ -184,37 +199,64 @@ sup {top: -0.5em} sub {bottom: -0.25em} /* Table rules */ +div.table-wrapper { + overflow-x: auto; +} table { width: 100%; border-collapse: collapse; - border-top: 1px solid #eee; - border-bottom: 1px solid #eee; + border: 1px solid var(--colour-background-accent-strong); } table caption { margin: 1rem 0 .75rem; } -table tbody tr:nth-of-type(odd) { - background-color: #eee; +table thead tr { + background-color: var(--colour-background-accent-medium); + color: var(--colour-text-strong); +} +table tbody tr { + border-top: 1px solid var(--colour-background-accent-strong); } table th, table td { text-align: left; padding: 0.25rem 0.5rem 0.2rem; } +table.pep-zero-table tr td:nth-child(1), +table.pep-zero-table tr td:nth-child(2) { + white-space: nowrap; +} +table th + th, table td + td { - border-left: 1px solid #ddd; + border-left: 1px solid var(--colour-background-accent-strong); +} +/* Common column widths for PEP status tables */ +table.pep-zero-table tr td:nth-child(1) { + width: 5.5%; +} +table.pep-zero-table tr td:nth-child(2) { + width: 6.5%; +} +table.pep-zero-table tr td:nth-child(3), +table.pep-zero-table tr td:nth-child(4){ + width: 44%; +} +/* Authors & Sponsors table */ +#authors-owners table td, +#authors-owners table th { + width: 50%; } /* Breadcrumbs rules */ section#pep-page-section > header { - border-bottom: 1px solid #ddd; + border-bottom: 1px solid var(--colour-rule-light); } section#pep-page-section > header > h1 { font-size: 1.1rem; margin: 0; display: inline-block; padding-right: .6rem; - border-right: 1px solid #888; + border-right: 1px solid var(--colour-rule-strong); } ul.breadcrumbs { margin: 0; @@ -229,19 +271,57 @@ ul.breadcrumbs a { text-decoration: none; } +/* Dark mode toggle rules */ +#colour-scheme-cycler { + background: transparent; + border: none; + padding: 0; + cursor: pointer; + width: 1.2rem; + height: 1.2rem; + float: right; + transform: translate(0, 50%); +} +#colour-scheme-cycler svg { + color: var(--colour-rule-strong); + height: 1.2rem; + width: 1.2rem; + display: none; +} +:root[data-colour_scheme="auto"] #colour-scheme-cycler svg.colour-scheme-icon-when-auto {display: initial} +:root[data-colour_scheme="dark"] #colour-scheme-cycler svg.colour-scheme-icon-when-dark {display: initial} +:root[data-colour_scheme="light"] #colour-scheme-cycler svg.colour-scheme-icon-when-light {display: initial} + /* Admonitions rules */ -div.note, -div.warning { - padding: 0.5rem 0.75rem; - margin-top: 1rem; +div.admonition { + background-color: var(--colour-background-accent-medium); margin-bottom: 1rem; + margin-top: 1rem; + padding: 0.5rem 0.75rem; } -div.note { - background-color: #eee; +div.admonition a { + color: var(--colour-links-light); +} + +div.danger, +div.error { + background-color: var(--colour-error); } div.warning { - background-color: #fee; + background-color: var(--colour-warning); } +div.attention, +div.caution { + background-color: var(--colour-caution); +} +div.important { + background-color: var(--colour-attention); +} +div.hint, +div.tip { + background-color: var(--colour-tip); +} + p.admonition-title { font-weight: bold; } @@ -251,42 +331,81 @@ dl.rfc2822, dl.footnote { display: grid; grid-template-columns: fit-content(30%) auto; - line-height: 1.875; width: 100%; - border-top: 1px solid #eee; - border-bottom: 1px solid #eee; } -dl.rfc2822 > dt, dl.rfc2822 > dd, -dl.footnote > dt, dl.footnote > dd { +dl.footnote { + border-top: 1px solid var(--colour-rule-strong); + line-height: 1.875; +} +dl.rfc2822 > dt, +dl.rfc2822 > dd { + padding: .1rem .3rem .1rem; +} +dl.footnote > dt, +dl.footnote > dd { padding: .25rem .5rem .2rem; + border-bottom: 1px solid var(--colour-rule-strong); } -dl.rfc2822 > dt:nth-of-type(even), dl.rfc2822 > dd:nth-of-type(even), -dl.footnote > dt:nth-of-type(even), dl.footnote > dd:nth-of-type(even) { - background-color: #eee; +dl.rfc2822 > dt { + text-align: right; } dl.footnote > dt { font-weight: normal; - border-right: 1px solid white; + border-right: 1px solid var(--colour-background); +} +dl.rfc2822 > dd, +dl.footnote > dd { + margin: 0; } /* Sidebar formatting */ -nav#pep-sidebar { - overflow-y: scroll; +#pep-sidebar { + overflow-y: auto; position: sticky; top: 0; height: 100vh; - scrollbar-width: thin; /* CSS Standards, not *yet* widely supported */ - scrollbar-color: #ccc transparent; } -nav#pep-sidebar::-webkit-scrollbar {width: 6px} -nav#pep-sidebar::-webkit-scrollbar-track {background: transparent} -nav#pep-sidebar::-webkit-scrollbar-thumb {background: #ccc} -nav#pep-sidebar > h2 { +#pep-sidebar > h2 { font-size: 1.4rem; } -nav#pep-sidebar ul { +#contents ol, +#contents ul, +#pep-sidebar ol, +#pep-sidebar ul { + padding: 0; + margin: 0 0 0 1.5rem; +} +#pep-sidebar ul { + font-size: .9rem; margin-left: 1rem; } -nav#pep-sidebar ul a { +#pep-sidebar ul a { text-decoration: none; } +#source { + padding-bottom: 2rem; + font-weight: bold; +} + +.reference.external > strong { + font-weight: normal; /* Fix strong links for :pep: and :rfc: roles */ +} + +.visually-hidden { + position: absolute !important; + width: 1px !important; + height: 1px !important; + padding: 0 !important; + margin: -1px !important; + overflow: hidden !important; + clip-path: polygon(0px 0px, 0px 0px,0px 0px, 0px 0px) !important; + white-space: nowrap !important; + border: 0 !important; +} + +/* Sticky banners */ +.sticky-banner { + top: 0; + position: sticky; + z-index: 1; +} diff --git a/pep_sphinx_extensions/pep_theme/static/wrap_tables.js b/pep_sphinx_extensions/pep_theme/static/wrap_tables.js new file mode 100644 index 000000000..70c0a3a4b --- /dev/null +++ b/pep_sphinx_extensions/pep_theme/static/wrap_tables.js @@ -0,0 +1,30 @@ +// Wrap the tables in PEP bodies in a div, to allow for responsive scrolling + +"use strict"; + +const pepContentId = "pep-content"; + + +// Wrap passed table element in wrapper divs +function wrapTable (table) { + const wrapper = document.createElement("div"); + wrapper.classList.add("table-wrapper"); + table.parentNode.insertBefore(wrapper, table); + wrapper.appendChild(table); +} + + +// Wrap all tables in the PEP content in wrapper divs +function wrapPepContentTables () { + const pepContent = document.getElementById(pepContentId); + const bodyTables = pepContent.getElementsByTagName("table"); + Array.from(bodyTables).forEach(wrapTable); +} + + +// Wrap the tables as soon as the DOM is loaded +document.addEventListener("DOMContentLoaded", () => { + if (document.getElementById(pepContentId)) { + wrapPepContentTables(); + } +}) diff --git a/pep_sphinx_extensions/pep_theme/templates/page.html b/pep_sphinx_extensions/pep_theme/templates/page.html index 38fb35451..46be8c5bb 100644 --- a/pep_sphinx_extensions/pep_theme/templates/page.html +++ b/pep_sphinx_extensions/pep_theme/templates/page.html @@ -1,26 +1,40 @@ {# Master template for simple pages (e.g. RST files) #} <!DOCTYPE html> -<html lang="en-GB"> +<html lang="en"> <head> - <meta charset="utf-8" /> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <title>{{ title + " | "|safe + docstitle }} - - - - - - + + + + {{ title + " | peps.python.org"|safe }} + + + + + + + + + {% include "partials/icons.html" %} +

Python Enhancement Proposals

  • Python »
  • PEP Index »
    • -
  • {{ title }}
    • +
  • {{ title.split("–")[0].strip() }}
+
- + + + diff --git a/pep_sphinx_extensions/pep_theme/templates/partials/icons.html b/pep_sphinx_extensions/pep_theme/templates/partials/icons.html new file mode 100644 index 000000000..1a008fc24 --- /dev/null +++ b/pep_sphinx_extensions/pep_theme/templates/partials/icons.html @@ -0,0 +1,34 @@ +{# Adapted from Just the Docs → Furo #} + + + Following system colour scheme + + + + + + + Selected dark colour scheme + + + + + + + Selected light colour scheme + + + + + + + + + + + + + diff --git a/pep_sphinx_extensions/pep_theme/theme.conf b/pep_sphinx_extensions/pep_theme/theme.conf index 8150ef720..bf410226a 100644 --- a/pep_sphinx_extensions/pep_theme/theme.conf +++ b/pep_sphinx_extensions/pep_theme/theme.conf @@ -2,3 +2,4 @@ # Theme options inherit = none pygments_style = tango +pygments_dark_style = native diff --git a/pep_sphinx_extensions/pep_zero_generator/author.py b/pep_sphinx_extensions/pep_zero_generator/author.py deleted file mode 100644 index 22299b056..000000000 --- a/pep_sphinx_extensions/pep_zero_generator/author.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -from typing import NamedTuple - - -class _Name(NamedTuple): - mononym: str = None - forename: str = None - surname: str = None - suffix: str = None - - -class Author(NamedTuple): - """Represent PEP authors.""" - last_first: str # The author's name in Surname, Forename, Suffix order. - nick: str # Author's nickname for PEP tables. Defaults to surname. - email: str # The author's email address. - - -def parse_author_email(author_email_tuple: tuple[str, str], authors_overrides: dict[str, dict[str, str]]) -> Author: - """Parse the name and email address of an author.""" - name, email = author_email_tuple - _first_last = name.strip() - email = email.lower() - - if _first_last in authors_overrides: - name_dict = authors_overrides[_first_last] - last_first = name_dict["Surname First"] - nick = name_dict["Name Reference"] - return Author(last_first, nick, email) - - name_parts = _parse_name(_first_last) - if name_parts.mononym is not None: - return Author(name_parts.mononym, name_parts.mononym, email) - - if name_parts.surname[1] == ".": - # Add an escape to avoid docutils turning `v.` into `22.`. - name_parts.surname = f"\\{name_parts.surname}" - - if name_parts.suffix: - last_first = f"{name_parts.surname}, {name_parts.forename}, {name_parts.suffix}" - return Author(last_first, name_parts.surname, email) - - last_first = f"{name_parts.surname}, {name_parts.forename}" - return Author(last_first, name_parts.surname, email) - - -def _parse_name(full_name: str) -> _Name: - """Decompose a full name into parts. - - If a mononym (e.g, 'Aahz') then return the full name. If there are - suffixes in the name (e.g. ', Jr.' or 'II'), then find and extract - them. If there is a middle initial followed by a full stop, then - combine the following words into a surname (e.g. N. Vander Weele). If - there is a leading, lowercase portion to the last name (e.g. 'van' or - 'von') then include it in the surname. - - """ - possible_suffixes = {"Jr", "Jr.", "II", "III"} - - pre_suffix, _, raw_suffix = full_name.partition(",") - name_parts = pre_suffix.strip().split(" ") - num_parts = len(name_parts) - suffix = raw_suffix.strip() - - if num_parts == 0: - raise ValueError("Name is empty!") - elif num_parts == 1: - return _Name(mononym=name_parts[0], suffix=suffix) - elif num_parts == 2: - return _Name(forename=name_parts[0].strip(), surname=name_parts[1], suffix=suffix) - - # handles rogue uncaught suffixes - if name_parts[-1] in possible_suffixes: - suffix = f"{name_parts.pop(-1)} {suffix}".strip() - - # handles von, van, v. etc. - if name_parts[-2].islower(): - forename = " ".join(name_parts[:-2]).strip() - surname = " ".join(name_parts[-2:]) - return _Name(forename=forename, surname=surname, suffix=suffix) - - # handles double surnames after a middle initial (e.g. N. Vander Weele) - elif any(s.endswith(".") for s in name_parts): - split_position = [i for i, x in enumerate(name_parts) if x.endswith(".")][-1] + 1 - forename = " ".join(name_parts[:split_position]).strip() - surname = " ".join(name_parts[split_position:]) - return _Name(forename=forename, surname=surname, suffix=suffix) - - # default to using the last item as the surname - else: - forename = " ".join(name_parts[:-1]).strip() - return _Name(forename=forename, surname=name_parts[-1], suffix=suffix) diff --git a/pep_sphinx_extensions/pep_zero_generator/constants.py b/pep_sphinx_extensions/pep_zero_generator/constants.py index 5b3ea5f6f..9ce1c4af5 100644 --- a/pep_sphinx_extensions/pep_zero_generator/constants.py +++ b/pep_sphinx_extensions/pep_zero_generator/constants.py @@ -19,8 +19,8 @@ STATUS_VALUES = { SPECIAL_STATUSES = { "April Fool!": STATUS_REJECTED, # See PEP 401 :) } -# Draft PEPs have no status displayed, Active shares a key with Accepted -HIDE_STATUS = {STATUS_DRAFT, STATUS_ACTIVE} +# Draft PEPs have no status displayed +HIDE_STATUS = {STATUS_DRAFT} # Dead PEP statuses DEAD_STATUSES = {STATUS_REJECTED, STATUS_WITHDRAWN, STATUS_SUPERSEDED} @@ -32,3 +32,32 @@ TYPE_STANDARDS = "Standards Track" TYPE_VALUES = {TYPE_STANDARDS, TYPE_INFO, TYPE_PROCESS} # Active PEPs can only be for Informational or Process PEPs. ACTIVE_ALLOWED = {TYPE_PROCESS, TYPE_INFO} + +# map of topic -> additional description +SUBINDICES_BY_TOPIC = { + "governance": """\ +These PEPs detail Python's governance, including governance model proposals +and selection, and the results of the annual steering council elections. + """, + "packaging": """\ +Packaging PEPs follow the `PyPA specification update process`_. +They are used to propose major additions or changes to the PyPA specifications. +The canonical, up-to-date packaging specifications can be found on the +`Python Packaging Authority`_ (PyPA) `specifications`_ page. + +.. _Python Packaging Authority: https://www.pypa.io/ +.. _specifications: https://packaging.python.org/en/latest/specifications/ +.. _PyPA specification update process: https://www.pypa.io/en/latest/specifications/#specification-update-process +""", + "release": """\ +A PEP is written to specify the release cycle for each feature release of Python. +See the `developer's guide`_ for more information. + +.. _developer's guide: https://devguide.python.org/devcycle/ +""", + "typing": """\ +Many recent PEPs propose changes to Python's static type system +or otherwise relate to type annotations. +They are listed here for reference. +""", +} diff --git a/pep_sphinx_extensions/pep_zero_generator/parser.py b/pep_sphinx_extensions/pep_zero_generator/parser.py index e5cec9ebb..193186c74 100644 --- a/pep_sphinx_extensions/pep_zero_generator/parser.py +++ b/pep_sphinx_extensions/pep_zero_generator/parser.py @@ -2,13 +2,10 @@ from __future__ import annotations +import dataclasses from email.parser import HeaderParser from pathlib import Path -import re -import textwrap -from typing import TYPE_CHECKING -from pep_sphinx_extensions.pep_zero_generator.author import parse_author_email from pep_sphinx_extensions.pep_zero_generator.constants import ACTIVE_ALLOWED from pep_sphinx_extensions.pep_zero_generator.constants import HIDE_STATUS from pep_sphinx_extensions.pep_zero_generator.constants import SPECIAL_STATUSES @@ -19,8 +16,12 @@ from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_STANDARDS from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_VALUES from pep_sphinx_extensions.pep_zero_generator.errors import PEPError -if TYPE_CHECKING: - from pep_sphinx_extensions.pep_zero_generator.author import Author + +@dataclasses.dataclass(order=True, frozen=True) +class _Author: + """Represent PEP authors.""" + full_name: str # The author's name. + email: str # The author's email address. class PEP: @@ -38,7 +39,7 @@ class PEP: # The required RFC 822 headers for all PEPs. required_headers = {"PEP", "Title", "Author", "Status", "Type", "Created"} - def __init__(self, filename: Path, authors_overrides: dict): + def __init__(self, filename: Path): """Init object from an open PEP file object. pep_file is full text of the PEP file, filename is path of the PEP file, author_lookup is author exceptions file @@ -89,7 +90,27 @@ class PEP: self.status: str = status # Parse PEP authors - self.authors: list[Author] = _parse_authors(self, metadata["Author"], authors_overrides) + self.authors: list[_Author] = _parse_author(metadata["Author"]) + if not self.authors: + raise _raise_pep_error(self, "no authors found", pep_num=True) + + # Topic (for sub-indices) + _topic = metadata.get("Topic", "").lower().split(",") + self.topic: set[str] = {topic for topic_raw in _topic if (topic := topic_raw.strip())} + + # Other headers + self.created = metadata["Created"] + self.discussions_to = metadata["Discussions-To"] + self.python_version = metadata["Python-Version"] + self.replaces = metadata["Replaces"] + self.requires = metadata["Requires"] + self.resolution = metadata["Resolution"] + self.superseded_by = metadata["Superseded-By"] + if metadata["Post-History"]: + # Squash duplicate whitespace + self.post_history = " ".join(metadata["Post-History"].split()) + else: + self.post_history = None def __repr__(self) -> str: return f"4} - {self.title}>" @@ -100,17 +121,46 @@ class PEP: def __eq__(self, other): return self.number == other.number - def details(self, *, title_length) -> dict[str, str | int]: + @property + def shorthand(self) -> str: + """Return reStructuredText tooltip for the PEP type and status.""" + type_code = self.pep_type[0].upper() + if self.status in HIDE_STATUS: + return f":abbr:`{type_code} ({self.pep_type}, {self.status})`" + status_code = self.status[0].upper() + return f":abbr:`{type_code}{status_code} ({self.pep_type}, {self.status})`" + + @property + def details(self) -> dict[str, str | int]: """Return the line entry for the PEP.""" return { - # how the type is to be represented in the index - "type": self.pep_type[0].upper(), "number": self.number, - "title": _title_abbr(self.title, title_length), - # how the status should be represented in the index - "status": " " if self.status in HIDE_STATUS else self.status[0].upper(), + "title": self.title, + # a tooltip representing the type and status + "shorthand": self.shorthand, # the author list as a comma-separated with only last names - "authors": ", ".join(author.nick for author in self.authors), + "authors": ", ".join(author.full_name for author in self.authors), + } + + @property + def full_details(self) -> dict[str, str | int]: + """Returns all headers of the PEP as a dict.""" + return { + "number": self.number, + "title": self.title, + "authors": ", ".join(author.full_name for author in self.authors), + "discussions_to": self.discussions_to, + "status": self.status, + "type": self.pep_type, + "topic": ", ".join(sorted(self.topic)), + "created": self.created, + "python_version": self.python_version, + "post_history": self.post_history, + "resolution": self.resolution, + "requires": self.requires, + "replaces": self.replaces, + "superseded_by": self.superseded_by, + "url": f"https://peps.python.org/pep-{self.number:0>4}/", } @@ -120,49 +170,27 @@ def _raise_pep_error(pep: PEP, msg: str, pep_num: bool = False) -> None: raise PEPError(msg, pep.filename) -def _parse_authors(pep: PEP, author_header: str, authors_overrides: dict) -> list[Author]: - """Parse Author header line""" - authors_and_emails = _parse_author(author_header) - if not authors_and_emails: - raise _raise_pep_error(pep, "no authors found", pep_num=True) - return [parse_author_email(author_tuple, authors_overrides) for author_tuple in authors_and_emails] +jr_placeholder = ",Jr" -author_angled = re.compile(r"(?P.+?) <(?P.+?)>(,\s*)?") -author_paren = re.compile(r"(?P.+?) \((?P.+?)\)(,\s*)?") -author_simple = re.compile(r"(?P[^,]+)(,\s*)?") - - -def _parse_author(data: str) -> list[tuple[str, str]]: +def _parse_author(data: str) -> list[_Author]: """Return a list of author names and emails.""" author_list = [] - for regex in (author_angled, author_paren, author_simple): - for match in regex.finditer(data): - # Watch out for suffixes like 'Jr.' when they are comma-separated - # from the name and thus cause issues when *all* names are only - # separated by commas. - match_dict = match.groupdict() - author = match_dict["author"] - if not author.partition(" ")[1] and author.endswith("."): - prev_author = author_list.pop() - author = ", ".join([prev_author, author]) - if "email" not in match_dict: - email = "" - else: - email = match_dict["email"] - author_list.append((author, email)) + data = (data.replace("\n", " ") + .replace(", Jr", jr_placeholder) + .rstrip().removesuffix(",")) + for author_email in data.split(", "): + if ' <' in author_email: + author, email = author_email.removesuffix(">").split(" <") + else: + author, email = author_email, "" - # If authors were found then stop searching as only expect one - # style of author citation. - if author_list: - break + author = author.strip() + if author == "": + raise ValueError("Name is empty!") + + author = author.replace(jr_placeholder, ", Jr") + email = email.lower() + author_list.append(_Author(author, email)) return author_list - - -def _title_abbr(title, title_length) -> str: - """Shorten the title to be no longer than the max title length.""" - if len(title) <= title_length: - return title - wrapped_title, *_excess = textwrap.wrap(title, title_length - 4) - return f"{wrapped_title} ..." diff --git a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py index 898ab2b0b..8fbf5cc7e 100644 --- a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py +++ b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py @@ -17,49 +17,56 @@ to allow it to be processed as normal. """ from __future__ import annotations -import csv +import json +import os from pathlib import Path -import re from typing import TYPE_CHECKING from pep_sphinx_extensions.pep_zero_generator import parser +from pep_sphinx_extensions.pep_zero_generator import subindices from pep_sphinx_extensions.pep_zero_generator import writer +from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC if TYPE_CHECKING: from sphinx.application import Sphinx from sphinx.environment import BuildEnvironment -def create_pep_zero(_: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None: - # Sphinx app object is unneeded by this function - +def _parse_peps(path: Path) -> list[parser.PEP]: # Read from root directory - path = Path(".") - - pep_zero_filename = "pep-0000" peps: list[parser.PEP] = [] - pep_pat = re.compile(r"pep-\d{4}") # Path.match() doesn't support regular expressions - - # AUTHOR_OVERRIDES.csv is an exception file for PEP0 name parsing - with open("AUTHOR_OVERRIDES.csv", "r", encoding="utf-8") as f: - authors_overrides = {} - for line in csv.DictReader(f): - full_name = line.pop("Overridden Name") - authors_overrides[full_name] = line for file_path in path.iterdir(): if not file_path.is_file(): continue # Skip directories etc. if file_path.match("pep-0000*"): continue # Skip pre-existing PEP 0 files - if pep_pat.match(str(file_path)) and file_path.suffix in {".txt", ".rst"}: - pep = parser.PEP(path.joinpath(file_path).absolute(), authors_overrides) + if file_path.match("pep-????.rst"): + pep = parser.PEP(path.joinpath(file_path).absolute()) peps.append(pep) - pep0_text = writer.PEPZeroWriter().write_pep0(sorted(peps)) - Path(f"{pep_zero_filename}.rst").write_text(pep0_text, encoding="utf-8") + return sorted(peps) - # Add to files for builder - docnames.insert(1, pep_zero_filename) - # Add to files for writer - env.found_docs.add(pep_zero_filename) + +def create_pep_json(peps: list[parser.PEP]) -> str: + return json.dumps({pep.number: pep.full_details for pep in peps}, indent=1) + + +def write_peps_json(peps: list[parser.PEP], path: Path) -> None: + # Create peps.json + json_peps = create_pep_json(peps) + Path(path, "peps.json").write_text(json_peps, encoding="utf-8") + os.makedirs(os.path.join(path, "api"), exist_ok=True) + Path(path, "api", "peps.json").write_text(json_peps, encoding="utf-8") + + +def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None: + peps = _parse_peps(Path(app.srcdir)) + + pep0_text = writer.PEPZeroWriter().write_pep0(peps, builder=env.settings["builder"]) + pep0_path = subindices.update_sphinx("pep-0000", pep0_text, docnames, env) + peps.append(parser.PEP(pep0_path)) + + subindices.generate_subindices(SUBINDICES_BY_TOPIC, peps, docnames, env) + + write_peps_json(peps, Path(app.outdir)) diff --git a/pep_sphinx_extensions/pep_zero_generator/subindices.py b/pep_sphinx_extensions/pep_zero_generator/subindices.py new file mode 100644 index 000000000..3f61b3dd4 --- /dev/null +++ b/pep_sphinx_extensions/pep_zero_generator/subindices.py @@ -0,0 +1,76 @@ +"""Utilities to support sub-indices for PEPs.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING + +from pep_sphinx_extensions.pep_zero_generator import writer + +if TYPE_CHECKING: + from sphinx.environment import BuildEnvironment + + from pep_sphinx_extensions.pep_zero_generator.parser import PEP + + +def update_sphinx(filename: str, text: str, docnames: list[str], env: BuildEnvironment) -> Path: + file_path = Path(env.srcdir, f"{filename}.rst") + file_path.write_text(text, encoding="utf-8") + + # Add to files for builder + docnames.append(filename) + # Add to files for writer + env.found_docs.add(filename) + + return file_path + + +def generate_subindices( + subindices: dict[str, str], + peps: list[PEP], + docnames: list[str], + env: BuildEnvironment, +) -> None: + # create topic directory + os.makedirs(os.path.join(env.srcdir, "topic"), exist_ok=True) + + # Create sub index page + generate_topic_contents(docnames, env) + + for subindex, additional_description in subindices.items(): + header_text = f"{subindex.title()} PEPs" + header_line = "#" * len(header_text) + header = header_text + "\n" + header_line + "\n" + + topic = subindex.lower() + filtered_peps = [pep for pep in peps if topic in pep.topic] + subindex_intro = f"""\ +This is the index of all Python Enhancement Proposals (PEPs) labelled +under the '{subindex.title()}' topic. This is a sub-index of :pep:`0`, +the PEP index. + +{additional_description} +""" + subindex_text = writer.PEPZeroWriter().write_pep0( + filtered_peps, header, subindex_intro, is_pep0=False, + ) + update_sphinx(f"topic/{subindex}", subindex_text, docnames, env) + + +def generate_topic_contents(docnames: list[str], env: BuildEnvironment): + update_sphinx("topic/index", """\ +.. _topic-index: + +Topic Index +*********** + +PEPs are indexed by topic on the pages below: + +.. toctree:: + :maxdepth: 1 + :titlesonly: + :glob: + + * +""", docnames, env) diff --git a/pep_sphinx_extensions/pep_zero_generator/writer.py b/pep_sphinx_extensions/pep_zero_generator/writer.py index 61f98f091..69a5fe4bc 100644 --- a/pep_sphinx_extensions/pep_zero_generator/writer.py +++ b/pep_sphinx_extensions/pep_zero_generator/writer.py @@ -2,13 +2,12 @@ from __future__ import annotations -import datetime -import functools from typing import TYPE_CHECKING import unicodedata +from pep_sphinx_extensions.pep_processor.transforms.pep_headers import ABBREVIATED_STATUSES +from pep_sphinx_extensions.pep_processor.transforms.pep_headers import ABBREVIATED_TYPES from pep_sphinx_extensions.pep_zero_generator.constants import DEAD_STATUSES -from pep_sphinx_extensions.pep_zero_generator.constants import HIDE_STATUS from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACCEPTED from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_ACTIVE from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_DEFERRED @@ -18,6 +17,7 @@ from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_PROVISIONA from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_REJECTED from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_VALUES from pep_sphinx_extensions.pep_zero_generator.constants import STATUS_WITHDRAWN +from pep_sphinx_extensions.pep_zero_generator.constants import SUBINDICES_BY_TOPIC from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_INFO from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_PROCESS from pep_sphinx_extensions.pep_zero_generator.constants import TYPE_VALUES @@ -26,39 +26,24 @@ from pep_sphinx_extensions.pep_zero_generator.errors import PEPError if TYPE_CHECKING: from pep_sphinx_extensions.pep_zero_generator.parser import PEP -title_length = 55 -author_length = 40 -table_separator = "== ==== " + "="*title_length + " " + "="*author_length - -# column format is called as a function with a mapping containing field values -column_format = functools.partial( - "{type}{status}{number: >5} {title: <{title_length}} {authors}".format, - title_length=title_length -) - -header = f"""\ +HEADER = """\ PEP: 0 Title: Index of Python Enhancement Proposals (PEPs) -Last-Modified: {datetime.date.today()} -Author: python-dev +Author: The PEP Editors Status: Active Type: Informational Content-Type: text/x-rst Created: 13-Jul-2000 """ -intro = """\ +INTRO = """\ This PEP contains the index of all Python Enhancement Proposals, -known as PEPs. PEP numbers are assigned by the PEP editors, and -once assigned are never changed [1_]. The version control history [2_] of +known as PEPs. PEP numbers are :pep:`assigned <1#pep-editors>` +by the PEP editors, and once assigned are never changed. The +`version control history `_ of the PEP texts represent their historical record. """ -references = """\ -.. [1] PEP 1: PEP Purpose and Guidelines -.. [2] View PEP history online: https://github.com/python/peps -""" - class PEPZeroWriter: # This is a list of reserved PEP numbers. Reservations are not to be used for @@ -84,134 +69,175 @@ class PEPZeroWriter: def emit_newline(self) -> None: self.output.append("") - def emit_table_separator(self) -> None: - self.output.append(table_separator) - def emit_author_table_separator(self, max_name_len: int) -> None: author_table_separator = "=" * max_name_len + " " + "=" * len("email address") self.output.append(author_table_separator) - def emit_pep_row(self, pep_details: dict[str, int | str]) -> None: - self.emit_text(column_format(**pep_details)) + def emit_pep_row( + self, *, shorthand: str, number: int, title: str, authors: str + ) -> None: + self.emit_text(f" * - {shorthand}") + self.emit_text(f" - :pep:`{number} <{number}>`") + self.emit_text(f" - :pep:`{title.replace('`', '')} <{number}>`") + self.emit_text(f" - {authors}") def emit_column_headers(self) -> None: """Output the column headers for the PEP indices.""" - self.emit_table_separator() - self.emit_pep_row({"status": ".", "type": ".", "number": "PEP", "title": "PEP Title", "authors": "PEP Author(s)"}) - self.emit_table_separator() + self.emit_text(".. list-table::") + self.emit_text(" :header-rows: 1") + self.emit_text(" :widths: auto") + self.emit_text(" :class: pep-zero-table") + self.emit_newline() + self.emit_text(" * - ") + self.emit_text(" - PEP") + self.emit_text(" - Title") + self.emit_text(" - Authors") - def emit_title(self, text: str, anchor: str, *, symbol: str = "=") -> None: - self.output.append(f".. _{anchor}:\n") + def emit_title(self, text: str, *, symbol: str = "=") -> None: self.output.append(text) self.output.append(symbol * len(text)) self.emit_newline() - def emit_subtitle(self, text: str, anchor: str) -> None: - self.emit_title(text, anchor, symbol="-") + def emit_subtitle(self, text: str) -> None: + self.emit_title(text, symbol="-") - def emit_pep_category(self, category: str, anchor: str, peps: list[PEP]) -> None: - self.emit_subtitle(category, anchor) + def emit_pep_category(self, category: str, peps: list[PEP]) -> None: + self.emit_subtitle(category) self.emit_column_headers() for pep in peps: - self.output.append(column_format(**pep.details(title_length=title_length))) - self.emit_table_separator() + self.emit_pep_row(**pep.details) + # list-table must have at least one body row + if len(peps) == 0: + self.emit_text(" * -") + self.emit_text(" -") + self.emit_text(" -") + self.emit_text(" -") self.emit_newline() - def write_pep0(self, peps: list[PEP]): + def write_pep0( + self, + peps: list[PEP], + header: str = HEADER, + intro: str = INTRO, + is_pep0: bool = True, + builder: str = None, + ): + if len(peps) == 0: + return "" # PEP metadata self.emit_text(header) self.emit_newline() # Introduction - self.emit_title("Introduction", "intro") + self.emit_title("Introduction") self.emit_text(intro) self.emit_newline() + # PEPs by topic + if is_pep0: + self.emit_title("Topics") + self.emit_text( + "PEPs for specialist subjects are :doc:`indexed by topic `." + ) + self.emit_newline() + for subindex in SUBINDICES_BY_TOPIC: + target = ( + f"topic/{subindex}.html" + if builder == "html" + else f"../topic/{subindex}/" + ) + self.emit_text(f"* `{subindex.title()} PEPs <{target}>`_") + self.emit_newline() + self.emit_newline() + # PEPs by category - self.emit_title("Index by Category", "by-category") + self.emit_title("Index by Category") meta, info, provisional, accepted, open_, finished, historical, deferred, dead = _classify_peps(peps) pep_categories = [ - ("Meta-PEPs (PEPs about PEPs or Processes)", "by-category-meta", meta), - ("Other Informational PEPs", "by-category-other-info", info), - ("Provisional PEPs (provisionally accepted; interface may still change)", "by-category-provisional", provisional), - ("Accepted PEPs (accepted; may not be implemented yet)", "by-category-accepted", accepted), - ("Open PEPs (under consideration)", "by-category-open", open_), - ("Finished PEPs (done, with a stable interface)", "by-category-finished", finished), - ("Historical Meta-PEPs and Informational PEPs", "by-category-historical", historical), - ("Deferred PEPs (postponed pending further research or updates)", "by-category-deferred", deferred), - ("Abandoned, Withdrawn, and Rejected PEPs", "by-category-abandoned", dead), + ("Meta-PEPs (PEPs about PEPs or Processes)", meta), + ("Other Informational PEPs", info), + ("Provisional PEPs (provisionally accepted; interface may still change)", provisional), + ("Accepted PEPs (accepted; may not be implemented yet)", accepted), + ("Open PEPs (under consideration)", open_), + ("Finished PEPs (done, with a stable interface)", finished), + ("Historical Meta-PEPs and Informational PEPs", historical), + ("Deferred PEPs (postponed pending further research or updates)", deferred), + ("Abandoned, Withdrawn, and Rejected PEPs", dead), ] - for (category, anchor, peps_in_category) in pep_categories: - self.emit_pep_category(category, anchor, peps_in_category) + for (category, peps_in_category) in pep_categories: + # For sub-indices, only emit categories with entries. + # For PEP 0, emit every category, but only with a table when it has entries. + if len(peps_in_category) > 0: + self.emit_pep_category(category, peps_in_category) + elif is_pep0: + # emit the category with no table + self.emit_subtitle(category) + self.emit_text("None.") + self.emit_newline() self.emit_newline() # PEPs by number - self.emit_title("Numerical Index", "by-pep-number") + self.emit_title("Numerical Index") self.emit_column_headers() - prev_pep = 0 for pep in peps: - if pep.number - prev_pep > 1: - self.emit_newline() - self.emit_pep_row(pep.details(title_length=title_length)) - prev_pep = pep.number + self.emit_pep_row(**pep.details) - self.emit_table_separator() self.emit_newline() # Reserved PEP numbers - self.emit_title("Reserved PEP Numbers", "reserved") - self.emit_column_headers() - for number, claimants in sorted(self.RESERVED.items()): - self.emit_pep_row({"type": ".", "status": ".", "number": number, "title": "RESERVED", "authors": claimants}) + if is_pep0: + self.emit_title("Reserved PEP Numbers") + self.emit_column_headers() + for number, claimants in sorted(self.RESERVED.items()): + self.emit_pep_row( + shorthand="", number=number, title="RESERVED", authors=claimants + ) - self.emit_table_separator() - self.emit_newline() - - # PEP types key - self.emit_title("PEP Types Key", "type-key") - for type_ in sorted(TYPE_VALUES): - self.emit_text(f" {type_[0]} - {type_} PEP") self.emit_newline() + # PEP types key + self.emit_title("PEP Types Key") + for type_ in sorted(TYPE_VALUES): + self.emit_text( + f"* **{type_[0]}** --- *{type_}*: {ABBREVIATED_TYPES[type_]}" + ) + self.emit_newline() + + self.emit_text(":pep:`More info in PEP 1 <1#pep-types>`.") self.emit_newline() # PEP status key - self.emit_title("PEP Status Key", "status-key") + self.emit_title("PEP Status Key") for status in sorted(STATUS_VALUES): # Draft PEPs have no status displayed, Active shares a key with Accepted - if status in HIDE_STATUS: - continue - if status == STATUS_ACCEPTED: - msg = " A - Accepted (Standards Track only) or Active proposal" - else: - msg = f" {status[0]} - {status} proposal" - self.emit_text(msg) + status_code = "" if status == STATUS_DRAFT else status[0] + self.emit_text( + f"* **{status_code}** --- *{status}*: {ABBREVIATED_STATUSES[status]}" + ) self.emit_newline() + self.emit_text(":pep:`More info in PEP 1 <1#pep-review-resolution>`.") self.emit_newline() - # PEP owners - authors_dict = _verify_email_addresses(peps) - max_name_len = max(len(author_name) for author_name in authors_dict) - self.emit_title("Authors/Owners", "authors") - self.emit_author_table_separator(max_name_len) - self.emit_text(f"{'Name':{max_name_len}} Email Address") - self.emit_author_table_separator(max_name_len) - for author_name in _sort_authors(authors_dict): - # Use the email from authors_dict instead of the one from "author" as - # the author instance may have an empty email. - self.emit_text(f"{author_name:{max_name_len}} {authors_dict[author_name]}") - self.emit_author_table_separator(max_name_len) - self.emit_newline() - self.emit_newline() + if is_pep0: + # PEP owners + authors_dict = _verify_email_addresses(peps) + max_name_len = max(len(author_name) for author_name in authors_dict) + self.emit_title("Authors/Owners") + self.emit_author_table_separator(max_name_len) + self.emit_text(f"{'Name':{max_name_len}} Email Address") + self.emit_author_table_separator(max_name_len) + for author_name in _sort_authors(authors_dict): + # Use the email from authors_dict instead of the one from "author" as + # the author instance may have an empty email. + self.emit_text(f"{author_name:{max_name_len}} {authors_dict[author_name]}") + self.emit_author_table_separator(max_name_len) + self.emit_newline() + self.emit_newline() - # References for introduction footnotes - self.emit_title("References", "references") - self.emit_text(references) - - pep0_string = "\n".join([str(s) for s in self.output]) + pep0_string = "\n".join(map(str, self.output)) return pep0_string @@ -235,7 +261,7 @@ def _classify_peps(peps: list[PEP]) -> tuple[list[PEP], ...]: elif pep.status == STATUS_DEFERRED: deferred.append(pep) elif pep.pep_type == TYPE_PROCESS: - if pep.status == STATUS_ACTIVE: + if pep.status in {STATUS_ACCEPTED, STATUS_ACTIVE}: meta.append(pep) elif pep.status in {STATUS_WITHDRAWN, STATUS_REJECTED}: dead.append(pep) @@ -247,7 +273,7 @@ def _classify_peps(peps: list[PEP]) -> tuple[list[PEP], ...]: # Hack until the conflict between the use of "Final" # for both API definition PEPs and other (actually # obsolete) PEPs is addressed - if pep.status == STATUS_ACTIVE or "Release Schedule" not in pep.title: + if pep.status == STATUS_ACTIVE or "release schedule" not in pep.title.lower(): info.append(pep) else: historical.append(pep) @@ -267,22 +293,22 @@ def _verify_email_addresses(peps: list[PEP]) -> dict[str, str]: for pep in peps: for author in pep.authors: # If this is the first time we have come across an author, add them. - if author.last_first not in authors_dict: - authors_dict[author.last_first] = set() + if author.full_name not in authors_dict: + authors_dict[author.full_name] = set() # If the new email is an empty string, move on. if not author.email: continue # If the email has not been seen, add it to the list. - authors_dict[author.last_first].add(author.email) + authors_dict[author.full_name].add(author.email) valid_authors_dict: dict[str, str] = {} too_many_emails: list[tuple[str, set[str]]] = [] - for last_first, emails in authors_dict.items(): + for full_name, emails in authors_dict.items(): if len(emails) > 1: - too_many_emails.append((last_first, emails)) + too_many_emails.append((full_name, emails)) else: - valid_authors_dict[last_first] = next(iter(emails), "") + valid_authors_dict[full_name] = next(iter(emails), "") if too_many_emails: err_output = [] for author, emails in too_many_emails: diff --git a/pep_sphinx_extensions/tests/__init__.py b/pep_sphinx_extensions/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pep_sphinx_extensions/tests/conftest.py b/pep_sphinx_extensions/tests/conftest.py new file mode 100644 index 000000000..15f3c907b --- /dev/null +++ b/pep_sphinx_extensions/tests/conftest.py @@ -0,0 +1,12 @@ +import importlib.util +import sys +from pathlib import Path + +_ROOT_PATH = Path(__file__, "..", "..", "..").resolve() +PEP_ROOT = _ROOT_PATH / "peps" + +# Import "check-peps.py" as "check_peps" +CHECK_PEPS_PATH = _ROOT_PATH / "check-peps.py" +spec = importlib.util.spec_from_file_location("check_peps", CHECK_PEPS_PATH) +sys.modules["check_peps"] = check_peps = importlib.util.module_from_spec(spec) +spec.loader.exec_module(check_peps) diff --git a/pep_sphinx_extensions/tests/pep_lint/__init__.py b/pep_sphinx_extensions/tests/pep_lint/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pep_sphinx_extensions/tests/pep_lint/test_date.py b/pep_sphinx_extensions/tests/pep_lint/test_date.py new file mode 100644 index 000000000..2218fc2a0 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_lint/test_date.py @@ -0,0 +1,105 @@ +import datetime as dt + +import check_peps # NoQA: inserted into sys.modules in conftest.py +import pytest + + +@pytest.mark.parametrize( + "line", + [ + # valid entries + "01-Jan-2000", + "29-Feb-2016", + "31-Dec-2000", + "01-Apr-2003", + "01-Apr-2007", + "01-Apr-2009", + "01-Jan-1990", + ], +) +def test_validate_created(line: str): + warnings = [warning for (_, warning) in check_peps._validate_created(1, line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "date_str", + [ + # valid entries + "01-Jan-2000", + "29-Feb-2016", + "31-Dec-2000", + "01-Apr-2003", + "01-Apr-2007", + "01-Apr-2009", + "01-Jan-1990", + ], +) +def test_date_checker_valid(date_str: str): + warnings = [warning for (_, warning) in check_peps._date(1, date_str, "")] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "date_str", + [ + # malformed + "2000-01-01", + "01 January 2000", + "1 Jan 2000", + "1-Jan-2000", + "1-January-2000", + "Jan-1-2000", + "January 1 2000", + "January 01 2000", + "01/01/2000", + "01/Jan/2000", # 🇬🇧, 🇩đŸ‡ș, 🇹🇩, 🇳🇿, 🇼đŸ‡Ș , ... + "Jan/01/2000", # đŸ‡ș🇾 + "1st January 2000", + "The First day of January in the year of Our Lord Two Thousand", + "Jan, 1, 2000", + "2000-Jan-1", + "2000-Jan-01", + "2000-January-1", + "2000-January-01", + "00 Jan 2000", + "00-Jan-2000", + ], +) +def test_date_checker_malformed(date_str: str): + warnings = [warning for (_, warning) in check_peps._date(1, date_str, "")] + expected = f" must be a 'DD-mmm-YYYY' date: {date_str!r}" + assert warnings == [expected], warnings + + +@pytest.mark.parametrize( + "date_str", + [ + # too early + "31-Dec-1989", + "01-Apr-1916", + "01-Jan-0020", + "01-Jan-0023", + ], +) +def test_date_checker_too_early(date_str: str): + warnings = [warning for (_, warning) in check_peps._date(1, date_str, "")] + expected = f" must not be before Python was invented: {date_str!r}" + assert warnings == [expected], warnings + + +@pytest.mark.parametrize( + "date_str", + [ + # the future + "31-Dec-2999", + "01-Jan-2042", + "01-Jan-2100", + (dt.datetime.now() + dt.timedelta(days=15)).strftime("%d-%b-%Y"), + (dt.datetime.now() + dt.timedelta(days=100)).strftime("%d-%b-%Y"), + ], +) +def test_date_checker_too_late(date_str: str): + warnings = [warning for (_, warning) in check_peps._date(1, date_str, "")] + expected = f" must not be in the future: {date_str!r}" + assert warnings == [expected], warnings diff --git a/pep_sphinx_extensions/tests/pep_lint/test_direct_links.py b/pep_sphinx_extensions/tests/pep_lint/test_direct_links.py new file mode 100644 index 000000000..66edc0552 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_lint/test_direct_links.py @@ -0,0 +1,30 @@ +import check_peps # NoQA: inserted into sys.modules in conftest.py +import pytest + + +@pytest.mark.parametrize( + "line", + [ + "http://www.python.org/dev/peps/pep-0000/", + "https://www.python.org/dev/peps/pep-0000/", + "http://peps.python.org/pep-0000/", + "https://peps.python.org/pep-0000/", + ], +) +def test_check_direct_links_pep(line: str): + warnings = [warning for (_, warning) in check_peps.check_direct_links(1, line)] + assert warnings == ["Use the :pep:`NNN` role to refer to PEPs"], warnings + + +@pytest.mark.parametrize( + "line", + [ + "http://www.rfc-editor.org/rfc/rfc2324", + "https://www.rfc-editor.org/rfc/rfc2324", + "http://datatracker.ietf.org/doc/html/rfc2324", + "https://datatracker.ietf.org/doc/html/rfc2324", + ], +) +def test_check_direct_links_rfc(line: str): + warnings = [warning for (_, warning) in check_peps.check_direct_links(1, line)] + assert warnings == ["Use the :rfc:`NNN` role to refer to RFCs"], warnings diff --git a/pep_sphinx_extensions/tests/pep_lint/test_email.py b/pep_sphinx_extensions/tests/pep_lint/test_email.py new file mode 100644 index 000000000..d9f19592d --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_lint/test_email.py @@ -0,0 +1,238 @@ +import check_peps # NoQA: inserted into sys.modules in conftest.py +import pytest + + +@pytest.mark.parametrize( + "line", + [ + "Alice", + "Alice,", + "Alice, Bob, Charlie", + "Alice,\nBob,\nCharlie", + "Alice,\n Bob,\n Charlie", + "Alice,\n Bob,\n Charlie", + "Cardinal XimĂ©nez", + "Alice ", + "Cardinal XimĂ©nez ", + ], + ids=repr, # the default calls str and renders newlines. +) +def test_validate_author(line: str): + warnings = [warning for (_, warning) in check_peps._validate_author(1, line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "Alice,\n Bob,\n Charlie", + "Alice,\n Bob,\n Charlie", + "Alice,\n Bob,\n Charlie", + "Alice,\n Bob", + ], + ids=repr, # the default calls str and renders newlines. +) +def test_validate_author_over__indented(line: str): + warnings = [warning for (_, warning) in check_peps._validate_author(1, line)] + assert {*warnings} == {"Author line must not be over-indented"}, warnings + + +@pytest.mark.parametrize( + "line", + [ + "Cardinal XimĂ©nez\nCardinal Biggles\nCardinal Fang", + "Cardinal XimĂ©nez,\nCardinal Biggles\nCardinal Fang", + "Cardinal XimĂ©nez\nCardinal Biggles,\nCardinal Fang", + ], + ids=repr, # the default calls str and renders newlines. +) +def test_validate_author_continuation(line: str): + warnings = [warning for (_, warning) in check_peps._validate_author(1, line)] + assert {*warnings} == {"Author continuation lines must end with a comma"}, warnings + + +@pytest.mark.parametrize( + "line", + [ + "Alice", + "Cardinal XimĂ©nez", + "Alice ", + "Cardinal XimĂ©nez ", + ], +) +def test_validate_sponsor(line: str): + warnings = [warning for (_, warning) in check_peps._validate_sponsor(1, line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "", + "Alice, Bob, Charlie", + "Alice, Bob, Charlie,", + "Alice ", + "Cardinal XimĂ©nez ", + ], +) +def test_validate_delegate(line: str): + warnings = [warning for (_, warning) in check_peps._validate_delegate(1, line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + ("email", "expected_warnings"), + [ + # ... entries must not contain multiple '...' + ("Cardinal XimĂ©nez <<", {"multiple <"}), + ("Cardinal XimĂ©nez <<<", {"multiple <"}), + ("Cardinal XimĂ©nez >>", {"multiple >"}), + ("Cardinal XimĂ©nez >>>", {"multiple >"}), + ("Cardinal XimĂ©nez <<<>>>", {"multiple <", "multiple >"}), + ("Cardinal XimĂ©nez @@", {"multiple @"}), + ("Cardinal XimĂ©nez <<@@@>", {"multiple <", "multiple @"}), + ("Cardinal XimĂ©nez <@@@>>", {"multiple >", "multiple @"}), + ("Cardinal XimĂ©nez <<@@>>", {"multiple <", "multiple >", "multiple @"}), + # valid names + ("Cardinal XimĂ©nez", set()), + (" Cardinal XimĂ©nez", set()), + ("\t\tCardinal XimĂ©nez", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez\t\t", set()), + ("Cardinal O'XimĂ©nez", set()), + ("Cardinal XimĂ©nez, Inquisitor", set()), + ("Cardinal XimĂ©nez-Biggles", set()), + ("Cardinal XimĂ©nez-Biggles, Inquisitor", set()), + ("Cardinal T. S. I. XimĂ©nez", set()), + # ... entries must have a valid 'Name' + ("Cardinal_XimĂ©nez", {"valid name"}), + ("Cardinal XimĂ©nez 3", {"valid name"}), + ("~ Cardinal XimĂ©nez ~", {"valid name"}), + ("Cardinal XimĂ©nez!", {"valid name"}), + ("@Cardinal XimĂ©nez", {"valid name"}), + ("Cardinal_XimĂ©nez <>", {"valid name"}), + ("Cardinal XimĂ©nez 3 <>", {"valid name"}), + ("~ Cardinal XimĂ©nez ~ <>", {"valid name"}), + ("Cardinal XimĂ©nez! <>", {"valid name"}), + ("@Cardinal XimĂ©nez <>", {"valid name"}), + # ... entries must be formatted as 'Name ' + ("Cardinal XimĂ©nez<>", {"name "}), + ("Cardinal XimĂ©nez<", {"name "}), + ("Cardinal XimĂ©nez <", {"name "}), + ("Cardinal XimĂ©nez <", {"name "}), + ("Cardinal XimĂ©nez <>", {"name "}), + # ... entries must contain a valid email address (missing) + ("Cardinal XimĂ©nez <>", {"valid email"}), + ("Cardinal XimĂ©nez <> ", {"valid email"}), + ("Cardinal XimĂ©nez <@> ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez < at > ", {"valid email"}), + # ... entries must contain a valid email address (local) + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez < Cardinal Ximenez @spanish.inquisition> ", {"valid email"}), + ("Cardinal XimĂ©nez <(Cardinal.Ximenez)@spanish.inquisition>", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ( + "Cardinal XimĂ©nez ", + {"multiple <", "multiple >", "valid email"}, + ), + ( + "Cardinal XimĂ©nez ", + {"multiple @", "valid email"}, + ), + (r"Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez <[Cardinal.Ximenez]@spanish.inquisition>", {"valid email"}), + ('Cardinal XimĂ©nez <"Cardinal"Ximenez"@spanish.inquisition>', {"valid email"}), + ("Cardinal Ximenez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + # ... entries must contain a valid email address (domain) + ( + "Cardinal XimĂ©nez ", + {"valid email"}, + ), + ("Cardinal XimĂ©nez ", {"valid email"}), + ("Cardinal XimĂ©nez ", {"valid email"}), + ( + "Cardinal XimĂ©nez ", + {"valid email"}, + ), + # valid name-emails + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez <{Cardinal.Ximenez}@spanish.inquisition>", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ("Cardinal XimĂ©nez ", set()), + ], + # call str() on each parameterised value in the test ID. + ids=str, +) +def test_email_checker(email: str, expected_warnings: set): + warnings = [warning for (_, warning) in check_peps._email(1, email, "")] + + found_warnings = set() + email = email.strip() + + if "multiple <" in expected_warnings: + found_warnings.add("multiple <") + expected = f" entries must not contain multiple '<': {email!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "multiple >" in expected_warnings: + found_warnings.add("multiple >") + expected = f" entries must not contain multiple '>': {email!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "multiple @" in expected_warnings: + found_warnings.add("multiple @") + expected = f" entries must not contain multiple '@': {email!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "valid name" in expected_warnings: + found_warnings.add("valid name") + expected = f" entries must begin with a valid 'Name': {email!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "name " in expected_warnings: + found_warnings.add("name ") + expected = f" entries must be formatted as 'Name ': {email!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "valid email" in expected_warnings: + found_warnings.add("valid email") + expected = f" entries must contain a valid email address: {email!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if expected_warnings == set(): + assert warnings == [], warnings + + assert found_warnings == expected_warnings diff --git a/pep_sphinx_extensions/tests/pep_lint/test_headers.py b/pep_sphinx_extensions/tests/pep_lint/test_headers.py new file mode 100644 index 000000000..227eb5397 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_lint/test_headers.py @@ -0,0 +1,408 @@ +import check_peps # NoQA: inserted into sys.modules in conftest.py +import pytest + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + # capitalisation + ("Header:", "Header"), + ("header:", "header"), + ("hEADER:", "hEADER"), + ("hEaDeR:", "hEaDeR"), + # trailing spaces + ("Header: ", "Header"), + ("Header: ", "Header"), + ("Header: \t", "Header"), + # trailing content + ("Header: Text", "Header"), + ("Header: 123", "Header"), + ("Header: !", "Header"), + # separators + ("Hyphenated-Header:", "Hyphenated-Header"), + ], +) +def test_header_pattern(test_input, expected): + assert check_peps.HEADER_PATTERN.match(test_input)[1] == expected + + +@pytest.mark.parametrize( + "test_input", + [ + # trailing content + "Header:Text", + "Header:123", + "Header:!", + # colon position + "Header", + "Header : ", + "Header :", + "SemiColonHeader;", + # separators + "Underscored_Header:", + "Spaced Header:", + "Plus+Header:", + ], +) +def test_header_pattern_no_match(test_input): + assert check_peps.HEADER_PATTERN.match(test_input) is None + + +def test_validate_required_headers(): + found_headers = dict.fromkeys( + ("PEP", "Title", "Author", "Status", "Type", "Created") + ) + warnings = [ + warning for (_, warning) in check_peps._validate_required_headers(found_headers) + ] + assert warnings == [], warnings + + +def test_validate_required_headers_missing(): + found_headers = dict.fromkeys(("PEP", "Title", "Author", "Type")) + warnings = [ + warning for (_, warning) in check_peps._validate_required_headers(found_headers) + ] + assert warnings == [ + "Must have required header: Status", + "Must have required header: Created", + ], warnings + + +def test_validate_required_headers_order(): + found_headers = dict.fromkeys( + ("PEP", "Title", "Sponsor", "Author", "Type", "Status", "Replaces", "Created") + ) + warnings = [ + warning for (_, warning) in check_peps._validate_required_headers(found_headers) + ] + assert warnings == [ + "Headers must be in PEP 12 order. Correct order: PEP, Title, Author, Sponsor, Status, Type, Created, Replaces" + ], warnings + + +@pytest.mark.parametrize( + "line", + [ + "!", + "The Zen of Python", + "A title that is exactly 79 characters long, but shorter than 80 characters long", + ], +) +def test_validate_title(line: str): + warnings = [warning for (_, warning) in check_peps._validate_title(1, line)] + assert warnings == [], warnings + + +def test_validate_title_blank(): + warnings = [warning for (_, warning) in check_peps._validate_title(1, "-" * 80)] + assert warnings == ["PEP title must be less than 80 characters"], warnings + + +def test_validate_title_too_long(): + warnings = [warning for (_, warning) in check_peps._validate_title(1, "")] + assert warnings == ["PEP must have a title"], warnings + + +@pytest.mark.parametrize( + "line", + [ + "Accepted", + "Active", + "April Fool!", + "Deferred", + "Draft", + "Final", + "Provisional", + "Rejected", + "Superseded", + "Withdrawn", + ], +) +def test_validate_status_valid(line: str): + warnings = [warning for (_, warning) in check_peps._validate_status(1, line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "Standards Track", + "Informational", + "Process", + "accepted", + "active", + "april fool!", + "deferred", + "draft", + "final", + "provisional", + "rejected", + "superseded", + "withdrawn", + ], +) +def test_validate_status_invalid(line: str): + warnings = [warning for (_, warning) in check_peps._validate_status(1, line)] + assert warnings == ["Status must be a valid PEP status"], warnings + + +@pytest.mark.parametrize( + "line", + [ + "Standards Track", + "Informational", + "Process", + ], +) +def test_validate_type_valid(line: str): + warnings = [warning for (_, warning) in check_peps._validate_type(1, line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "standards track", + "informational", + "process", + "Accepted", + "Active", + "April Fool!", + "Deferred", + "Draft", + "Final", + "Provisional", + "Rejected", + "Superseded", + "Withdrawn", + ], +) +def test_validate_type_invalid(line: str): + warnings = [warning for (_, warning) in check_peps._validate_type(1, line)] + assert warnings == ["Type must be a valid PEP type"], warnings + + +@pytest.mark.parametrize( + ("line", "expected_warnings"), + [ + # valid entries + ("Governance", set()), + ("Packaging", set()), + ("Typing", set()), + ("Release", set()), + ("Governance, Packaging", set()), + ("Packaging, Typing", set()), + # duplicates + ("Governance, Governance", {"duplicates"}), + ("Release, Release", {"duplicates"}), + ("Packaging, Packaging", {"duplicates"}), + ("Spam, Spam", {"duplicates", "valid"}), + ("lobster, lobster", {"duplicates", "capitalisation", "valid"}), + ("governance, governance", {"duplicates", "capitalisation"}), + # capitalisation + ("governance", {"capitalisation"}), + ("packaging", {"capitalisation"}), + ("typing", {"capitalisation"}), + ("release", {"capitalisation"}), + ("Governance, release", {"capitalisation"}), + # validity + ("Spam", {"valid"}), + ("lobster", {"capitalisation", "valid"}), + # sorted + ("Packaging, Governance", {"sorted"}), + ("Typing, Release", {"sorted"}), + ("Release, Governance", {"sorted"}), + ("spam, packaging", {"capitalisation", "valid", "sorted"}), + ], + # call str() on each parameterised value in the test ID. + ids=str, +) +def test_validate_topic(line: str, expected_warnings: set): + warnings = [warning for (_, warning) in check_peps._validate_topic(1, line)] + + found_warnings = set() + + if "duplicates" in expected_warnings: + found_warnings.add("duplicates") + expected = "Topic must not contain duplicates" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "capitalisation" in expected_warnings: + found_warnings.add("capitalisation") + expected = "Topic must be properly capitalised (Title Case)" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "valid" in expected_warnings: + found_warnings.add("valid") + expected = "Topic must be for a valid sub-index" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "sorted" in expected_warnings: + found_warnings.add("sorted") + expected = "Topic must be sorted lexicographically" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if expected_warnings == set(): + assert warnings == [], warnings + + assert found_warnings == expected_warnings + + +def test_validate_content_type_valid(): + warnings = [ + warning for (_, warning) in check_peps._validate_content_type(1, "text/x-rst") + ] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "text/plain", + "text/markdown", + "text/csv", + "text/rtf", + "text/javascript", + "text/html", + "text/xml", + ], +) +def test_validate_content_type_invalid(line: str): + warnings = [warning for (_, warning) in check_peps._validate_content_type(1, line)] + assert warnings == ["Content-Type must be 'text/x-rst'"], warnings + + +@pytest.mark.parametrize( + "line", + [ + "0, 1, 8, 12, 20,", + "101, 801,", + "3099, 9999", + ], +) +def test_validate_pep_references(line: str): + warnings = [ + warning for (_, warning) in check_peps._validate_pep_references(1, line) + ] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "0,1,8, 12, 20,", + "101,801,", + "3099, 9998,9999", + ], +) +def test_validate_pep_references_separators(line: str): + warnings = [ + warning for (_, warning) in check_peps._validate_pep_references(1, line) + ] + assert warnings == [ + "PEP references must be separated by comma-spaces (', ')" + ], warnings + + +@pytest.mark.parametrize( + ("line", "expected_warnings"), + [ + # valid entries + ("1.0, 2.4, 2.7, 2.8, 3.0, 3.1, 3.4, 3.7, 3.11, 3.14", set()), + ("2.x", set()), + ("3.x", set()), + ("3.0.1", set()), + # segments + ("", {"segments"}), + ("1", {"segments"}), + ("1.2.3.4", {"segments"}), + # major + ("0.0", {"major"}), + ("4.0", {"major"}), + ("9.0", {"major"}), + # minor number + ("3.a", {"minor numeric"}), + ("3.spam", {"minor numeric"}), + ("3.0+", {"minor numeric"}), + ("3.0-9", {"minor numeric"}), + ("9.Z", {"major", "minor numeric"}), + # minor leading zero + ("3.01", {"minor zero"}), + ("0.00", {"major", "minor zero"}), + # micro empty + ("3.x.1", {"micro empty"}), + ("9.x.1", {"major", "micro empty"}), + # micro leading zero + ("3.3.0", {"micro zero"}), + ("3.3.00", {"micro zero"}), + ("3.3.01", {"micro zero"}), + ("3.0.0", {"micro zero"}), + ("3.00.0", {"minor zero", "micro zero"}), + ("0.00.0", {"major", "minor zero", "micro zero"}), + # micro number + ("3.0.a", {"micro numeric"}), + ("0.3.a", {"major", "micro numeric"}), + ], + # call str() on each parameterised value in the test ID. + ids=str, +) +def test_validate_python_version(line: str, expected_warnings: set): + warnings = [ + warning for (_, warning) in check_peps._validate_python_version(1, line) + ] + + found_warnings = set() + + if "segments" in expected_warnings: + found_warnings.add("segments") + expected = f"Python-Version must have two or three segments: {line}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "major" in expected_warnings: + found_warnings.add("major") + expected = f"Python-Version major part must be 1, 2, or 3: {line}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "minor numeric" in expected_warnings: + found_warnings.add("minor numeric") + expected = f"Python-Version minor part must be numeric: {line}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "minor zero" in expected_warnings: + found_warnings.add("minor zero") + expected = f"Python-Version minor part must not have leading zeros: {line}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "micro empty" in expected_warnings: + found_warnings.add("micro empty") + expected = ( + f"Python-Version micro part must be empty if minor part is 'x': {line}" + ) + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "micro zero" in expected_warnings: + found_warnings.add("micro zero") + expected = f"Python-Version micro part must not have leading zeros: {line}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "micro numeric" in expected_warnings: + found_warnings.add("micro numeric") + expected = f"Python-Version micro part must be numeric: {line}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if expected_warnings == set(): + assert warnings == [], warnings + + assert found_warnings == expected_warnings diff --git a/pep_sphinx_extensions/tests/pep_lint/test_pep_lint.py b/pep_sphinx_extensions/tests/pep_lint/test_pep_lint.py new file mode 100644 index 000000000..2c52fa397 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_lint/test_pep_lint.py @@ -0,0 +1,48 @@ +from pathlib import Path + +import check_peps # NoQA: inserted into sys.modules in conftest.py + +PEP_9002 = Path(__file__).parent.parent / "peps" / "pep-9002.rst" + + +def test_with_fake_pep(): + content = PEP_9002.read_text(encoding="utf-8").splitlines() + warnings = list(check_peps.check_peps(PEP_9002, content)) + assert warnings == [ + (1, "PEP must begin with the 'PEP:' header"), + (9, "Must not have duplicate header: Sponsor "), + (10, "Must not have invalid header: Horse-Guards"), + (1, "Must have required header: PEP"), + (1, "Must have required header: Type"), + ( + 1, + "Headers must be in PEP 12 order. Correct order: Title, Version, " + "Author, Sponsor, BDFL-Delegate, Discussions-To, Status, Topic, " + "Content-Type, Requires, Created, Python-Version, Post-History, " + "Resolution", + ), + (4, "Author continuation lines must end with a comma"), + (5, "Author line must not be over-indented"), + (7, "Python-Version major part must be 1, 2, or 3: 4.0"), + ( + 8, + "Sponsor entries must begin with a valid 'Name': " + r"'Sponsor:\nHorse-Guards: Parade'", + ), + (11, "Created must be a 'DD-mmm-YYYY' date: '1-Jan-1989'"), + (12, "Delegate entries must begin with a valid 'Name': 'Barry!'"), + (13, "Status must be a valid PEP status"), + (14, "Topic must not contain duplicates"), + (14, "Topic must be properly capitalised (Title Case)"), + (14, "Topic must be for a valid sub-index"), + (14, "Topic must be sorted lexicographically"), + (15, "Content-Type must be 'text/x-rst'"), + (16, "PEP references must be separated by comma-spaces (', ')"), + (17, "Discussions-To must be a valid thread URL or mailing list"), + (18, "Post-History must be a 'DD-mmm-YYYY' date: '2-Feb-2000'"), + (18, "Post-History must be a valid thread URL"), + (19, "Post-History must be a 'DD-mmm-YYYY' date: '3-Mar-2001'"), + (19, "Post-History must be a valid thread URL"), + (20, "Resolution must be a valid thread URL"), + (23, "Use the :pep:`NNN` role to refer to PEPs"), + ] diff --git a/pep_sphinx_extensions/tests/pep_lint/test_pep_number.py b/pep_sphinx_extensions/tests/pep_lint/test_pep_number.py new file mode 100644 index 000000000..3443f7144 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_lint/test_pep_number.py @@ -0,0 +1,108 @@ +import check_peps # NoQA: inserted into sys.modules in conftest.py +import pytest + + +@pytest.mark.parametrize( + "line", + [ + "PEP: 0", + "PEP: 12", + ], +) +def test_validate_pep_number(line: str): + warnings = [warning for (_, warning) in check_peps._validate_pep_number(line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "0", + "PEP:12", + "PEP 0", + "PEP 12", + "PEP:0", + ], +) +def test_validate_pep_number_invalid_header(line: str): + warnings = [warning for (_, warning) in check_peps._validate_pep_number(line)] + assert warnings == ["PEP must begin with the 'PEP:' header"], warnings + + +@pytest.mark.parametrize( + ("pep_number", "expected_warnings"), + [ + # valid entries + ("0", set()), + ("1", set()), + ("12", set()), + ("20", set()), + ("101", set()), + ("801", set()), + ("3099", set()), + ("9999", set()), + # empty + ("", {"not blank"}), + # leading zeros + ("01", {"leading zeros"}), + ("001", {"leading zeros"}), + ("0001", {"leading zeros"}), + ("00001", {"leading zeros"}), + # non-numeric + ("a", {"non-numeric"}), + ("123abc", {"non-numeric"}), + ("0123A", {"leading zeros", "non-numeric"}), + ("", {"non-numeric"}), + ("10", {"non-numeric"}), + ("999", {"non-numeric"}), + ("𝟎", {"non-numeric"}), + ("𝟘", {"non-numeric"}), + ("𝟏𝟚", {"non-numeric"}), + ("𝟾𝟬", {"non-numeric"}), + ("-1", {"non-numeric"}), + ("+1", {"non-numeric"}), + # out of bounds + ("10000", {"range"}), + ("54321", {"range"}), + ("99999", {"range"}), + ("32768", {"range"}), + ], + # call str() on each parameterised value in the test ID. + ids=str, +) +def test_pep_num_checker(pep_number: str, expected_warnings: set): + warnings = [ + warning for (_, warning) in check_peps._pep_num(1, pep_number, "") + ] + + found_warnings = set() + pep_number = pep_number.strip() + + if "not blank" in expected_warnings: + found_warnings.add("not blank") + expected = f" must not be blank: {pep_number!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "leading zeros" in expected_warnings: + found_warnings.add("leading zeros") + expected = f" must not contain leading zeros: {pep_number!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "non-numeric" in expected_warnings: + found_warnings.add("non-numeric") + expected = f" must be numeric: {pep_number!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if "range" in expected_warnings: + found_warnings.add("range") + expected = f" must be between 0 and 9999: {pep_number!r}" + matching = [w for w in warnings if w == expected] + assert matching == [expected], warnings + + if expected_warnings == set(): + assert warnings == [], warnings + + assert found_warnings == expected_warnings diff --git a/pep_sphinx_extensions/tests/pep_lint/test_post_url.py b/pep_sphinx_extensions/tests/pep_lint/test_post_url.py new file mode 100644 index 000000000..982f4612e --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_lint/test_post_url.py @@ -0,0 +1,305 @@ +import check_peps # NoQA: inserted into sys.modules in conftest.py +import pytest + + +@pytest.mark.parametrize( + "line", + [ + "list-name@python.org", + "distutils-sig@python.org", + "csv@python.org", + "python-3000@python.org", + "ipaddr-py-dev@googlegroups.com", + "python-tulip@googlegroups.com", + "https://discuss.python.org/t/thread-name/123456", + "https://discuss.python.org/t/thread-name/123456/", + "https://discuss.python.org/t/thread_name/123456", + "https://discuss.python.org/t/thread_name/123456/", + "https://discuss.python.org/t/123456/", + "https://discuss.python.org/t/123456", + ], +) +def test_validate_discussions_to_valid(line: str): + warnings = [ + warning for (_, warning) in check_peps._validate_discussions_to(1, line) + ] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "$pecial+chars@python.org", + "a-discussions-to-list!@googlegroups.com", + ], +) +def test_validate_discussions_to_list_name(line: str): + warnings = [ + warning for (_, warning) in check_peps._validate_discussions_to(1, line) + ] + assert warnings == ["Discussions-To must be a valid mailing list"], warnings + + +@pytest.mark.parametrize( + "line", + [ + "list-name@python.org.uk", + "distutils-sig@mail-server.example", + ], +) +def test_validate_discussions_to_invalid_list_domain(line: str): + warnings = [ + warning for (_, warning) in check_peps._validate_discussions_to(1, line) + ] + assert warnings == [ + "Discussions-To must be a valid thread URL or mailing list" + ], warnings + + +@pytest.mark.parametrize( + "body", + [ + "", + ( + "01-Jan-2001, 02-Feb-2002,\n " + "03-Mar-2003, 04-Apr-2004,\n " + "05-May-2005," + ), + ( + "`01-Jan-2000 `__,\n " + "`11-Mar-2005 `__,\n " + "`21-May-2010 `__,\n " + "`31-Jul-2015 `__," + ), + "01-Jan-2001, `02-Feb-2002 `__,\n03-Mar-2003", + ], +) +def test_validate_post_history_valid(body: str): + warnings = [warning for (_, warning) in check_peps._validate_post_history(1, body)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123/", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123#Anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/#Anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123#Anchor123", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/#Anchor123", + ], +) +def test_validate_resolution_valid(line: str): + warnings = [warning for (_, warning) in check_peps._validate_resolution(1, line)] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "line", + [ + "https://mail.python.org/archives/list/list-name@python.org/thread", + "https://mail.python.org/archives/list/list-name@python.org/message", + "https://mail.python.org/archives/list/list-name@python.org/thread/", + "https://mail.python.org/archives/list/list-name@python.org/message/", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123#anchor", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123/#anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/#abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/message/#abcXYZ123/", + "https://mail.python.org/archives/list/list-name@python.org/spam/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/spam/abcXYZ123/", + ], +) +def test_validate_resolution_invalid(line: str): + warnings = [warning for (_, warning) in check_peps._validate_resolution(1, line)] + assert warnings == ["Resolution must be a valid thread URL"], warnings + + +@pytest.mark.parametrize( + "thread_url", + [ + "https://discuss.python.org/t/thread-name/123456", + "https://discuss.python.org/t/thread-name/123456/", + "https://discuss.python.org/t/thread_name/123456", + "https://discuss.python.org/t/thread_name/123456/", + "https://discuss.python.org/t/thread-name/123456/654321/", + "https://discuss.python.org/t/thread-name/123456/654321", + "https://discuss.python.org/t/123456", + "https://discuss.python.org/t/123456/", + "https://discuss.python.org/t/123456/654321/", + "https://discuss.python.org/t/123456/654321", + "https://discuss.python.org/t/1", + "https://discuss.python.org/t/1/", + "https://mail.python.org/pipermail/list-name/0000-Month/0123456.html", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123/", + ], +) +def test_thread_checker_valid(thread_url: str): + warnings = [ + warning for (_, warning) in check_peps._thread(1, thread_url, "") + ] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "thread_url", + [ + "http://link.example", + "list-name@python.org", + "distutils-sig@python.org", + "csv@python.org", + "python-3000@python.org", + "ipaddr-py-dev@googlegroups.com", + "python-tulip@googlegroups.com", + "https://link.example", + "https://discuss.python.org", + "https://discuss.python.org/", + "https://discuss.python.org/c/category", + "https://discuss.python.org/t/thread_name/123456//", + "https://discuss.python.org/t/thread+name/123456", + "https://discuss.python.org/t/thread+name/123456#", + "https://discuss.python.org/t/thread+name/123456/#", + "https://discuss.python.org/t/thread+name/123456/#anchor", + "https://discuss.python.org/t/thread+name/", + "https://discuss.python.org/t/thread+name", + "https://discuss.python.org/t/thread-name/123abc", + "https://discuss.python.org/t/thread-name/123abc/", + "https://discuss.python.org/t/thread-name/123456/123abc", + "https://discuss.python.org/t/thread-name/123456/123abc/", + "https://discuss.python.org/t/123/456/789", + "https://discuss.python.org/t/123/456/789/", + "https://discuss.python.org/t/#/", + "https://discuss.python.org/t/#", + "https://mail.python.org/pipermail/list+name/0000-Month/0123456.html", + "https://mail.python.org/pipermail/list-name/YYYY-Month/0123456.html", + "https://mail.python.org/pipermail/list-name/0123456/0123456.html", + "https://mail.python.org/pipermail/list-name/0000-Month/0123456", + "https://mail.python.org/pipermail/list-name/0000-Month/0123456/", + "https://mail.python.org/pipermail/list-name/0000-Month/", + "https://mail.python.org/pipermail/list-name/0000-Month", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123#anchor", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123/#anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123#anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/#anchor", + "https://mail.python.org/archives/list/list-name@python.org/spam/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/spam/abcXYZ123/", + ], +) +def test_thread_checker_invalid(thread_url: str): + warnings = [ + warning for (_, warning) in check_peps._thread(1, thread_url, "") + ] + assert warnings == [" must be a valid thread URL"], warnings + + +@pytest.mark.parametrize( + "thread_url", + [ + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123/", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123#Anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/#Anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123#Anchor123", + "https://mail.python.org/archives/list/list-name@python.org/message/abcXYZ123/#Anchor123", + ], +) +def test_thread_checker_valid_allow_message(thread_url: str): + warnings = [ + warning + for (_, warning) in check_peps._thread( + 1, thread_url, "", allow_message=True + ) + ] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "thread_url", + [ + "https://mail.python.org/archives/list/list-name@python.org/thread", + "https://mail.python.org/archives/list/list-name@python.org/message", + "https://mail.python.org/archives/list/list-name@python.org/thread/", + "https://mail.python.org/archives/list/list-name@python.org/message/", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123#anchor", + "https://mail.python.org/archives/list/list-name@python.org/thread/abcXYZ123/#anchor", + "https://mail.python.org/archives/list/list-name@python.org/message/#abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/message/#abcXYZ123/", + "https://mail.python.org/archives/list/list-name@python.org/spam/abcXYZ123", + "https://mail.python.org/archives/list/list-name@python.org/spam/abcXYZ123/", + ], +) +def test_thread_checker_invalid_allow_message(thread_url: str): + warnings = [ + warning + for (_, warning) in check_peps._thread( + 1, thread_url, "", allow_message=True + ) + ] + assert warnings == [" must be a valid thread URL"], warnings + + +@pytest.mark.parametrize( + "thread_url", + [ + "list-name@python.org", + "distutils-sig@python.org", + "csv@python.org", + "python-3000@python.org", + "ipaddr-py-dev@googlegroups.com", + "python-tulip@googlegroups.com", + "https://discuss.python.org/t/thread-name/123456", + "https://discuss.python.org/t/thread-name/123456/", + "https://discuss.python.org/t/thread_name/123456", + "https://discuss.python.org/t/thread_name/123456/", + "https://discuss.python.org/t/123456/", + "https://discuss.python.org/t/123456", + ], +) +def test_thread_checker_valid_discussions_to(thread_url: str): + warnings = [ + warning + for (_, warning) in check_peps._thread( + 1, thread_url, "", discussions_to=True + ) + ] + assert warnings == [], warnings + + +@pytest.mark.parametrize( + "thread_url", + [ + "https://discuss.python.org/t/thread-name/123456/000", + "https://discuss.python.org/t/thread-name/123456/000/", + "https://discuss.python.org/t/thread_name/123456/000", + "https://discuss.python.org/t/thread_name/123456/000/", + "https://discuss.python.org/t/123456/000/", + "https://discuss.python.org/t/12345656/000", + "https://discuss.python.org/t/thread-name", + "https://discuss.python.org/t/thread_name", + "https://discuss.python.org/t/thread+name", + ], +) +def test_thread_checker_invalid_discussions_to(thread_url: str): + warnings = [ + warning + for (_, warning) in check_peps._thread( + 1, thread_url, "", discussions_to=True + ) + ] + assert warnings == [" must be a valid thread URL"], warnings + + +def test_thread_checker_allow_message_discussions_to(): + with pytest.raises(ValueError, match="cannot both be True"): + list( + check_peps._thread( + 1, "", "", allow_message=True, discussions_to=True + ) + ) diff --git a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py new file mode 100644 index 000000000..9f040c5e1 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_footer.py @@ -0,0 +1,36 @@ +import datetime as dt + +from pep_sphinx_extensions.pep_processor.transforms import pep_footer + +from ...conftest import PEP_ROOT + + +def test_add_source_link(): + out = pep_footer._add_source_link(PEP_ROOT / "pep-0008.rst") + + assert "https://github.com/python/peps/blob/main/peps/pep-0008.rst" in str(out) + + +def test_add_commit_history_info(): + out = pep_footer._add_commit_history_info(PEP_ROOT / "pep-0008.rst") + + assert str(out).startswith( + "Last modified: " + '' + ) + # A variable timestamp comes next, don't test that + assert str(out).endswith("") + + +def test_add_commit_history_info_invalid(): + out = pep_footer._add_commit_history_info(PEP_ROOT / "pep-not-found.rst") + + assert str(out) == "" + + +def test_get_last_modified_timestamps(): + out = pep_footer._get_last_modified_timestamps() + + assert len(out) >= 585 + # Should be a Unix timestamp and at least this + assert dt.datetime.fromisoformat(out["pep-0008"]).timestamp() >= 1643124055 diff --git a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_headers.py b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_headers.py new file mode 100644 index 000000000..772c24158 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_headers.py @@ -0,0 +1,185 @@ +import pytest + +from pep_sphinx_extensions.pep_processor.transforms import pep_headers +from pep_sphinx_extensions.pep_zero_generator.constants import ( + STATUS_ACCEPTED, + STATUS_ACTIVE, + STATUS_DEFERRED, + STATUS_DRAFT, + STATUS_FINAL, + STATUS_PROVISIONAL, + STATUS_REJECTED, + STATUS_SUPERSEDED, + STATUS_WITHDRAWN, + TYPE_INFO, + TYPE_PROCESS, + TYPE_STANDARDS, +) + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ("my-mailing-list@example.com", "my-mailing-list@example.com"), + ("python-tulip@googlegroups.com", "https://groups.google.com/g/python-tulip"), + ("db-sig@python.org", "https://mail.python.org/mailman/listinfo/db-sig"), + ("import-sig@python.org", "https://mail.python.org/pipermail/import-sig/"), + ( + "python-announce@python.org", + "https://mail.python.org/archives/list/python-announce@python.org/", + ), + ], +) +def test_generate_list_url(test_input, expected): + out = pep_headers._generate_list_url(test_input) + + assert out == expected + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ( + "https://mail.python.org/pipermail/python-3000/2006-November/004190.html", + ("Python-3000", "message"), + ), + ( + "https://mail.python.org/archives/list/python-dev@python.org/thread/HW2NFOEMCVCTAFLBLC3V7MLM6ZNMKP42/", + ("Python-Dev", "thread"), + ), + ( + "https://mail.python.org/mailman3/lists/capi-sig.python.org/", + ("Capi-SIG", "list"), + ), + ( + "https://mail.python.org/mailman/listinfo/web-sig", + ("Web-SIG", "list"), + ), + ( + "https://discuss.python.org/t/pep-643-metadata-for-package-source-distributions/5577", + ("Discourse", "thread"), + ), + ( + "https://discuss.python.org/c/peps/", + ("PEPs Discourse", "category"), + ), + ], +) +def test_process_pretty_url(test_input, expected): + out = pep_headers._process_pretty_url(test_input) + + assert out == expected + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ( + "https://example.com/", + "https://example.com/ not a link to a recognized domain to prettify", + ), + ( + "https://mail.python.org", + "https://mail.python.org not a link to a list, message or thread", + ), + ( + "https://discuss.python.org/", + "https://discuss.python.org not a link to a Discourse thread or category", + ), + ], +) +def test_process_pretty_url_invalid(test_input, expected): + with pytest.raises(ValueError, match=expected): + pep_headers._process_pretty_url(test_input) + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ( + "https://mail.python.org/pipermail/python-3000/2006-November/004190.html", + "Python-3000 message", + ), + ( + "https://mail.python.org/archives/list/python-dev@python.org/thread/HW2NFOEMCVCTAFLBLC3V7MLM6ZNMKP42/", + "Python-Dev thread", + ), + ( + "https://mail.python.org/mailman3/lists/capi-sig.python.org/", + "Capi-SIG list", + ), + ( + "https://mail.python.org/mailman/listinfo/web-sig", + "Web-SIG list", + ), + ( + "https://discuss.python.org/t/pep-643-metadata-for-package-source-distributions/5577", + "Discourse thread", + ), + ( + "https://discuss.python.org/c/peps/", + "PEPs Discourse category", + ), + ], +) +def test_make_link_pretty(test_input, expected): + out = pep_headers._make_link_pretty(test_input) + + assert out == expected + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + (STATUS_ACCEPTED, "Normative proposal accepted for implementation"), + (STATUS_ACTIVE, "Currently valid informational guidance, or an in-use process"), + (STATUS_DEFERRED, "Inactive draft that may be taken up again at a later time"), + (STATUS_DRAFT, "Proposal under active discussion and revision"), + (STATUS_FINAL, "Accepted and implementation complete, or no longer active"), + (STATUS_REJECTED, "Formally declined and will not be accepted"), + ("April Fool!", "Formally declined and will not be accepted"), + (STATUS_SUPERSEDED, "Replaced by another succeeding PEP"), + (STATUS_WITHDRAWN, "Removed from consideration by sponsor or authors"), + (STATUS_PROVISIONAL, "Provisionally accepted but additional feedback needed"), + ], +) +def test_abbreviate_status(test_input, expected): + out = pep_headers._abbreviate_status(test_input) + + assert out == expected + + +def test_abbreviate_status_unknown(): + with pytest.raises(pep_headers.PEPParsingError): + pep_headers._abbreviate_status("an unknown status") + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ( + TYPE_INFO, + "Non-normative PEP containing background, guidelines or other information " + "relevant to the Python ecosystem", + ), + ( + TYPE_PROCESS, + "Normative PEP describing or proposing a change to a Python community " + "process, workflow or governance", + ), + ( + TYPE_STANDARDS, + "Normative PEP with a new feature for Python, implementation change for " + "CPython or interoperability standard for the ecosystem", + ), + ], +) +def test_abbreviate_type(test_input, expected): + out = pep_headers._abbreviate_type(test_input) + + assert out == expected + + +def test_abbreviate_type_unknown(): + with pytest.raises(pep_headers.PEPParsingError): + pep_headers._abbreviate_type("an unknown type") diff --git a/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_zero.py b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_zero.py new file mode 100644 index 000000000..e25d37fd2 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_processor/transform/test_pep_zero.py @@ -0,0 +1,25 @@ +import pytest +from docutils import nodes + +from pep_sphinx_extensions.pep_processor.transforms import pep_zero + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ( + nodes.reference( + "", text="user@example.com", refuri="mailto:user@example.com" + ), + 'user at example.com', + ), + ( + nodes.reference("", text="Introduction", refid="introduction"), + 'Introduction', + ), + ], +) +def test_generate_list_url(test_input, expected): + out = pep_zero._mask_email(test_input) + + assert str(out) == expected diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py new file mode 100644 index 000000000..daeb77c70 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_parser.py @@ -0,0 +1,113 @@ +import pytest + +from pep_sphinx_extensions.pep_zero_generator import parser +from pep_sphinx_extensions.pep_zero_generator.constants import ( + STATUS_ACCEPTED, + STATUS_ACTIVE, + STATUS_DEFERRED, + STATUS_DRAFT, + STATUS_FINAL, + STATUS_PROVISIONAL, + STATUS_REJECTED, + STATUS_SUPERSEDED, + STATUS_WITHDRAWN, + TYPE_INFO, + TYPE_PROCESS, + TYPE_STANDARDS, +) +from pep_sphinx_extensions.pep_zero_generator.parser import _Author + +from ..conftest import PEP_ROOT + + +def test_pep_repr(): + pep8 = parser.PEP(PEP_ROOT / "pep-0008.rst") + + assert repr(pep8) == "" + + +def test_pep_less_than(): + pep8 = parser.PEP(PEP_ROOT / "pep-0008.rst") + pep3333 = parser.PEP(PEP_ROOT / "pep-3333.rst") + + assert pep8 < pep3333 + + +def test_pep_equal(): + pep_a = parser.PEP(PEP_ROOT / "pep-0008.rst") + pep_b = parser.PEP(PEP_ROOT / "pep-0008.rst") + + assert pep_a == pep_b + + +def test_pep_details(monkeypatch): + pep8 = parser.PEP(PEP_ROOT / "pep-0008.rst") + + assert pep8.details == { + "authors": "Guido van Rossum, Barry Warsaw, Nick Coghlan", + "number": 8, + "shorthand": ":abbr:`PA (Process, Active)`", + "title": "Style Guide for Python Code", + } + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ( + "First Last ", + [_Author(full_name="First Last", email="user@example.com")], + ), + ( + "First Last", + [_Author(full_name="First Last", email="")], + ), + pytest.param( + "First Last ", + [_Author(full_name="First Last", email="user@example.com")], + marks=pytest.mark.xfail, + ), + pytest.param( + " , First Last,", + {"First Last": ""}, + marks=pytest.mark.xfail(raises=ValueError), + ), + ], +) +def test_parse_authors(test_input, expected): + # Act + out = parser._parse_author(test_input) + + # Assert + assert out == expected + + +def test_parse_authors_invalid(): + with pytest.raises(ValueError, match="Name is empty!"): + assert parser._parse_author("") + + +@pytest.mark.parametrize( + ("test_type", "test_status", "expected"), + [ + (TYPE_INFO, STATUS_DRAFT, ":abbr:`I (Informational, Draft)`"), + (TYPE_INFO, STATUS_ACTIVE, ":abbr:`IA (Informational, Active)`"), + (TYPE_INFO, STATUS_ACCEPTED, ":abbr:`IA (Informational, Accepted)`"), + (TYPE_INFO, STATUS_DEFERRED, ":abbr:`ID (Informational, Deferred)`"), + (TYPE_PROCESS, STATUS_ACCEPTED, ":abbr:`PA (Process, Accepted)`"), + (TYPE_PROCESS, STATUS_ACTIVE, ":abbr:`PA (Process, Active)`"), + (TYPE_PROCESS, STATUS_FINAL, ":abbr:`PF (Process, Final)`"), + (TYPE_PROCESS, STATUS_SUPERSEDED, ":abbr:`PS (Process, Superseded)`"), + (TYPE_PROCESS, STATUS_WITHDRAWN, ":abbr:`PW (Process, Withdrawn)`"), + (TYPE_STANDARDS, STATUS_ACCEPTED, ":abbr:`SA (Standards Track, Accepted)`"), + (TYPE_STANDARDS, STATUS_REJECTED, ":abbr:`SR (Standards Track, Rejected)`"), + (TYPE_STANDARDS, STATUS_PROVISIONAL, ":abbr:`SP (Standards Track, Provisional)`"), # fmt: skip + ], +) +def test_abbreviate_type_status(test_type, test_status, expected): + # set up dummy PEP object and monkeypatch attributes + pep = parser.PEP(PEP_ROOT / "pep-0008.rst") + pep.pep_type = test_type + pep.status = test_status + + assert pep.shorthand == expected diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py new file mode 100644 index 000000000..75c16f624 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_pep_index_generator.py @@ -0,0 +1,11 @@ +from pep_sphinx_extensions.pep_zero_generator import parser, pep_index_generator + +from ..conftest import PEP_ROOT + + +def test_create_pep_json(): + peps = [parser.PEP(PEP_ROOT / "pep-0008.rst")] + + out = pep_index_generator.create_pep_json(peps) + + assert '"url": "https://peps.python.org/pep-0008/"' in out diff --git a/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py b/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py new file mode 100644 index 000000000..1ccf8c418 --- /dev/null +++ b/pep_sphinx_extensions/tests/pep_zero_generator/test_writer.py @@ -0,0 +1,71 @@ +from pathlib import Path + +import pytest + +from pep_sphinx_extensions.pep_zero_generator import parser, writer + + +def test_pep_zero_writer_emit_text_newline(): + pep0_writer = writer.PEPZeroWriter() + pep0_writer.emit_text("my text 1") + pep0_writer.emit_newline() + pep0_writer.emit_text("my text 2") + + assert pep0_writer.output == ["my text 1", "", "my text 2"] + + +def test_pep_zero_writer_emit_title(): + pep0_writer = writer.PEPZeroWriter() + pep0_writer.emit_title("My Title") + pep0_writer.emit_subtitle("My Subtitle") + + assert pep0_writer.output == [ + "My Title", + "========", + "", + "My Subtitle", + "-----------", + "", + ] + + +@pytest.mark.parametrize( + ("test_input", "expected"), + [ + ( + "pep-9000.rst", + { + "Francis Fussyreverend": "one@example.com", + "Javier Soulfulcommodore": "two@example.com", + }, + ), + ( + "pep-9001.rst", + {"Francis Fussyreverend": "", "Javier Soulfulcommodore": ""}, + ), + ], +) +def test_verify_email_addresses(test_input, expected): + # Arrange + peps = [parser.PEP(Path(f"pep_sphinx_extensions/tests/peps/{test_input}"))] + + # Act + out = writer._verify_email_addresses(peps) + + # Assert + assert out == expected + + +def test_sort_authors(): + # Arrange + authors_dict = { + "Zebra, ZoĂ«": "zoe@example.com", + "lowercase, laurence": "laurence@example.com", + "Aardvark, Alfred": "alfred@example.com", + } + + # Act + out = writer._sort_authors(authors_dict) + + # Assert + assert out == ["Aardvark, Alfred", "lowercase, laurence", "Zebra, ZoĂ«"] diff --git a/pep_sphinx_extensions/tests/peps/pep-9000.rst b/pep_sphinx_extensions/tests/peps/pep-9000.rst new file mode 100644 index 000000000..84a117c17 --- /dev/null +++ b/pep_sphinx_extensions/tests/peps/pep-9000.rst @@ -0,0 +1,7 @@ +PEP: 9000 +Title: Test with authors with email addresses +Author: Francis Fussyreverend , + Javier Soulfulcommodore +Created: 20-Apr-2022 +Status: Draft +Type: Process diff --git a/pep_sphinx_extensions/tests/peps/pep-9001.rst b/pep_sphinx_extensions/tests/peps/pep-9001.rst new file mode 100644 index 000000000..4a1a9e115 --- /dev/null +++ b/pep_sphinx_extensions/tests/peps/pep-9001.rst @@ -0,0 +1,7 @@ +PEP: 9001 +Title: Test with authors with no email addresses +Author: Francis Fussyreverend, + Javier Soulfulcommodore +Created: 20-Apr-2022 +Status: Draft +Type: Process diff --git a/pep_sphinx_extensions/tests/peps/pep-9002.rst b/pep_sphinx_extensions/tests/peps/pep-9002.rst new file mode 100644 index 000000000..208569e03 --- /dev/null +++ b/pep_sphinx_extensions/tests/peps/pep-9002.rst @@ -0,0 +1,23 @@ +PEP:9002 +Title: Nobody expects the example PEP! +Author: Cardinal XimĂ©nez , + Cardinal Biggles + Cardinal Fang +Version: 4.0 +Python-Version: 4.0 +Sponsor: +Sponsor: +Horse-Guards: Parade +Created: 1-Jan-1989 +BDFL-Delegate: Barry! +Status: Draught +Topic: Inquisiting, Governance, Governance, packaging +Content-Type: video/quicktime +Requires: 0020,1,2,3, 7, 8 +Discussions-To: MR ALBERT SPIM, I,OOO,OO8 LONDON ROAD, OXFORD +Post-History: `2-Feb-2000 `__ + `3-Mar-2001 `__ +Resolution: + + +https://peps.python.org/pep-9002.html diff --git a/peps/conf.py b/peps/conf.py new file mode 100644 index 000000000..09b9b3ed3 --- /dev/null +++ b/peps/conf.py @@ -0,0 +1,77 @@ +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +"""Configuration for building PEPs using Sphinx.""" + +import os +from pathlib import Path +import sys + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.append(os.fspath(_ROOT)) + +# -- Project information ----------------------------------------------------- + +project = "PEPs" +master_doc = "contents" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. +extensions = [ + "pep_sphinx_extensions", + "sphinx.ext.intersphinx", + "sphinx.ext.githubpages", +] + +# The file extensions of source files. Sphinx uses these suffixes as sources. +source_suffix = { + ".rst": "pep", +} + +# List of patterns (relative to source dir) to ignore when looking for source files. +include_patterns = [ + # Required for Sphinx + "contents.rst", + # PEP files + "pep-????.rst", + # PEP ancillary files + "pep-????/*.rst", + # Documentation + "docs/*.rst", +] +exclude_patterns = [ + # PEP Template + "pep-0012/pep-NNNN.rst", +] + +# Warn on missing references +nitpicky = True + +# Intersphinx configuration +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'packaging': ('https://packaging.python.org/en/latest/', None), + 'devguide': ('https://devguide.python.org/', None), + 'py3.11': ('https://docs.python.org/3.11/', None), + 'py3.12': ('https://docs.python.org/3.12/', None), +} +intersphinx_disabled_reftypes = [] + +# -- Options for HTML output ------------------------------------------------- + +_PSE_PATH = _ROOT / "pep_sphinx_extensions" + +# HTML output settings +html_math_renderer = "maths_to_html" # Maths rendering + +# Theme settings +html_theme_path = [os.fspath(_PSE_PATH)] +html_theme = "pep_theme" # The actual theme directory (child of html_theme_path) +html_use_index = False # Disable index (we use PEP 0) +html_style = "" # must be defined here or in theme.conf, but is unused +html_permalinks = False # handled in the PEPContents transform +html_baseurl = "https://peps.python.org" # to create the CNAME file +gettext_auto_build = False # speed-ups + +templates_path = [os.fspath(_PSE_PATH / "pep_theme" / "templates")] # Theme template relative paths from `confdir` diff --git a/peps/contents.rst b/peps/contents.rst new file mode 100644 index 000000000..d791f08f8 --- /dev/null +++ b/peps/contents.rst @@ -0,0 +1,18 @@ +.. This file is placed in the public domain or under the + CC0-1.0-Universal license, whichever is more permissive. + +Python Enhancement Proposals (PEPs) +*********************************** + +This is an internal Sphinx page; please go to the :doc:`PEP Index `. + + +.. toctree:: + :maxdepth: 3 + :titlesonly: + :hidden: + :glob: + :caption: PEP Table of Contents (needed for Sphinx): + + pep-* + topic/* diff --git a/pep-0001.txt b/peps/pep-0001.rst similarity index 66% rename from pep-0001.txt rename to peps/pep-0001.rst index 2650d41a8..66da707dc 100644 --- a/pep-0001.txt +++ b/peps/pep-0001.rst @@ -26,7 +26,9 @@ documenting dissenting opinions. Because the PEPs are maintained as text files in a versioned repository, their revision history is the historical record of the -feature proposal [1]_. +feature proposal. This historical record is available by the normal git +commands for retrieving older revisions, and can also be browsed +`on GitHub `__. PEP Audience @@ -80,7 +82,7 @@ Python's Steering Council There are several references in this PEP to the "Steering Council" or "Council". This refers to the current members of the elected Steering Council described -in PEP 13 [5]_, in their role as the final authorities on whether or not PEPs +in :pep:`13`, in their role as the final authorities on whether or not PEPs will be accepted or rejected. @@ -88,7 +90,7 @@ Python's Core Developers ------------------------ There are several references in this PEP to "core developers". This refers to -the currently active Python core team members described in PEP 13 [5]_. +the currently active Python core team members described in :pep:`13`. Python's BDFL @@ -121,10 +123,10 @@ Start with an idea for Python The PEP process begins with a new idea for Python. It is highly recommended that a single PEP contain a single key proposal or new -idea. Small enhancements or patches often don't need -a PEP and can be injected into the Python development workflow with a -patch submission to the Python `issue tracker`_. The more focused the -PEP, the more successful it tends to be. The PEP editors reserve the +idea; the more focused the PEP, the more successful it tends to be. +Most enhancements and bug fixes don't need a PEP and +can be submitted directly to the `Python issue tracker`_. +The PEP editors reserve the right to reject PEP proposals if they appear too unfocused or too broad. If in doubt, split your PEP into several well-focused ones. @@ -132,9 +134,10 @@ Each PEP must have a champion -- someone who writes the PEP using the style and format described below, shepherds the discussions in the appropriate forums, and attempts to build community consensus around the idea. The PEP champion (a.k.a. Author) should first attempt to ascertain whether the idea is -PEP-able. Posting to the comp.lang.python newsgroup -(a.k.a. python-list@python.org mailing list) or the python-ideas@python.org -mailing list is the best way to go about this. +PEP-able. Posting to the `Ideas category`_ of the `Python Discourse`_ is usually +the best way to go about this, unless a more specialized venue is appropriate, +such as `Typing-SIG`_ for static typing or the `Packaging category`_ of the +Python Discourse for packaging issues. Vetting an idea publicly before going as far as writing a PEP is meant to save the potential author time. Many ideas have been brought @@ -149,7 +152,8 @@ mean it will work for most people in most areas where Python is used. Once the champion has asked the Python community as to whether an idea has any chance of acceptance, a draft PEP should be presented to -python-ideas. This gives the author a chance to flesh out the draft +the appropriate venue mentioned above. +This gives the author a chance to flesh out the draft PEP to make properly formatted, of high quality, and to address initial concerns about the proposal. @@ -157,7 +161,7 @@ initial concerns about the proposal. Submitting a PEP ---------------- -Following a discussion on python-ideas, the workflow varies based on whether +Following the above initial discussion, the workflow varies based on whether any of the PEP's co-authors are core developers. If one or more of the PEP's co-authors are core developers, they are responsible for following the process outlined below. Otherwise (i.e. none of the co-authors are core developers), @@ -181,32 +185,36 @@ corrected by the editors). The standard PEP workflow is: * You, the PEP author, fork the `PEP repository`_, and create a file named - ``pep-9999.rst`` that contains your new PEP. Use "9999" as your draft PEP - number. + :file:`pep-{NNNN}.rst` that contains your new PEP. :samp:`{NNNN}` should be the next + available PEP number not used by a published or in-PR PEP. + +* In the "PEP:" header field, enter the PEP number that matches your filename + as your draft PEP number. * In the "Type:" header field, enter "Standards Track", "Informational", or "Process" as appropriate, and for the "Status:" field enter "Draft". For full details, see `PEP Header Preamble`_. -* Update `.github/CODEOWNERS` [7]_ such that any core developer co-author(s) or - sponsor are listed for your new file such that any future pull requests will - be assigned to them. +* Update `.github/CODEOWNERS`_ such that any co-author(s) or sponsors + with write access to the `PEP repository`_ are listed for your new file. + This ensures any future pull requests changing the file will be assigned + to them. * Push this to your GitHub fork and submit a pull request. * The PEP editors review your PR for structure, formatting, and other - errors. For a reST-formatted PEP, PEP 12 is provided as a template. + errors. For a reST-formatted PEP, :pep:`12` is provided as a template. It also provides a complete introduction to reST markup that is used in PEPs. Approval criteria are: - * It sound and complete. The ideas must make technical sense. The + * It is sound and complete. The ideas must make technical sense. The editors do not consider whether they seem likely to be accepted. * The title accurately describes the content. * The PEP's language (spelling, grammar, sentence structure, etc.) - and code style (examples should match PEP 8 & PEP 7) should be - correct and conformant. The PEP will be checked for formatting - (plain text or reStructuredText) by Travis CI, and will not be - approved until this passes. + and code style (examples should match :pep:`7` & :pep:`8`) should be + correct and conformant. The PEP text will be automatically checked for + correct reStructuredText formatting when the pull request is submitted. + PEPs with invalid reST markup will not be approved. Editors are generally quite lenient about this initial review, expecting that problems will be corrected by the reviewing process. @@ -221,7 +229,7 @@ The standard PEP workflow is: Once the review process is complete, and the PEP editors approve it (note that this is *not* the same as accepting your PEP!), they will squash commit your -pull request onto master. +pull request onto main. The PEP editors will not unreasonably deny publication of a PEP. Reasons for denying PEP status include duplication of effort, being technically unsound, @@ -229,7 +237,7 @@ not providing proper motivation or addressing backwards compatibility, or not in keeping with the Python philosophy. The Steering Council can be consulted during the approval phase, and are the final arbiter of a draft's PEP-ability. -Developers with git push privileges for the `PEP repository`_ may claim PEP +Developers with write access to the `PEP repository`_ may claim PEP numbers directly by creating and committing a new PEP. When doing so, the developer must handle the tasks that would normally be taken care of by the PEP editors (see `PEP Editor Responsibilities & Workflow`_). This includes @@ -240,14 +248,10 @@ if you need assistance from PEP editors, mention ``@python/pep-editors`` on GitHub. As updates are necessary, the PEP author can check in new versions if they -(or a collaborating developer) have git push privileges. - -After a PEP number has been assigned, a draft PEP may be discussed further on -python-ideas (getting a PEP number assigned early can be useful for ease of +(or a collaborating developer) have write access to the `PEP repository`_. +Getting a PEP number assigned early can be useful for ease of reference, especially when multiple draft PEPs are being considered at the -same time). Eventually, all Standards Track PEPs must be sent to the -`python-dev list `__ for review as described -in the next section. +same time. Standards Track PEPs consist of two parts, a design document and a reference implementation. It is generally recommended that at least a @@ -255,23 +259,76 @@ prototype implementation be co-developed with the PEP, as ideas that sound good in principle sometimes turn out to be impractical when subjected to the test of implementation. + +Discussing a PEP +---------------- + +As soon as a PEP number has been assigned +and the draft PEP is committed to the `PEP repository`_, +a discussion thread for the PEP should be created +to provide a central place to discuss and review its contents, and the +PEP should be updated so that the ``Discussions-To`` header links to it. + +The PEP authors (or sponsor, if applicable) may select any reasonable venue +for the discussion, so long as the the following criteria are met: + +* The forum is appropriate to the PEP's topic. +* The thread is publicly available on the web so that all interested parties + can participate. +* The discussion is subject to the `Python Community Code of Conduct + `_. +* A direct link to the current discussion thread is provided in the PEP + under the ``Discussions-To`` header. + +The `PEPs category`_ of the `Python Discourse`_ +is the preferred choice for most new PEPs, +whereas historically the `Python-Dev`_ mailing list was commonly used. +Some specialized topics have specific venues, such as +`Typing-SIG`_ for typing PEPs or the `Packaging category`_ on the Python +Discourse for packaging PEPs. If the PEP authors are unsure of the best venue, +the PEP Sponsor and PEP editors can advise them accordingly. + +If a PEP undergoes a significant re-write or other major, substantive +changes to its proposed specification, a new thread should typically be created +in the chosen venue to solicit additional feedback. If this occurs, the +``Discussions-To`` link must be updated and a new ``Post-History`` entry added +pointing to this new thread. + +If it is not chosen as the discussion venue, +a brief announcement post should be made to the `PEPs category`_ +with at least a link to the rendered PEP and the ``Discussions-To`` thread +when the draft PEP is committed to the repository +and if a major-enough change is made to trigger a new thread. + PEP authors are responsible for collecting community feedback on a PEP -before submitting it for review. However, wherever possible, long -open-ended discussions on public mailing lists should be avoided. -Strategies to keep the discussions efficient include: setting up a -separate SIG mailing list for the topic, having the PEP author accept -private comments in the early design phases, setting up a wiki page, etc. +before submitting it for review. However, to avoid long-winded and +open-ended discussions, strategies such as soliciting private or more +narrowly-tailored feedback in the early design phase, +collaborating with other community members with expertise in the PEP's +subject matter, and picking an appropriately-specialized discussion for the +PEP's topic (if applicable) should be considered. PEP authors should use their discretion here. +Once the PEP is assigned a number and committed to the PEP repository, +substantive issues should generally be discussed on the canonical public +thread, as opposed to private channels, GitHub pull request reviews or +unrelated venues. This ensures everyone can follow and contribute, +avoids fragmenting the discussion, +and makes sure it is fully considered as part of the PEP review process. +Comments, support, concerns and other feedback on this designated thread +are a critical part of what the Steering Council or PEP-Delegate will +consider when reviewing the PEP. + PEP Review & Resolution ----------------------- Once the authors have completed a PEP, they may request a review for style and consistency from the PEP editors. - -However, content review and final acceptance of the PEP must be requested of the -core developers, usually via an email to the python-dev mailing list. +However, content review and acceptance of the PEP is ultimately the +responsibility of the Steering Council, which is formally initiated by +opening a `Steering Council issue`_ once the authors (and sponsor, if any) +determine the PEP is ready for final review and resolution. To expedite the process in selected cases (e.g. when a change is clearly beneficial and ready to be accepted, but the PEP hasn't been formally submitted @@ -279,18 +336,31 @@ for review yet), the Steering Council may also initiate a PEP review, first notifying the PEP author(s) and giving them a chance to make revisions. The final authority for PEP approval is the Steering Council. However, whenever -a new PEP is put forward, any core developer that believes they are suitably -experienced to make the final decision on that PEP may offer to serve as -the PEP-Delegate for that PEP, and they will then have the authority to approve -(or reject) that PEP. Individuals taking on this responsibility are free to seek +a new PEP is put forward, any core developer who believes they are suitably +experienced to make the final decision on that PEP may offer to serve as its +PEP-Delegate by `notifying the Steering Council `_ +of their intent. If the Steering Council approves their offer, +the PEP-Delegate will then have the authority to approve or reject that PEP. + +The term "PEP-Delegate" is used under the Steering Council governance model +for the PEP's designated decision maker, +who is recorded in the "PEP-Delegate" field in the PEP's header. +The term "BDFL-Delegate" is a deprecated alias for PEP-Delegate, a legacy of +the time when when Python was led by `a BDFL `_. +Any legacy references to "BDFL-Delegate" should be treated as equivalent to +"PEP-Delegate". + +An individual offering to nominate themselves as a PEP-Delegate must notify +the relevant authors and (when present) the sponsor for the PEP, and submit +their request to the Steering Council +(which can be done via a `new issue `_ ). +Those taking on this responsibility are free to seek additional guidance from the Steering Council at any time, and are also expected to take the advice and perspectives of other core developers into account. -The designated decision maker for each PEP is recorded in the "PEP-Delegate" -header in the PEP. - -Such self-nominations are accepted by default, but may be explicitly declined by -the Steering Council. Possible reasons for the Steering Council declining a +The Steering Council will generally approve such self-nominations by default, +but may choose to decline them. +Possible reasons for the Steering Council declining a self-nomination as PEP-Delegate include, but are not limited to, perceptions of a potential conflict of interest (e.g. working for the same organisation as the PEP submitter), or simply considering another potential PEP-Delegate to be @@ -311,13 +381,6 @@ replacement can be found). In the event that a PEP-Delegate is asked to step down, this will overrule any prior acceptance or rejection of the PEP, and it will revert to Draft status. -With the approval of the Steering Council, PEP review and resolution may also -occur on a list other than python-dev (for example, distutils-sig for packaging -related PEPs that don't immediately affect the standard library). In these -cases, the "Discussions-To" heading in the PEP will identify the appropriate -alternative list where discussion, review and pronouncement on the PEP will -occur. - When such standing delegations are put in place, the Steering Council will maintain sufficient public records to allow subsequent Councils, the core developers, and the wider Python community to understand the delegations that @@ -331,9 +394,13 @@ implementation, if applicable, must be solid and must not complicate the interpreter unduly. Finally, a proposed enhancement must be "pythonic" in order to be accepted by the Steering Council. (However, "pythonic" is an imprecise term; it may be defined as whatever is acceptable to -the Steering Council. This logic is intentionally circular.) See PEP 2 [2]_ +the Steering Council. This logic is intentionally circular.) See :pep:`2` for standard library module acceptance criteria. +Except where otherwise approved by the Steering Council, +pronouncements of PEP resolution will be posted to the +`PEPs category`_ on the `Python Discourse`_. + Once a PEP has been accepted, the reference implementation must be completed. When the reference implementation is complete and incorporated into the main source code repository, the status will be changed to "Final". @@ -350,7 +417,7 @@ been included in a Python release*. Wherever possible, it is considered preferable to reduce the scope of a proposal to avoid the need to rely on the "Provisional" status (e.g. by deferring some features to later PEPs), as this status can lead to version compatibility -challenges in the wider Python ecosystem. PEP 411 provides additional details +challenges in the wider Python ecosystem. :pep:`411` provides additional details on potential use cases for the Provisional status. A PEP can also be assigned the status "Deferred". The PEP author or an @@ -365,9 +432,9 @@ themselves has decided that the PEP is actually a bad idea, or has accepted that a competing proposal is a better alternative. When a PEP is Accepted, Rejected or Withdrawn, the PEP should be updated -accordingly. In addition to updating the status field, at the very least -the Resolution header should be added with a link to the relevant post -in the python-dev mailing list archives. +accordingly. In addition to updating the Status field, at the very least +the Resolution header should be added with a direct link +to the relevant post making a decision on the PEP. PEPs can also be superseded by a different PEP, rendering the original obsolete. This is intended for Informational PEPs, where version 2 of @@ -375,7 +442,8 @@ an API can replace version 1. The possible paths of the status of PEPs are as follows: -.. image:: pep-0001-process_flow.png +.. image:: pep-0001/process_flow.svg + :class: invert-in-dark-mode :alt: PEP process flow diagram While not shown in the diagram, "Accepted" PEPs may technically move to @@ -388,26 +456,31 @@ deprecation process (which may require a new PEP providing the rationale for the deprecation). Some Informational and Process PEPs may also have a status of "Active" -if they are never meant to be completed. E.g. PEP 1 (this PEP). +if they are never meant to be completed. E.g. :pep:`1` (this PEP). PEP Maintenance --------------- -In general, Standards track PEPs are no longer modified after they have -reached the Final state. Once a PEP has been completed, the Language and -Standard Library References become the formal documentation of the expected -behavior. +In general, PEPs are no longer substantially modified after they have reached +the Accepted, Final, Rejected or Superseded state. Once resolution is reached, +a PEP is considered a historical document rather than a living specification. +Formal documentation of the expected behavior should be maintained elsewhere, +such as the `Language Reference`_ for core features, the `Library Reference`_ +for standard library modules or the `PyPA Specifications`_ for packaging. If changes based on implementation experience and user feedback are made to -Standards track PEPs while in the Accepted or Provisional State, those changes -should be noted in the PEP, such that the PEP accurately describes the state of +Standards track PEPs while in the Provisional or (with SC approval) Accepted +state, they should be noted in the PEP, such that the PEP accurately describes the implementation at the point where it is marked Final. -Informational and Process PEPs may be updated over time to reflect changes -to development practices and other details. The precise process followed in -these cases will depend on the nature and purpose of the PEP being updated. +Active (Informational and Process) PEPs may be updated over time to reflect +changes to development practices and other details. The precise process +followed in these cases will depend on the nature and purpose of the PEP +in question. +Occasionally, a Deferred or even a Withdrawn PEP may be resurrected +with major updates, but it is often better to just propose a new one. What belongs in a successful PEP? @@ -415,7 +488,7 @@ What belongs in a successful PEP? Each PEP should have the following parts/sections: -1. Preamble -- RFC 822 style headers containing meta-data about the +1. Preamble -- :rfc:`2822` style headers containing meta-data about the PEP, including the PEP number, a short descriptive title (limited to a maximum of 44 characters), the names, and optionally the contact info for each author, etc. @@ -496,7 +569,8 @@ Each PEP should have the following parts/sections: ready for consideration are complete and reduces people duplicating prior discussion. -12. References -- A collection of URLs used as references through the PEP. +12. Footnotes -- A collection of footnotes cited in the PEP, and + a place to list non-inline hyperlink targets. 13. Copyright/license -- Each new PEP must be placed under a dual license of public domain and CC0-1.0-Universal_ (see this PEP for an example). @@ -506,56 +580,63 @@ PEP Formats and Templates ========================= PEPs are UTF-8 encoded text files using the reStructuredText_ format. -ReStructuredText_ allows for rich markup that is still quite easy to -read, but also results in good-looking and functional HTML. PEP 12 -contains instructions and a template [4]_ for reStructuredText PEPs. +reStructuredText allows for rich markup that is still quite easy to +read, but also results in good-looking and functional HTML. :pep:`12` +contains instructions and a :pep:`PEP template <12#suggested-sections>`. -The PEP text files are automatically converted to HTML [6]_ for easier -`online reading `__. +The PEP text files are automatically +`converted to HTML `__ +(via a `Sphinx `_-based :pep:`build system <676>`) +for easier `online reading `__. PEP Header Preamble =================== -Each PEP must begin with an RFC 822 style header preamble. The headers +Each PEP must begin with an :rfc:`2822` style header preamble. The headers must appear in the following order. Headers marked with "*" are -optional and are described below. All other headers are required. :: +optional and are described below. All other headers are required. + +.. code-block:: text PEP: Title: Author: * Sponsor: * PEP-Delegate: - * Discussions-To: + Discussions-To: Status: Type: - * Content-Type: + * Topic: + * Content-Type: text/x-rst * Requires: Created: * Python-Version: - Post-History: + Post-History: * Replaces: * Superseded-By: * Resolution: The Author header lists the names, and optionally the email addresses of all the authors/owners of the PEP. The format of the Author header -value must be +values must be: - Random J. User +.. code-block:: text -if the email address is included, and just + Random J. User + +if the email address is included, and just: + +.. code-block:: text Random J. User -if the address is not given. For historical reasons the format -"address@dom.ain (Random J. User)" may appear in a PEP, however new -PEPs must use the mandated format above, and it is acceptable to -change to this format when PEPs are updated. +if the address is not given. If there are multiple authors, each should be on a separate line -following RFC 2822 continuation line conventions. Note that personal +following :rfc:`2822` continuation line conventions. Note that personal email addresses in PEPs will be obscured as a defense against spam harvesters. @@ -571,31 +652,31 @@ limitation in the email address masking for reStructuredText PEPs) *Note: The Resolution header is required for Standards Track PEPs only. It contains a URL that should point to an email message or -other web resource where the pronouncement about the PEP is made.* +other web resource where the pronouncement about +(i.e. approval or rejection of) the PEP is made.* -For a PEP where final pronouncement will be made on a list other than -python-dev, a Discussions-To header will indicate the mailing list -or URL where the pronouncement will occur. A temporary Discussions-To header -may also be used when a draft PEP is being discussed prior to submission for -pronouncement. No Discussions-To header is necessary if the PEP is being -discussed privately with the author, or on the python-list, python-ideas -or python-dev mailing lists. Note that email addresses in the -Discussions-To header will not be obscured. +The Discussions-To header provides the URL to the current +canonical discussion thread for the PEP. +For email lists, this should be a direct link to the thread in the list's +archives, rather than just a mailto: or hyperlink to the list itself. The Type header specifies the type of PEP: Standards Track, Informational, or Process. -The format of a PEP is specified with a Content-Type header. The -acceptable values are "text/plain" for plaintext PEPs (see PEP 9 [3]_) -and "text/x-rst" for reStructuredText PEPs (see PEP 12 [4]_). -reStructuredText is strongly preferred, but for backwards -compatibility plain text is currently still the default if no -Content-Type header is present. +The optional Topic header lists the special topic, if any, +the PEP belongs under. +See the :ref:`topic-index` for the existing topics. + +The format of a PEP is specified with a Content-Type header. +All PEPs must use reStructuredText (see :pep:`12`), +and have a value of ``text/x-rst``, the default. +Previously, plaintext PEPs used ``text/plain`` (see :pep:`9`). The Created header records the date that the PEP was assigned a -number, while Post-History is used to record the dates of when new -versions of the PEP are posted to python-ideas and/or python-dev. Both -headers should be in dd-mmm-yyyy format, e.g. 14-Aug-2001. +number, while Post-History is used to record the dates of and corresponding +URLs to the Discussions-To threads for the PEP, with the former as the +linked text, and the latter as the link target. +Both sets of dates should be in ``dd-mmm-yyyy`` format, e.g. ``14-Aug-2001``. Standards Track PEPs will typically have a Python-Version header which indicates the version of Python that the feature will be released with. @@ -628,23 +709,21 @@ Alternatively, all support files may be placed in a subdirectory called are no constraints on the names used in files. -Reporting PEP Bugs, or Submitting PEP Updates -============================================= +Changing Existing PEPs +====================== -How you report a bug, or submit a PEP update depends on several -factors, such as the maturity of the PEP, the preferences of the PEP -author, and the nature of your comments. For the early draft stages -of the PEP, it's probably best to send your comments and changes -directly to the PEP author. For more mature, or finished PEPs you may -want to submit corrections as a `GitHub issue`_ or `GitHub pull request`_ so that -your changes don't get lost. +Draft PEPs are freely open for discussion and proposed modification, at the +discretion of the authors, until submitted to the Steering Council or +PEP-Delegate for review and resolution. Substantive content changes should +generally be first proposed on the PEP's discussion thread listed in its +``Discussions-To`` header, while copyedits and corrections can be submitted +as a `GitHub issue`_ or `GitHub pull request`_. +PEP authors with write access to the PEP repository can update the PEPs +themselves by using ``git push`` or a GitHub PR to submit their changes. +For guidance on modifying other PEPs, consult the `PEP Maintenance`_ section. -When in doubt about where to send your changes, please check first -with the PEP author and/or a PEP editor. - -PEP authors with git push privileges for the PEP repository can update the -PEPs themselves by using "git push" or the GitHub PR interface to submit their -changes. +See the `Contributing Guide`_ for additional details, and when in doubt, +please check first with the PEP author and/or a PEP editor. Transferring PEP Ownership @@ -677,7 +756,7 @@ PEP Editor Responsibilities & Workflow A PEP editor must be added to the ``@python/pep-editors`` group on GitHub and must watch the `PEP repository`_. -Note that developers with git push privileges for the `PEP repository`_ may +Note that developers with write access to the `PEP repository`_ may handle the tasks that would normally be taken care of by the PEP editors. Alternately, even developers may request assistance from PEP editors by mentioning ``@python/pep-editors`` on GitHub. @@ -696,13 +775,13 @@ For each new PEP that comes in an editor does the following: * The file name extension is correct (i.e. ``.rst``). -* Make sure the appropriate core developer(s) is added to - `.github/CODEOWNERS` [7]_. +* Ensure that everyone listed as a sponsor or co-author of the PEP who has write + access to the `PEP repository`_ is added to `.github/CODEOWNERS`_. * Skim the PEP for obvious defects in language (spelling, grammar, sentence structure, etc.), and code style (examples should conform to - PEP 8 & PEP 7). Editors may correct problems themselves, but are - not required to do so. (Text format is checked by Travis CI.) + :pep:`7` & :pep:`8`). Editors may correct problems themselves, but are + not required to do so (reStructuredText syntax is checked by the repo's CI). * If a project is portrayed as benefiting from or supporting the PEP, make sure there is some direct indication from the project included to make the support @@ -711,41 +790,27 @@ For each new PEP that comes in an editor does the following: If the PEP isn't ready, an editor will send it back to the author for revision, with specific instructions. If reST formatting is a -problem, ask the author(s) to use PEP 12 as a template and resubmit. +problem, ask the author(s) to use :pep:`12` as a template and resubmit. Once the PEP is ready for the repository, a PEP editor will: -* Assign a PEP number (almost always just the next available number, - but sometimes it's a special/joke number, like 666 or 3141). - (Clarification: For Python 3, numbers in the 3000s were used for - Py3k-specific proposals. But now that all new features go into - Python 3 only, the process is back to using numbers in the 100s again. - Remember that numbers below 100 are meta-PEPs.) +* Check that the author has selected a valid PEP number or assign them a + number if they have not (almost always just the next available number, but + sometimes it's a special/joke number, like 666 or 3141). + + Remember that numbers below 100 are meta-PEPs. * Check that the author has correctly labeled the PEP's type ("Standards Track", "Informational", or "Process"), and marked its status as "Draft". -* Add the PEP to a local fork of the PEP repository. For workflow - instructions, follow `The Python Developers Guide `_ +* Ensure all CI build and lint checks pass without errors, + and there are no obvious issues in the rendered preview output. - The git repo for the peps is:: +* Merge the new (or updated) PEP. - https://github.com/python/peps - -* Run ``./genpepindex.py`` and ``./pep2html.py `` to ensure they - are generated without errors. If either triggers errors, then the web site - will not be updated to reflect the PEP changes. - -* Commit and push the new (or updated) PEP - -* Monitor python.org to make sure the PEP gets added to the site - properly. If it fails to appear, running ``make`` will build all of the - current PEPs. If any of these are triggering errors, they must be - corrected before any PEP will update on the site. - -* Send email back to the PEP author with next steps (post to - python-list & -dev). +* Inform the author of the next steps (open a discussion thread and + update the PEP with it, post an announcement, etc). Updates to existing PEPs should be submitted as a `GitHub pull request`_. @@ -759,57 +824,52 @@ administrative & editorial part (which is generally a low volume task). Resources: -* `Index of Python Enhancement Proposals `_ +* `Index of Python Enhancement Proposals `_ * `Following Python's Development - `_ + `_ -* `Python Developer's Guide `_ - -* `Frequently Asked Questions for Developers - `_ +* `Python Developer's Guide `_ -References and Footnotes -======================== +Footnotes +========= -.. [1] This historical record is available by the normal git commands - for retrieving older revisions, and can also be browsed via HTTP here: - https://github.com/python/peps +.. _.github/CODEOWNERS: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners -.. [2] PEP 2, Procedure for Adding New Modules - (http://www.python.org/dev/peps/pep-0002) - -.. [3] PEP 9, Sample Plaintext PEP Template - (http://www.python.org/dev/peps/pep-0009) - -.. [4] PEP 12, Sample reStructuredText PEP Template - (http://www.python.org/dev/peps/pep-0012) - -.. [5] PEP 13, Python Language Governance - (http://www.python.org/dev/peps/pep-0013) - -.. [6] More details on the PEP rendering and publication process can be found - in the PEPs repo README at - https://github.com/python/peps/blob/master/README.rst - -.. [7] `CODEOWNERS` documentation - (https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners) - -.. _issue tracker: - http://bugs.python.org/ +.. _Python issue tracker: https://github.com/python/cpython/issues .. _CC0-1.0-Universal: https://choosealicense.com/licenses/cc0-1.0/ -.. _reStructuredText: http://docutils.sourceforge.net/rst.html - -.. _Docutils: http://docutils.sourceforge.net/ +.. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html .. _PEP repository: https://github.com/python/peps -.. _`GitHub pull request`: https://github.com/python/peps/pulls +.. _GitHub pull request: https://github.com/python/peps/pulls -.. _`GitHub issue`: https://github.com/python/peps/issues +.. _GitHub issue: https://github.com/python/peps/issues + +.. _Steering Council issue: https://github.com/python/steering-council/issues/new/choose + +.. _Python-Dev: https://mail.python.org/mailman3/lists/python-dev.python.org/ + +.. _Python Discourse: https://discuss.python.org/ + +.. _Ideas category: https://discuss.python.org/c/ideas/ + +.. _PEPs category: https://discuss.python.org/c/peps/ + +.. _Typing-SIG: https://mail.python.org/mailman3/lists/typing-sig.python.org/ + +.. _Packaging category: https://discuss.python.org/c/packaging/ + +.. _Language Reference: https://docs.python.org/3/reference/index.html + +.. _Library Reference: https://docs.python.org/3/library/index.html + +.. _PyPA Specifications: https://packaging.python.org/en/latest/specifications/ + +.. _Contributing Guide: https://github.com/python/peps/blob/main/CONTRIBUTING.rst Copyright diff --git a/pep-0001/process_flow.svg b/peps/pep-0001/process_flow.svg similarity index 100% rename from pep-0001/process_flow.svg rename to peps/pep-0001/process_flow.svg diff --git a/peps/pep-0002.rst b/peps/pep-0002.rst new file mode 100644 index 000000000..0cfbfe179 --- /dev/null +++ b/peps/pep-0002.rst @@ -0,0 +1,64 @@ +PEP: 2 +Title: Procedure for Adding New Modules +Version: $Revision$ +Last-Modified: $Date$ +Author: Brett Cannon , + Martijn Faassen +Status: Active +Type: Process +Content-Type: text/x-rst +Created: 07-Jul-2001 +Post-History: 07-Jul-2001, 09-Mar-2002 + + +Introduction +============ + +The Python Standard Library contributes significantly to Python's +success. The language comes with "batteries included", so it is easy +for people to become productive with just the standard library alone. +It is therefore important that the usefulness of the standard library +be maintained. + +Due to the visibility and importance of the standard library, it must +be maintained thoughtfully. As such, any code within it must be +maintained by Python's development team which leads to a perpetual +cost to each addition made. There is also added cognitive load for +users in familiarizing themselves with what is in the standard +library to be considered. + +New functionality is commonly added to the library in the form of new +modules. This PEP will describe the procedure for the *addition* of +new modules. :pep:`4` deals with procedures for deprecation of modules; +the *removal* of old and unused modules from the standard library. + + +Acceptance Procedure +==================== + +For top-level modules/packages, a PEP is required. The procedure for +writing a PEP is covered in :pep:`1`. + +For submodules of a preexisting package in the standard library, +additions are at the discretion of the general Python development team +and its members. + +General guidance on what modules typically are accepted into the +standard library, the overall process, etc. are covered in the +`developer's guide `_. + + +Maintenance Procedure +===================== + +Anything accepted into the standard library is expected to be +primarily maintained there, within Python's development infrastructure. +While some members of the development team may choose to maintain a +backport of a module outside of the standard library, it is up to them +to keep their external code in sync as appropriate. + + +Copyright +========= + +This document has been placed in the public domain. diff --git a/pep-0003.txt b/peps/pep-0003.rst similarity index 88% rename from pep-0003.txt rename to peps/pep-0003.rst index 2d0e9118e..185f1adfa 100644 --- a/pep-0003.txt +++ b/peps/pep-0003.rst @@ -1,8 +1,6 @@ PEP: 3 Title: Guidelines for Handling Bug Reports -Version: $Revision$ -Last-Modified: $Date$ -Author: jeremy@alum.mit.edu (Jeremy Hylton) +Author: Jeremy Hylton Status: Withdrawn Type: Process Content-Type: text/x-rst @@ -35,8 +33,8 @@ Original Guidelines out, say, what all the open Tkinter bugs are. 2. If it's a minor feature request that you don't plan to address - right away, add it to PEP 42 or ask the owner to add it for you. - If you add the bug to PEP 42, mark the bug as "feature request", + right away, add it to :pep:`42` or ask the owner to add it for you. + If you add the bug to :pep:`42`, mark the bug as "feature request", "later", and "closed"; and add a comment to the bug saying that this is the case (mentioning the PEP explicitly). @@ -69,4 +67,4 @@ Original Guidelines References ========== -.. [1] http://bugs.python.org/ +* https://bugs.python.org diff --git a/peps/pep-0004.rst b/peps/pep-0004.rst new file mode 100644 index 000000000..68b980031 --- /dev/null +++ b/peps/pep-0004.rst @@ -0,0 +1,43 @@ +PEP: 4 +Title: Deprecation of Standard Modules +Version: $Revision$ +Last-Modified: $Date$ +Author: Brett Cannon , Martin von Löwis +Status: Active +Type: Process +Content-Type: text/x-rst +Created: 01-Oct-2000 +Post-History: + + +Introduction +============ + +When new modules were added to the standard Python library in the +past, it was not possible to foresee whether they would still be +useful in the future. Even though Python "Comes With Batteries +Included", batteries may discharge over time. Carrying old modules +around is a burden on the maintainer, especially when there is no +interest in the module anymore. + +At the same time, removing a module from the distribution is +difficult, as it is not known in general whether anybody is still +using it. This PEP defines a procedure for removing modules from the +standard Python library. Usage of a module may be 'deprecated', which +means that it may be removed from a future Python release. + + +Procedure for declaring a module deprecated +=========================================== + +To remove a top-level module/package from the standard library, a PEP +is required. The deprecation process is outlined in :pep:`387`. + +For removing a submodule of a package in the standard library, +:pep:`387` must be followed, but a PEP is not required. + + +Copyright +========= + +This document has been placed in the public domain. diff --git a/pep-0005.txt b/peps/pep-0005.rst similarity index 94% rename from pep-0005.txt rename to peps/pep-0005.rst index 8fdd69bf1..d829e9373 100644 --- a/pep-0005.txt +++ b/peps/pep-0005.rst @@ -1,8 +1,6 @@ PEP: 5 Title: Guidelines for Language Evolution -Version: $Revision$ -Last-Modified: $Date$ -Author: paul@prescod.net (Paul Prescod) +Author: Paul Prescod Status: Active Type: Process Content-Type: text/x-rst @@ -75,9 +73,3 @@ Steps For Introducing Backwards-Incompatible Features the backwards incompatible version. Users will have at least a year to test their programs and migrate them from use of the deprecated construct to the alternative one. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: \ No newline at end of file diff --git a/pep-0006.txt b/peps/pep-0006.rst similarity index 91% rename from pep-0006.txt rename to peps/pep-0006.rst index 93cb72d84..fd5389929 100644 --- a/pep-0006.txt +++ b/peps/pep-0006.rst @@ -1,13 +1,11 @@ PEP: 6 Title: Bug Fix Releases -Version: $Revision$ -Last-Modified: $Date$ -Author: aahz@pythoncraft.com (Aahz), anthony@interlink.com.au (Anthony Baxter) +Author: Aahz , Anthony Baxter Status: Active Type: Process Content-Type: text/x-rst Created: 15-Mar-2001 -Post-History: 15-Mar-2001 18-Apr-2001 19-Aug-2004 +Post-History: 15-Mar-2001, 18-Apr-2001, 19-Aug-2004 @@ -47,7 +45,7 @@ Prohibitions Bug fix releases are required to adhere to the following restrictions: -1. There must be zero syntax changes. All `.pyc` and `.pyo` files must +1. There must be zero syntax changes. All ``.pyc`` and ``.pyo`` files must work (no regeneration needed) with all bugfix releases forked off from a major release. @@ -143,7 +141,7 @@ the maintenance branch, or else mark the patch in the commit message. In addition, anyone from the Python community is free to suggest patches for inclusion. Patches may be submitted specifically for -bugfix releases; they should follow the guidelines in PEP 3 [2]_. In +bugfix releases; they should follow the guidelines in :pep:`3`. In general, though, it's probably better that a bug in a specific release also be fixed on the HEAD as well as the branch. @@ -194,7 +192,7 @@ sticking with a strict bug fix policy. Following feedback from the BDFL and others, the draft PEP was written containing an expanded bugfix release cycle that permitted any previous major release to obtain patches and also relaxed the strict -bug fix requirement (mainly due to the example of PEP 235 [3]_, which +bug fix requirement (mainly due to the example of :pep:`235`, which could be argued as either a bug fix or a feature). Discussion then mostly moved to python-dev, where BDFL finally issued @@ -213,21 +211,7 @@ References .. [1] http://www.tcl.tk/cgi-bin/tct/tip/28.html -.. [2] PEP 3, Guidelines for Handling Bug Reports, Hylton - (http://www.python.org/dev/peps/pep-0003/) - -.. [3] PEP 235, Import on Case-Insensitive Platforms, Peters - (http://www.python.org/dev/peps/pep-0235/) - - Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0007.txt b/peps/pep-0007.rst similarity index 89% rename from pep-0007.txt rename to peps/pep-0007.rst index 4278e4c12..d88373f29 100644 --- a/pep-0007.txt +++ b/peps/pep-0007.rst @@ -15,7 +15,7 @@ Introduction This document gives coding conventions for the C code comprising the C implementation of Python. Please see the companion informational PEP -describing style guidelines for Python code [1]_. +describing :pep:`style guidelines for Python code <8>`. Note, rules are there to be broken. Two good reasons to break a particular rule: @@ -31,13 +31,11 @@ particular rule: C dialect ========= -* Python versions before 3.6 use ANSI/ISO standard C (the 1989 version - of the standard). This means (amongst many other things) that all - declarations must be at the top of a block (not necessarily at the - top of function). +* Python 3.11 and newer versions use C11 without `optional features + `_. + The public C API should be compatible with C++. -* Python versions greater than or equal to 3.6 use C89 with several - select C99 features: +* Python 3.6 to 3.10 use C89 with several select C99 features: - Standard integer types in ```` and ````. We require the fixed width integer types. @@ -47,19 +45,22 @@ C dialect - booleans - C++-style line comments - Future C99 features may be added to this list in the future - depending on compiler support (mostly significantly MSVC). +* Python versions before 3.6 used ANSI/ISO standard C (the 1989 version + of the standard). This meant (amongst many other things) that all + declarations must be at the top of a block (not necessarily at the + top of function). -* Don't use GCC extensions (e.g. don't write multi-line strings - without trailing backslashes). +* Don't use compiler-specific extensions, such as those of GCC or MSVC + (e.g. don't write multi-line strings without trailing backslashes). * All function declarations and definitions must use full prototypes (i.e. specify the types of all arguments). -* Only use C++ style // one-line comments in Python 3.6 or later. - * No compiler warnings with major compilers (gcc, VC++, a few others). +* ``static inline`` functions should be preferred over macros in new + code. + Code lay-out ============ @@ -144,10 +145,12 @@ Code lay-out * For external functions and variables, we always have a declaration in an appropriate header file in the "Include" directory, which uses - the ``PyAPI_FUNC()`` macro, like this:: + the ``PyAPI_FUNC()`` macro and ``PyAPI_DATA()`` macro, like this:: PyAPI_FUNC(PyObject *) PyObject_Repr(PyObject *); + PyAPI_DATA(PyTypeObject) PySuper_Type; + Naming conventions ================== @@ -217,13 +220,6 @@ Documentation Strings not all do; the MSVC compiler is known to complain about this. -References -========== - -.. [1] PEP 8, "Style Guide for Python Code", van Rossum, Warsaw - (http://www.python.org/dev/peps/pep-0008) - - Copyright ========= diff --git a/pep-0008.txt b/peps/pep-0008.rst similarity index 96% rename from pep-0008.txt rename to peps/pep-0008.rst index f4ae81d5c..6c4ac9099 100644 --- a/pep-0008.txt +++ b/peps/pep-0008.rst @@ -17,10 +17,10 @@ Introduction This document gives coding conventions for the Python code comprising the standard library in the main Python distribution. Please see the -companion informational PEP describing style guidelines for the C code -in the C implementation of Python [1]_. +companion informational PEP describing :pep:`style guidelines for the C code +in the C implementation of Python <7>`. -This document and PEP 257 (Docstring Conventions) were adapted from +This document and :pep:`257` (Docstring Conventions) were adapted from Guido's original Python Style Guide essay, with some additions from Barry's style guide [2]_. @@ -38,7 +38,7 @@ A Foolish Consistency is the Hobgoblin of Little Minds One of Guido's key insights is that code is read much more often than it is written. The guidelines provided here are intended to improve the readability of code and make it consistent across the wide -spectrum of Python code. As PEP 20 says, "Readability counts". +spectrum of Python code. As :pep:`20` says, "Readability counts". A style guide is about consistency. Consistency with this style guide is important. Consistency within a project is more important. @@ -227,8 +227,8 @@ parentheses. These should be used in preference to using a backslash for line continuation. Backslashes may still be appropriate at times. For example, long, -multiple ``with``-statements cannot use implicit continuation, so -backslashes are acceptable:: +multiple ``with``-statements could not use implicit continuation +before Python 3.10, so backslashes were acceptable for that case:: with open('/path/to/some/file/you/want/to/read') as file_1, \ open('/path/to/some/file/being/written', 'w') as file_2: @@ -296,7 +296,7 @@ related one-liners (e.g. a set of dummy implementations). Use blank lines in functions, sparingly, to indicate logical sections. Python accepts the control-L (i.e. ^L) form feed character as -whitespace; Many tools treat these characters as page separators, so +whitespace; many tools treat these characters as page separators, so you may use them to separate pages of related sections of your file. Note, some editors and web-based code viewers may not recognize control-L as a form feed and will show another glyph in its place. @@ -431,7 +431,7 @@ characters, however, use the other one to avoid backslashes in the string. It improves readability. For triple-quoted strings, always use double quote characters to be -consistent with the docstring convention in PEP 257. +consistent with the docstring convention in :pep:`257`. Whitespace in Expressions and Statements @@ -465,12 +465,12 @@ Avoid extraneous whitespace in the following situations: - Immediately before a comma, semicolon, or colon:: # Correct: - if x == 4: print x, y; x, y = y, x + if x == 4: print(x, y); x, y = y, x :: # Wrong: - if x == 4 : print x , y ; x , y = y , x + if x == 4 : print(x , y) ; x , y = y , x - However, in a slice the colon acts like a binary operator, and should have equal amounts on either side (treating it as the @@ -490,7 +490,7 @@ Avoid extraneous whitespace in the following situations: # Wrong: ham[lower + offset:upper + offset] ham[1: 9], ham[1 :9], ham[1:9 :3] - ham[lower : : upper] + ham[lower : : step] ham[ : upper] - Immediately before the open parenthesis that starts the argument @@ -705,8 +705,8 @@ letter (never alter the case of identifiers!). Block comments generally consist of one or more paragraphs built out of complete sentences, with each sentence ending in a period. -You should use two spaces after a sentence-ending period in multi- -sentence comments, except after the final sentence. +You should use one or two spaces after a sentence-ending period in +multi-sentence comments, except after the final sentence. Ensure that your comments are clear and easily understandable to other speakers of the language you are writing in. @@ -748,14 +748,14 @@ Documentation Strings --------------------- Conventions for writing good documentation strings -(a.k.a. "docstrings") are immortalized in PEP 257. +(a.k.a. "docstrings") are immortalized in :pep:`257`. - Write docstrings for all public modules, functions, classes, and methods. Docstrings are not necessary for non-public methods, but you should have a comment that describes what the method does. This comment should appear after the ``def`` line. -- PEP 257 describes good docstring conventions. Note that most +- :pep:`257` describes good docstring conventions. Note that most importantly, the ``"""`` that ends a multiline docstring should be on a line by itself:: @@ -865,8 +865,8 @@ ASCII Compatibility Identifiers used in the standard library must be ASCII compatible as described in the -`policy section `_ -of PEP 3131. +:pep:`policy section <3131#policy-specification>` +of :pep:`3131`. Package and Module Names ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -896,7 +896,7 @@ convention used only for exception names and builtin constants. Type Variable Names ~~~~~~~~~~~~~~~~~~~ -Names of type variables introduced in PEP 484 should normally use CapWords +Names of type variables introduced in :pep:`484` should normally use CapWords preferring short names: ``T``, ``AnyStr``, ``Num``. It is recommended to add suffixes ``_co`` or ``_contra`` to the variables used to declare covariant or contravariant behavior correspondingly:: @@ -1125,7 +1125,7 @@ Programming Recommendations To minimize the effort involved, the ``functools.total_ordering()`` decorator provides a tool to generate missing comparison methods. - PEP 207 indicates that reflexivity rules *are* assumed by Python. + :pep:`207` indicates that reflexivity rules *are* assumed by Python. Thus, the interpreter may swap ``y > x`` with ``x < y``, ``y >= x`` with ``x <= y``, and may swap the arguments of ``x == y`` and ``x != y``. The ``sort()`` and ``min()`` operations are guaranteed to use @@ -1159,7 +1159,7 @@ Programming Recommendations *catching* the exceptions is likely to need, rather than the locations where the exceptions are raised. Aim to answer the question "What went wrong?" programmatically, rather than only stating that - "A problem occurred" (see PEP 3151 for an example of this lesson being + "A problem occurred" (see :pep:`3151` for an example of this lesson being learned for the builtin exception hierarchy) Class naming conventions apply here, although you should add the @@ -1356,18 +1356,18 @@ Programming Recommendations Function Annotations -------------------- -With the acceptance of PEP 484, the style rules for function +With the acceptance of :pep:`484`, the style rules for function annotations have changed. -- Function annotations should use PEP 484 syntax (There are some +- Function annotations should use :pep:`484` syntax (there are some formatting recommendations for annotations in the previous section). - The experimentation with annotation styles that was recommended previously in this PEP is no longer encouraged. -- However, outside the stdlib, experiments within the rules of PEP 484 +- However, outside the stdlib, experiments within the rules of :pep:`484` are now encouraged. For example, marking up a large third party - library or application with PEP 484 style type annotations, + library or application with :pep:`484` style type annotations, reviewing how easy it was to add those annotations, and observing whether their presence increases code understandability. @@ -1382,7 +1382,7 @@ annotations have changed. near the top of the file; this tells type checkers to ignore all annotations. (More fine-grained ways of disabling complaints from - type checkers can be found in PEP 484.) + type checkers can be found in :pep:`484`.) - Like linters, type checkers are optional, separate tools. Python interpreters by default should not issue any messages due to type @@ -1391,7 +1391,7 @@ annotations have changed. - Users who don't want to use type checkers are free to ignore them. However, it is expected that users of third party library packages may want to run type checkers over those packages. For this purpose - PEP 484 recommends the use of stub files: .pyi files that are read + :pep:`484` recommends the use of stub files: .pyi files that are read by the type checker in preference of the corresponding .py files. Stub files can be distributed with a library, or separately (with the library author's permission) through the typeshed repo [5]_. @@ -1400,7 +1400,7 @@ annotations have changed. Variable Annotations -------------------- -PEP 526 introduced variable annotations. The style recommendations for them are +:pep:`526` introduced variable annotations. The style recommendations for them are similar to those on function annotations described above: - Annotations for module level variables, class and instance variables, @@ -1429,9 +1429,9 @@ similar to those on function annotations described above: class Test: result: int=0 # No spaces around equality sign -- Although the PEP 526 is accepted for Python 3.6, the variable annotation +- Although the :pep:`526` is accepted for Python 3.6, the variable annotation syntax is the preferred syntax for stub files on all versions of Python - (see PEP 484 for details). + (see :pep:`484` for details). .. rubric:: Footnotes @@ -1446,8 +1446,6 @@ similar to those on function annotations described above: References ========== -.. [1] PEP 7, Style Guide for C Code, van Rossum - .. [2] Barry's GNU Mailman style guide http://barry.warsaw.us/software/STYLEGUIDE.txt diff --git a/pep-0009.txt b/peps/pep-0009.rst similarity index 95% rename from pep-0009.txt rename to peps/pep-0009.rst index a94b4f591..107ed077d 100644 --- a/pep-0009.txt +++ b/peps/pep-0009.rst @@ -8,9 +8,11 @@ Type: Process Content-Type: text/x-rst Created: 14-Aug-2001 Post-History: -Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.html +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/2YMHVPRDWGQLA5A2FKXE2JMLM2HQEEGW/ + :: + Update As of 05-Jan-2016, this PEP is officially deprecated and replaced @@ -196,12 +198,12 @@ Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.htm References [7] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - http://www.python.org/dev/peps/pep-0001/ + http://peps.python.org/pep-0001/ If you decide to provide an explicit URL for a PEP, please use this as the URL template: - http://www.python.org/dev/peps/pep-xxxx/ + http://peps.python.org/pep-xxxx/ PEP numbers in URLs must be padded with zeros from the left, so as to be exactly 4 characters wide, however PEP numbers in the text @@ -211,10 +213,10 @@ Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.htm References [1] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - http://www.python.org/dev/peps/pep-0001/ + http://peps.python.org/pep-0001/ [2] PEP 12, Sample reStructuredText PEP Template, Goodger, Warsaw - http://www.python.org/dev/peps/pep-0012/ + http://peps.python.org/pep-0012/ [3] http://www.opencontent.org/openpub/ @@ -226,10 +228,11 @@ Resolution: https://mail.python.org/mailman/private/peps/2016-January/001165.htm -Local Variables: -mode: indented-text -indent-tabs-mode: nil -sentence-end-double-space: t -fill-column: 70 -coding: utf-8 -End: +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0010.txt b/peps/pep-0010.rst similarity index 90% rename from pep-0010.txt rename to peps/pep-0010.rst index bed820524..f27b94cd2 100644 --- a/pep-0010.txt +++ b/peps/pep-0010.rst @@ -1,8 +1,6 @@ PEP: 10 Title: Voting Guidelines -Version: $Revision$ -Last-Modified: $Date$ -Author: barry@python.org (Barry Warsaw) +Author: Barry Warsaw Status: Active Type: Process Content-Type: text/x-rst @@ -71,11 +69,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/peps/pep-0011.rst b/peps/pep-0011.rst new file mode 100644 index 000000000..63131c3a2 --- /dev/null +++ b/peps/pep-0011.rst @@ -0,0 +1,383 @@ +PEP: 11 +Title: CPython platform support +Author: Martin von Löwis , + Brett Cannon +Status: Active +Type: Process +Content-Type: text/x-rst +Created: 07-Jul-2002 +Post-History: `18-Aug-2007 `__, + `14-May-2014 `__, + `20-Feb-2015 `__, + `10-Mar-2022 `__, + + +Abstract +======== + +This PEP documents how an operating system (platform) becomes +supported in CPython, what platforms are currently supported, and +documents past support. + + +Rationale +========= + +Over time, the CPython source code has collected various pieces of +platform-specific code, which, at some point in time, was +considered necessary to use CPython on a specific platform. +Without access to this platform, it is not possible to determine +whether this code is still needed. As a result, this code may +either break during CPython's evolution, or it may become +unnecessary as the platforms evolve as well. + +Allowing these fragments to grow poses the risk of +unmaintainability: without having experts for a large number of +platforms, it is not possible to determine whether a certain +change to the CPython source code will work on all supported +platforms. + +To reduce this risk, this PEP specifies what is required for a +platform to be considered supported by CPython as well as providing a +procedure to remove code for platforms with few or no CPython +users. + +This PEP also lists what platforms *are* supported by the CPython +interpreter. This lets people know what platforms are directly +supported by the CPython development team. + + +Support tiers +============= + +Platform support is broken down into *tiers*. Each tier comes with +different requirements which lead to different promises being made +about support. + +To be promoted to a tier, steering council support is required and is +expected to be driven by team consensus. Demotion to a lower tier +occurs when the requirements of the current tier are no longer met for +a platform for an extended period of time based on the judgment of +the release manager or steering council. For platforms which no longer +meet the requirements of any tier by b1 of a new feature release, an +announcement will be made to warn the community of the pending removal +of support for the platform (e.g. in the b1 announcement). If the +platform is not brought into line for at least one of the tiers by the +first release candidate, it will be listed as unsupported in this PEP. + +Tier 1 +------ + + +- `STATUS `__ +- CI failures block releases. +- Changes which would break the ``main`` branch are not allowed to be merged; + any breakage should be fixed or reverted immediately. +- All core developers are responsible to keep ``main``, and thus these + platforms, working. +- Failures on these platforms **block a release**. + +======================== ===== +Target Triple Notes +======================== ===== +i686-pc-windows-msvc +x86_64-pc-windows-msvc +x86_64-apple-darwin BSD libc, clang +x86_64-unknown-linux-gnu glibc, gcc +======================== ===== + + +Tier 2 +------ + +- `STATUS `__ +- Must have a reliable buildbot. +- At least **two** core developers are signed up to support the platform. +- Changes which break any of these platforms are to be **fixed or + reverted within 24 hours**. +- Failures on these platforms **block a release**. + +============================= ========================== ======== +Target Triple Notes Contacts +============================= ========================== ======== +aarch64-apple-darwin clang Ned Deily, Ronald Oussoren, Dong-hee Na +aarch64-unknown-linux-gnu glibc, gcc Petr Viktorin, Victor Stinner + + glibc, clang Victor Stinner, Gregory P. Smith +powerpc64le-unknown-linux-gnu glibc, gcc Petr Viktorin, Victor Stinner +x86_64-unknown-linux-gnu glibc, clang Victor Stinner, Gregory P. Smith +============================= ========================== ======== + + +Tier 3 +------ + +- `STATUS `__ +- Must have a reliable buildbot. +- At least **one** core developer is signed up to support the platform. +- No response SLA to failures. +- Failures on these platforms do **not** block a release. + +================================ =========================== ======== +Target Triple Notes Contacts +================================ =========================== ======== +aarch64-pc-windows-msvc Steve Dower +armv7l-unknown-linux-gnueabihf Raspberry Pi OS, glibc, gcc Gregory P. Smith +powerpc64le-unknown-linux-gnu glibc, clang Victor Stinner +s390x-unknown-linux-gnu glibc, gcc Victor Stinner +wasm32-unknown-emscripten Christian Heimes, Brett Cannon +wasm32-unknown-wasi Christian Heimes, Brett Cannon +x86_64-unknown-freebsd BSD libc, clang Victor Stinner +================================ =========================== ======== + + +All other platforms +------------------- + +Support for a platform may be partial within the code base, such as +from active development around platform support or accidentally. +Code changes to platforms not listed in the above tiers may be rejected +or removed from the code base without a deprecation process if they +cause a maintenance burden or obstruct general improvements. + +Platforms not listed here may be supported by the wider Python +community in some way. If your desired platform is not listed above, +please perform a search online to see if someone is already providing +support in some form. + + +Notes +----- + +Microsoft Windows +''''''''''''''''' + +Windows versions prior to Windows 10 follow Microsoft's `Fixed Lifecycle Policy +`__, +with a mainstream support phase for 5 years after release, +where the product is generally commercially available, +and an additional 5 year extended support phase, +where paid support is still available and certain bug fixes are released. +`Extended Security Updates (ESU) +`_ +is a paid program available to high-volume enterprise customers +as a "last resort" option to receive certain security updates after extended support ends. +ESU is considered a distinct phase that follows the expiration of extended support. + +Windows 10 and later follow Microsoft's `Modern Lifecycle Policy +`__, +which varies per-product, per-version, per-edition and per-channel. +Generally, feature updates (1709, 22H2) occur every 6-12 months +and are supported for 18-36 months; +Server and IoT editions, and LTSC channel releases are supported for 5-10 years, +and the latest feature release of a major version (Windows 10, Windows 11) +generally receives new updates for at least 10 years following release. +Microsoft's `Windows Lifecycle FAQ +`_ +has more specific and up-to-date guidance. + +CPython's Windows support currently follows Microsoft's lifecycles. +A new feature release X.Y.0 will support all Windows versions +whose *extended support* phase has not yet expired. +Subsequent bug fix releases will support the same Windows versions +as the original feature release, even if no longer supported by Microsoft. +New versions of Windows released while CPython is in maintenance mode +may be supported at the discretion of the core team and release manager. + +Each feature release is built by a specific version of Microsoft +Visual Studio. That version should have mainstream support when the +release is made. Developers of extension modules will generally need +to use the same Visual Studio release; they are concerned both with +the availability of the versions they need to use, and with keeping +the zoo of versions small. The CPython source tree will keep +unmaintained build files for older Visual Studio releases, for which +patches will be accepted. Such build files will be removed from the +source tree 3 years after the extended support for the compiler has +ended (but continue to remain available in revision control). + + +Legacy C Locale +''''''''''''''' + +Starting with CPython 3.7.0, \*nix platforms are expected to provide +at least one of ``C.UTF-8`` (full locale), ``C.utf8`` (full locale) or +``UTF-8`` (``LC_CTYPE``-only locale) as an alternative to the legacy ``C`` +locale. + +Any Unicode-related integration problems that occur only in the legacy ``C`` +locale and cannot be reproduced in an appropriately configured non-ASCII +locale will be closed as "won't fix". + + +Unsupporting platforms +====================== + +If a platform drops out of tiered support, a note must be posted +in this PEP that the platform is no longer actively supported. This +note must include: + +- the name of the system +- the first release number that does not support this platform + anymore, and +- the first release where the historical support code is actively + removed + +In some cases, it is not possible to identify the specific list of +systems for which some code is used (e.g. when autoconf tests for +absence of some feature which is considered present on all +supported systems). In this case, the name will give the precise +condition (usually a preprocessor symbol) that will become +unsupported. + +At the same time, the CPython source code must be changed to +produce a build-time error if somebody tries to install CPython on +this platform. On platforms using autoconf, configure must fail. +This gives potential users of the platform a chance to step +forward and offer maintenance. + + +No-longer-supported platforms +============================= + +* | Name: MS-DOS, MS-Windows 3.x + | Unsupported in: Python 2.0 + | Code removed in: Python 2.1 + +* | Name: SunOS 4 + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: DYNIX + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: dgux + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: Minix + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: Irix 4 and --with-sgi-dl + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: Linux 1 + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: Systems defining __d6_pthread_create (configure.in) + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: Systems defining PY_PTHREAD_D4, PY_PTHREAD_D6, + or PY_PTHREAD_D7 in thread_pthread.h + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: Systems using --with-dl-dld + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: Systems using --without-universal-newlines, + | Unsupported in: Python 2.3 + | Code removed in: Python 2.4 + +* | Name: MacOS 9 + | Unsupported in: Python 2.4 + | Code removed in: Python 2.4 + +* | Name: Systems using --with-wctype-functions + | Unsupported in: Python 2.6 + | Code removed in: Python 2.6 + +* | Name: Win9x, WinME, NT4 + | Unsupported in: Python 2.6 (warning in 2.5 installer) + | Code removed in: Python 2.6 + +* | Name: AtheOS + | Unsupported in: Python 2.6 (with "AtheOS" changed to "Syllable") + | Build broken in: Python 2.7 (edit configure to re-enable) + | Code removed in: Python 3.0 + | Details: http://www.syllable.org/discussion.php?id=2320 + +* | Name: BeOS + | Unsupported in: Python 2.6 (warning in configure) + | Build broken in: Python 2.7 (edit configure to re-enable) + | Code removed in: Python 3.0 + +* | Name: Systems using Mach C Threads + | Unsupported in: Python 3.2 + | Code removed in: Python 3.3 + +* | Name: SunOS lightweight processes (LWP) + | Unsupported in: Python 3.2 + | Code removed in: Python 3.3 + +* | Name: Systems using --with-pth (GNU pth threads) + | Unsupported in: Python 3.2 + | Code removed in: Python 3.3 + +* | Name: Systems using Irix threads + | Unsupported in: Python 3.2 + | Code removed in: Python 3.3 + +* | Name: OSF* systems (issue 8606) + | Unsupported in: Python 3.2 + | Code removed in: Python 3.3 + +* | Name: OS/2 (issue 16135) + | Unsupported in: Python 3.3 + | Code removed in: Python 3.4 + +* | Name: VMS (issue 16136) + | Unsupported in: Python 3.3 + | Code removed in: Python 3.4 + +* | Name: Windows 2000 + | Unsupported in: Python 3.3 + | Code removed in: Python 3.4 + +* | Name: Windows systems where COMSPEC points to command.com + | Unsupported in: Python 3.3 + | Code removed in: Python 3.4 + +* | Name: RISC OS + | Unsupported in: Python 3.0 (some code actually removed) + | Code removed in: Python 3.4 + +* | Name: IRIX + | Unsupported in: Python 3.7 + | Code removed in: Python 3.7 + +* | Name: Systems without multithreading support + | Unsupported in: Python 3.7 + | Code removed in: Python 3.7 + + +Discussions +=========== + +* April 2022: `Consider adding a Tier 3 to tiered platform support + `_ + (Victor Stinner) +* March 2022: `Proposed tiered platform support + `_ + (Brett Cannon) +* February 2015: `Update to PEP 11 to clarify garnering platform support + `_ + (Brett Cannon) +* May 2014: `Where is our official policy of what platforms we do support? + `_ + (Brett Cannon) +* August 2007: `PEP 11 update - Call for port maintainers to step forward + `_ + (Skip Montanaro) + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0012.rst b/peps/pep-0012.rst similarity index 55% rename from pep-0012.rst rename to peps/pep-0012.rst index b6a351a3e..7ad01dc43 100644 --- a/pep-0012.rst +++ b/peps/pep-0012.rst @@ -7,26 +7,30 @@ Status: Active Type: Process Content-Type: text/x-rst Created: 05-Aug-2002 -Post-History: 30-Aug-2002 +Post-History: `30-Aug-2002 `__ +.. highlight:: rst + .. note:: - For those who have written a PEP before, there is a template_. + For those who have written a PEP before, there is a template_ + (which is included as a file in the `PEPs repository`_). Abstract ======== This PEP provides a boilerplate or sample template for creating your own reStructuredText PEPs. In conjunction with the content guidelines -in PEP 1 [1]_, this should make it easy for you to conform your own +in :pep:`1`, this should make it easy for you to conform your own PEPs to the format outlined below. Note: if you are reading this PEP via the web, you should first grab the text (reStructuredText) source of this PEP in order to complete the steps below. **DO NOT USE THE HTML FILE AS YOUR TEMPLATE!** -The source for this (or any) PEP can be found in the PEPs repository, -viewable on the web at https://github.com/python/peps/ . +The source for this (or any) PEP can be found in the +`PEPs repository `_, +as well as via a link at the bottom of each PEP. Rationale @@ -49,21 +53,17 @@ How to Use This Template To use this template you must first decide whether your PEP is going to be an Informational or Standards Track PEP. Most PEPs are Standards Track because they propose a new feature for the Python -language or standard library. When in doubt, read PEP 1 for details, +language or standard library. When in doubt, read :pep:`1` for details, or open a tracker issue on the PEPs repo to ask for assistance. Once you've decided which type of PEP yours is going to be, follow the directions below. - Make a copy of this file (the ``.rst`` file, **not** the HTML!) and - perform the following edits. Name the new file ``pep-9999.rst`` if - you don't yet have a PEP number assignment, or ``pep-NNNN.rst`` if - you do. Those with push permissions are welcome to claim the next - available number (ignoring the special blocks 3000 and 8000, and a - handful of special allocations - currently 666, 754, and 801) and - push it directly. + perform the following edits. Name the new file :file:`pep-{NNNN}.rst`, using + the next available number (not used by a published or in-PR PEP). -- Replace the "PEP: 12" header with "PEP: 9999" or "PEP: NNNN", +- Replace the "PEP: 12" header with "PEP: NNNN", matching the file name. Note that the file name should be padded with zeros (eg ``pep-0012.rst``), but the header should not (``PEP: 12``). @@ -79,11 +79,13 @@ directions below. - If none of the authors are Python core developers, include a Sponsor header with the name of the core developer sponsoring your PEP. -- For many PEPs, discussions will take place on python-ideas@python.org - and/or python-dev@python.org. If there is another mailing list or - public forum more appropriate for discussion of your new feature, - add a Discussions-To header right after the Author header. Most - Informational PEPs don't need a Discussions-To header. +- Add the direct URL of the PEP's canonical discussion thread + (on e.g. Python-Dev, Discourse, etc) under the Discussions-To header. + If the thread will be created after the PEP is submitted as an official + draft, it is okay to just list the venue name initially, but remember to + update the PEP with the URL as soon as the PEP is successfully merged + to the PEPs repository and you create the corresponding discussion thread. + See :pep:`PEP 1 <1#discussing-a-pep>` for more details. - Change the Status header to "Draft". @@ -109,20 +111,32 @@ directions below. first appearance in. Do not use an alpha or beta release designation here. Thus, if the last version of Python was 2.2 alpha 1 and you're hoping to get your new feature into Python 2.2, set the - header to:: + header to: + + .. code-block:: email Python-Version: 2.2 -- Leave Post-History alone for now; you'll add dates to this header - each time you post your PEP to the designated discussion forum (see - the Discussions-To header above). If you posted your PEP to the lists - on August 14, 2001 and September 3, 2001, the Post-History header - would look like:: +- Add a Topic header if the PEP belongs under one shown at the :ref:`topic-index`. + Most PEPs don't. - Post-History: 14-Aug-2001, 03-Sept-2001 +- Leave Post-History alone for now; you'll add dates and corresponding links + to this header each time you post your PEP to the designated discussion forum + (and update the Discussions-To header with said link, as above). + For each thread, use the date (in the ``dd-mmm-yyy`` format) as the + linked text, and insert the URLs inline as anonymous reST `hyperlinks`_, + with commas in between each posting. - You must manually add new dates and commit them (with a pull request - if you don't have push privileges). + If you posted threads for your PEP on August 14, 2001 and September 3, 2001, + the Post-History header would look like, e.g.: + + .. code-block:: email + + Post-History: `14-Aug-2001 `__, + `03-Sept-2001 `__ + + You should add the new dates/links here as soon as you post a + new discussion thread. - Add a Replaces header if your PEP obsoletes an earlier PEP. The value of this header is the number of the PEP that your new PEP is @@ -136,38 +150,35 @@ directions below. prohibition of tab characters and the indentation requirements. See "Suggested Sections" below for a template of sections to include. -- Update your References and Copyright section. Usually you'll place - your PEP into the public domain, in which case just leave the - Copyright section alone. Alternatively, you can use the `Open - Publication License`__, but public domain is still strongly - preferred. +- Update your Footnotes section, listing any footnotes and + non-inline link targets referenced by the text. - __ http://www.opencontent.org/openpub/ +- Run ``./build.py`` to ensure the PEP is rendered without errors, + and check that the output in :file:`build/pep-{NNNN}.html` looks as you intend. -- Leave the Emacs stanza at the end of this file alone, including the - formfeed character ("^L", or ``\f``). - -- Create a pull request against the https://github.com/python/peps - repository. +- Create a pull request against the `PEPs repository`_. For reference, here are all of the possible header fields (everything in brackets should either be replaced or have the field removed if it has a leading ``*`` marking it as optional and it does not apply to -your PEP):: +your PEP): + +.. code-block:: email PEP: [NNN] Title: [...] Author: [Full Name ] Sponsor: *[Full Name ] PEP-Delegate: - Discussions-To: *[...] + Discussions-To: [URL] Status: Draft Type: [Standards Track | Informational | Process] + Topic: *[Governance | Packaging | Release | Typing] Content-Type: text/x-rst Requires: *[NNN] Created: [DD-MMM-YYYY] Python-Version: *[M.N] - Post-History: [DD-MMM-YYYY] + Post-History: [`DD-MMM-YYYY `__] Replaces: *[NNN] Superseded-By: *[NNN] Resolution: @@ -186,14 +197,9 @@ illustrate the plaintext markup. General ------- -You must adhere to the Emacs convention of adding two spaces at the -end of every sentence. You should fill your paragraphs to column 70, -but under no circumstances should your lines extend past column 79. -If your code samples spill over column 79, you should rewrite them. - -Tab characters must never appear in the document at all. A PEP should -include the standard Emacs stanza included by example at the bottom of -this PEP. +Lines should usually not extend past column 79, +excepting URLs and similar circumstances. +Tab characters must never appear in the document at all. Section Headings @@ -301,33 +307,52 @@ Literal Blocks By the way, this is a comment, described in "Comments" below. -Literal blocks are used for code samples or preformatted ASCII art. To -indicate a literal block, preface the indented text block with -"``::``" (two colons). The literal block continues until the end of -the indentation. Indent the text block by 4 spaces. For example:: +Literal blocks are used for code samples and other preformatted text. +To indicate a literal block, preface the indented text block with +"``::``" (two colons), or use the ``.. code-block::`` directive. +Indent the text block by 4 spaces; the literal block continues until the end +of the indentation. For example:: This is a typical paragraph. A literal block follows. :: - for a in [5,4,3,2,1]: # this is program code, shown as-is - print a - print "it's..." - # a literal block continues until the indentation ends + for a in [5, 4, 3, 2, 1]: # this is program code, shown as-is + print(a) + print("it's...") -The paragraph containing only "``::``" will be completely removed from -the output; no empty paragraph will remain. "``::``" is also -recognized at the end of any paragraph. If immediately preceded by -whitespace, both colons will be removed from the output. When text -immediately precedes the "``::``", *one* colon will be removed from -the output, leaving only one colon visible (i.e., "``::``" will be -replaced by "``:``"). For example, one colon will remain visible -here:: +"``::``" is also recognized at the end of any paragraph; if not immediately +preceded by whitespace, one colon will remain visible in the final output:: - Paragraph:: + This is an example:: Literal block +By default, literal blocks will be syntax-highlighted as Python code. +For specific blocks that contain code or data in other languages/formats, +use the ``.. code-block:: language`` directive, substituting the "short name" +of the appropriate `Pygments lexer `_ +(or ``text`` to disable highlighting) for ``language``. For example:: + + .. code-block:: rst + + An example of the ``rst`` lexer (i.e. *reStructuredText*). + +For PEPs that predominantly contain literal blocks of a specific language, +use the ``.. highlight:: language`` directive with the appropriate ``language`` +at the top of the PEP body (below the headers and above the Abstract). +All literal blocks will then be treated as that language, +unless specified otherwise in the specific ``.. code-block``. For example:: + + .. highlight:: c + + Abstract + ======== + + Here's some C code:: + + printf("Hello, World!\n"); + Lists ----- @@ -437,24 +462,52 @@ Hyperlinks ---------- When referencing an external web page in the body of a PEP, you should -include the title of the page in the text, with either an inline -hyperlink reference to the URL or a footnote reference (see -`Footnotes`_ below). Do not include the URL in the body text of the -PEP. +include the title of the page or a suitable description in the text, with +either an inline hyperlink or a separate explicit target with the URL. +Do not include bare URLs in the body text of the PEP, and use HTTPS +links wherever available. Hyperlink references use backquotes and a trailing underscore to mark up the reference text; backquotes are optional if the reference text -is a single word. For example:: +is a single word. For example, to reference a hyperlink target named +``Python website``, you would write: - In this paragraph, we refer to the `Python web site`_. +.. code-block:: rst -An explicit target provides the URL. Put targets in a References -section at the end of the PEP, or immediately after the reference. + In this paragraph, we refer to the `Python website`_. + +If you intend to only reference a link once, and want to define it inline +with the text, insert the link into angle brackets (``<>``) after the text +you want to link, but before the closing backtick, with a space between the +text and the opening backtick. You should also use a double-underscore after +the closing backtick instead of a single one, which makes it an anonymous +reference to avoid conflicting with other target names. For example: + +.. code-block:: rst + + Visit the `website `__ for more. + +If you want to use one link multiple places with different linked text, +or want to ensure you don't have to update your link target names when +changing the linked text, include the target name within angle brackets +following the text to link, *with an underscore after the target name +but before the closing angle bracket* (or the link **will not work**). +For example: + +.. code-block:: rst + + For further examples, see the `documentation `_. + +An explicit target provides the URL. Put targets in the Footnotes section +at the end of the PEP, or immediately after the paragraph with the reference. Hyperlink targets begin with two periods and a space (the "explicit markup start"), followed by a leading underscore, the reference text, -a colon, and the URL (absolute or relative):: +a colon, and the URL. - .. _Python web site: http://www.python.org/ +.. code-block:: rst + + .. _Python web site: https://www.python.org/ + .. _pydocs: https://docs.python.org/ The reference text and the target text must match (although the match is case-insensitive and ignores differences in whitespace). Note that @@ -462,116 +515,122 @@ the underscore trails the reference text but precedes the target text. If you think of the underscore as a right-pointing arrow, it points *away* from the reference and *toward* the target. -The same mechanism can be used for internal references. Every unique -section title implicitly defines an internal hyperlink target. We can -make a link to the Abstract section like this:: + +Internal and PEP/RFC Links +-------------------------- + +The same mechanism as hyperlinks can be used for internal references. +Every unique section title implicitly defines an internal hyperlink target. +We can make a link to the Abstract section like this: + +.. code-block:: rst Here is a hyperlink reference to the `Abstract`_ section. The backquotes are optional since the reference text is a single word; we can also just write: Abstract_. -Footnotes containing the URLs from external targets will be generated -automatically at the end of the References section of the PEP, along -with footnote references linking the reference text to the footnotes. +To refer to PEPs or RFCs, always use the ``:pep:`` and ``:rfc:`` roles, +never hardcoded URLs. +For example: -Text of the form "PEP x" or "RFC x" (where "x" is a number) will be -linked automatically to the appropriate URLs. +.. code-block:: rst + + See :pep:`1` for more information on how to write a PEP, + and :pep:`the Hyperlink section of PEP 12 <12#hyperlinks>` for how to link. + +This renders as: + + See :pep:`1` for more information on how to write a PEP, + and :pep:`the Hyperlink section of PEP 12 <12#hyperlinks>` for how to link. + +PEP numbers in the text are never padded, and there is a space (not a dash) +between "PEP" or "RFC" and the number; the above roles will take care of +that for you. Footnotes --------- -Footnote references consist of a left square bracket, a number, a -right square bracket, and a trailing underscore:: +Footnote references consist of a left square bracket, a label, a +right square bracket, and a trailing underscore. +Instead of a number, use a label of the +form "#word", where "word" is a mnemonic consisting of alphanumerics +plus internal hyphens, underscores, and periods (no whitespace or +other characters are allowed). +For example: - This sentence ends with a footnote reference [1]_. +.. code-block:: rst + + Refer to The TeXbook [#TeXbook]_ for more information. + +which renders as + + Refer to The TeXbook [#TeXbook]_ for more information. Whitespace must precede the footnote reference. Leave a space between the footnote reference and the preceding word. -When referring to another PEP, include the PEP number in the body -text, such as "PEP 1". The title may optionally appear. Add a -footnote reference following the title. For example:: +Use footnotes for additional notes, explanations and caveats, as well as +for references to books and other sources not readily available online. +Native reST hyperlink targets or inline hyperlinks in the text should be +used in preference to footnotes for including URLs to online resources. - Refer to PEP 1 [2]_ for more information. - -Add a footnote that includes the PEP's title and author. It may -optionally include the explicit URL on a separate line, but only in -the References section. Footnotes begin with ".. " (the explicit +Footnotes begin with ".. " (the explicit markup start), followed by the footnote marker (no underscores), -followed by the footnote body. For example:: +followed by the footnote body. For example: - References - ========== +.. code-block:: rst - .. [2] PEP 1, "PEP Purpose and Guidelines", Warsaw, Hylton - (http://www.python.org/dev/peps/pep-0001) + .. [#TeXbook] Donald Knuth's *The TeXbook*, pages 195 and 196. -If you decide to provide an explicit URL for a PEP, please use this as -the URL template:: +which renders as - http://www.python.org/dev/peps/pep-xxxx - -PEP numbers in URLs must be padded with zeros from the left, so as to -be exactly 4 characters wide; however, PEP numbers in the text are -never padded. - -During the course of developing your PEP, you may have to add, remove, -and rearrange footnote references, possibly resulting in mismatched -references, obsolete footnotes, and confusion. Auto-numbered -footnotes allow more freedom. Instead of a number, use a label of the -form "#word", where "word" is a mnemonic consisting of alphanumerics -plus internal hyphens, underscores, and periods (no whitespace or -other characters are allowed). For example:: - - Refer to PEP 1 [#PEP-1]_ for more information. - - References - ========== - - .. [#PEP-1] PEP 1, "PEP Purpose and Guidelines", Warsaw, Hylton - - http://www.python.org/dev/peps/pep-0001 + .. [#TeXbook] Donald Knuth's *The TeXbook*, pages 195 and 196. Footnotes and footnote references will be numbered automatically, and -the numbers will always match. Once a PEP is finalized, auto-numbered -labels should be replaced by numbers for simplicity. +the numbers will always match. Images ------ -If your PEP contains a diagram, you may include it in the processed -output using the "image" directive:: +If your PEP contains a diagram or other graphic, you may include it in the +processed output using the ``image`` directive: + +.. code-block:: rst .. image:: diagram.png -Any browser-friendly graphics format is possible: .png, .jpeg, .gif, -.tiff, etc. +Any browser-friendly graphics format is possible; PNG should be +preferred for graphics, JPEG for photos and GIF for animations. +Currently, SVG must be avoided due to compatibility issues with the +PEP build system. -Since this image will not be visible to readers of the PEP in source -text form, you should consider including a description or ASCII art -alternative, using a comment (below). +For accessibility and readers of the source text, you should include +a description of the image and any key information contained within +using the ``:alt:`` option to the ``image`` directive: + +.. code-block:: rst + + .. image:: dataflow.png + :alt: Data flows from the input module, through the "black box" + module, and finally into (and through) the output module. Comments -------- -A comment block is an indented block of arbitrary text immediately +A comment is an indented block of arbitrary text immediately following an explicit markup start: two periods and whitespace. Leave the ".." on a line by itself to ensure that the comment is not misinterpreted as another explicit markup construct. Comments are not -visible in the processed document. For the benefit of those reading -your PEP in source form, please consider including a descriptions of -or ASCII art alternatives to any images you include. For example:: +visible in the processed document. For example: - .. image:: dataflow.png +.. code-block:: rst .. - Data flows from the input module, through the "black box" - module, and finally into (and through) the output module. - -The Emacs stanza at the bottom of this document is inside a comment. + This section should be updated in the final PEP. + Ensure the date is accurate. Escaping Mechanism @@ -590,11 +649,73 @@ If you find that you need to use a backslash in your text, consider using inline literals or a literal block instead. +Canonical Documentation and Intersphinx +--------------------------------------- + +As :pep:`PEP 1 describes <1#pep-maintenance>`, +PEPs are considered historical documents once marked Final, +and their canonical documentation/specification should be moved elsewhere. +To indicate this, use the ``canonical-docs`` directive +or an appropriate subclass +(currently ``canonical-pypa-spec`` for packaging standards). + +Furthermore, you can use +`Intersphinx references +`_ +to other Sphinx sites, +currently the `Python documentation `_ +and `packaging.python.org `_, +to easily cross-reference pages, sections and Python/C objects. +This works with both the "canonical" directives and anywhere in your PEP. + +Add the directive between the headers and the first section of the PEP +(typically the Abstract) +and pass as an argument an Intersphinx reference of the canonical doc/spec +(or if the target is not on a Sphinx site, a `reST hyperlink `__). + +For example, +to create a banner pointing to the :mod:`python:sqlite3` docs, +you would write the following:: + + .. canonical-doc:: :mod:`python:sqlite3` + +which would generate the banner: + + .. canonical-doc:: :mod:`python:sqlite3` + +Or for a PyPA spec, +such as the :ref:`packaging:core-metadata`, +you would use:: + + .. canonical-pypa-spec:: :ref:`packaging:core-metadata` + +which renders as: + + .. canonical-pypa-spec:: :ref:`packaging:core-metadata` + +The argument accepts arbitrary reST, +so you can include multiple linked docs/specs and name them whatever you like, +and you can also include directive content that will be inserted into the text. +The following advanced example:: + + .. canonical-doc:: the :ref:`python:sqlite3-connection-objects` and :exc:`python:~sqlite3.DataError` docs + + Also, see the :ref:`Data Persistence docs ` for other examples. + +would render as: + + .. canonical-doc:: the :ref:`python:sqlite3-connection-objects` and :exc:`python:sqlite3.DataError` docs + + Also, see the :ref:`Data Persistence docs ` for other examples. + + Habits to Avoid =============== Many programmers who are familiar with TeX often write quotation marks -like this:: +like this: + +.. code-block:: text `single-quoted' or ``double-quoted'' @@ -610,133 +731,43 @@ Suggested Sections ================== Various sections are found to be common across PEPs and are outlined in -PEP 1 [1]_. Those sections are provided here for convenience. +:pep:`1`. Those sections are provided here for convenience. .. _template: -:: - - Abstract - ======== - - [A short (~200 word) description of the technical issue being addressed.] - - - Motivation - ========== - - [Clearly explain why the existing language specification is inadequate to address the problem that the PEP solves.] - - - Rationale - ========= - - [Describe why particular design decisions were made.] - - - Specification - ============= - - [Describe the syntax and semantics of any new language feature.] - - - Backwards Compatibility - ======================= - - [Describe potential impact and severity on pre-existing code.] - - - Security Implications - ===================== - - [How could a malicious user take advantage of this new feature?] - - - How to Teach This - ================= - - [How to teach users, new and experienced, how to apply the PEP to their work.] - - - Reference Implementation - ======================== - - [Link to any existing implementation and details about its state, e.g. proof-of-concept.] - - - Rejected Ideas - ============== - - [Why certain ideas that were brought while discussing this PEP were not ultimately pursued.] - - - Open Issues - =========== - - [Any points that are still being decided/discussed.] - - - References - ========== - - [A collection of URLs used as references through the PEP.] - - - Copyright - ========= - - This document is placed in the public domain or under the - CC0-1.0-Universal license, whichever is more permissive. - - - - .. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: +.. include:: pep-0012/pep-NNNN.rst + :code: rst Resources ========= -Many other constructs and variations are possible. For more details -about the reStructuredText markup, in increasing order of -thoroughness, please see: +Many other constructs and variations are possible, +both those supported by basic `Docutils `_ +and the extensions added by `Sphinx `_. -* `A ReStructuredText Primer`__, a gentle introduction. +A number of resources are available to learn more about them: - __ http://docutils.sourceforge.net/docs/rst/quickstart.html +* `Sphinx ReStructuredText Primer + `_, + a gentle but fairly detailed introduction. -* `Quick reStructuredText`__, a users' quick reference. +* `reStructuredText Markup Specification + `_, + the authoritative, comprehensive documentation of the basic reST syntax, + directives, roles and more. - __ http://docutils.sourceforge.net/docs/rst/quickref.html +* `Sphinx Roles + `_ + and `Sphinx Directives + `_, + the extended constructs added by the Sphinx documentation system used to + render the PEPs to HTML. -* `reStructuredText Markup Specification`__, the final authority. - - __ http://docutils.sourceforge.net/spec/rst/reStructuredText.html - -The processing of reStructuredText PEPs is done using Docutils_. If -you have a question or require assistance with reStructuredText or -Docutils, please `post a message`_ to the `Docutils-users mailing -list`_. The `Docutils project web site`_ has more information. - -.. _Docutils: -.. _Docutils project web site: http://docutils.sourceforge.net/ -.. _post a message: - mailto:docutils-users@lists.sourceforge.net?subject=PEPs -.. _Docutils-users mailing list: - http://docutils.sf.net/docs/user/mailing-lists.html#docutils-users - - -References -========== - -.. [1] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - (http://www.python.org/dev/peps/pep-0001) +If you have questions or require assistance with writing a PEP that the above +resources don't address, ping ``@python/pep-editors`` on GitHub, open an +`issue on the PEPs repository `_ +or reach out to a PEP editor directly. Copyright @@ -744,14 +775,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/peps/pep-0012/pep-NNNN.rst b/peps/pep-0012/pep-NNNN.rst new file mode 100644 index 000000000..011616f6f --- /dev/null +++ b/peps/pep-0012/pep-NNNN.rst @@ -0,0 +1,90 @@ +PEP: +Title: +Author: +Sponsor: +PEP-Delegate: +Discussions-To: +Status: +Type: +Topic: +Content-Type: text/x-rst +Requires: +Created: +Python-Version: +Post-History: +Replaces: +Superseded-By: +Resolution: + + +Abstract +======== + +[A short (~200 word) description of the technical issue being addressed.] + + +Motivation +========== + +[Clearly explain why the existing language specification is inadequate to address the problem that the PEP solves.] + + +Rationale +========= + +[Describe why particular design decisions were made.] + + +Specification +============= + +[Describe the syntax and semantics of any new language feature.] + + +Backwards Compatibility +======================= + +[Describe potential impact and severity on pre-existing code.] + + +Security Implications +===================== + +[How could a malicious user take advantage of this new feature?] + + +How to Teach This +================= + +[How to teach users, new and experienced, how to apply the PEP to their work.] + + +Reference Implementation +======================== + +[Link to any existing implementation and details about its state, e.g. proof-of-concept.] + + +Rejected Ideas +============== + +[Why certain ideas that were brought while discussing this PEP were not ultimately pursued.] + + +Open Issues +=========== + +[Any points that are still being decided/discussed.] + + +Footnotes +========= + +[A collection of footnotes cited in the PEP, and a place to list non-inline hyperlink targets.] + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0013.rst b/peps/pep-0013.rst similarity index 93% rename from pep-0013.rst rename to peps/pep-0013.rst index 03a030854..36e1de51a 100644 --- a/pep-0013.rst +++ b/peps/pep-0013.rst @@ -2,7 +2,8 @@ PEP: 13 Title: Python Language Governance Author: The Python core team and community Status: Active -Type: Informational +Type: Process +Topic: Governance Content-Type: text/x-rst Created: 16-Dec-2018 @@ -21,13 +22,13 @@ Current steering council The current steering council consists of: -* Barry Warsaw * Brett Cannon -* Carol Willing -* Thomas Wouters +* Emily Morehouse +* Gregory P. Smith * Pablo Galindo Salgado +* Thomas Wouters -Per the results of the vote tracked in PEP 8102. +Per the results of the vote tracked in :pep:`8104`. The core team consists of those listed in the private https://github.com/python/voters/ repository which is publicly @@ -266,7 +267,7 @@ Core team membership acknowledges sustained and valuable efforts that align well with the philosophy and the goals of the Python project. It is granted by receiving at least two-thirds positive votes in a -core team vote that is open for one week and with no veto by the +core team vote that is open for one week and is not vetoed by the steering council. Core team members are always looking for promising contributors, @@ -287,7 +288,7 @@ active privileges like voting or nominating for the steering council, and commit access. The initial active core team members will consist of everyone -currently listed in the `"Python core" team on Github +currently listed in the `"Python core" team on GitHub `__ (access granted for core members only), and the initial inactive members will consist of everyone else who has been a @@ -314,14 +315,14 @@ when he `stepped down After discussion, a number of proposals were put forward for a new governance model, and the core devs voted to choose between them. The -overall process is described in PEP 8000 and PEP 8001, a review of -other projects was performed in PEP 8002, and the proposals themselves +overall process is described in :pep:`8000` and :pep:`8001`, a review of +other projects was performed in :pep:`8002`, and the proposals themselves were written up as the 801x series of PEPs. Eventually the proposal in -PEP 8016 was `selected +:pep:`8016` was `selected `__ as the new governance model, and was used to create the initial version of this PEP. The 8000-series PEPs are preserved for historical -reference (and in particular, PEP 8016 contains additional rationale +reference (and in particular, :pep:`8016` contains additional rationale and links to contemporary discussions), but this PEP is now the official reference, and will evolve following the rules described herein. @@ -330,10 +331,11 @@ herein. History of council elections ---------------------------- -* January 2019: PEP 8100 -* December 2019: PEP 8101 -* December 2020: PEP 8102 -* December 2021: PEP 8103 +* January 2019: :pep:`8100` +* December 2019: :pep:`8101` +* December 2020: :pep:`8102` +* December 2021: :pep:`8103` +* December 2022: :pep:`8104` History of amendments @@ -346,7 +348,7 @@ History of amendments Acknowledgements ================ -This PEP began as PEP 8016, which was written by Nathaniel J. Smith +This PEP began as :pep:`8016`, which was written by Nathaniel J. Smith and Donald Stufft, based on a Django governance document written by Aymeric Augustin, and incorporated feedback and assistance from numerous others. @@ -356,14 +358,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0020.txt b/peps/pep-0020.rst similarity index 77% rename from pep-0020.txt rename to peps/pep-0020.rst index ffdbaf8bf..f81a44f10 100644 --- a/pep-0020.txt +++ b/peps/pep-0020.rst @@ -1,8 +1,6 @@ PEP: 20 Title: The Zen of Python -Version: $Revision$ -Last-Modified: $Date$ -Author: tim.peters@gmail.com (Tim Peters) +Author: Tim Peters Status: Active Type: Informational Content-Type: text/x-rst @@ -10,7 +8,6 @@ Created: 19-Aug-2004 Post-History: 22-Aug-2004 - Abstract ======== @@ -22,7 +19,7 @@ have been written down. The Zen of Python ================= -:: +.. code-block:: text Beautiful is better than ugly. Explicit is better than implicit. @@ -48,7 +45,7 @@ The Zen of Python Easter Egg ========== -:: +.. code-block:: pycon >>> import this @@ -56,21 +53,12 @@ Easter Egg References ========== -* Originally posted to comp.lang.python/python-list@python.org under a - thread called "The Way of Python" - https://groups.google.com/d/msg/comp.lang.python/B_VxeTBClM0/L8W9KlsiriUJ +Originally posted to comp.lang.python/python-list@python.org under a +thread called `"The Way of Python" +`__ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0042.txt b/peps/pep-0042.rst similarity index 97% rename from pep-0042.txt rename to peps/pep-0042.rst index 08ba663f9..23beb2b93 100644 --- a/pep-0042.txt +++ b/peps/pep-0042.rst @@ -3,15 +3,14 @@ Title: Feature Requests Version: $Revision$ Last-Modified: $Date$ Author: Jeremy Hylton -Status: Rejected +Status: Withdrawn Type: Process Content-Type: text/x-rst Created: 12-Sep-2000 Post-History: -Resolution: https://github.com/python/peps/pull/108#issuecomment-249603204 -.. note:: This PEP has been rejected as obsolete. +.. note:: This PEP has been `withdrawn as obsolete`_. All new feature requests should either go to the `Python bug tracker`_ for very simple requests or the `python-ideas`_ mailing list for everything else. The rest of this document is retained for historical @@ -25,7 +24,7 @@ This PEP contains a list of feature requests that may be considered for future versions of Python. Large feature requests should not be included here, but should be described in separate PEPs; however a large feature request that doesn't have its own PEP can be listed here -until its own PEP is created. See PEP 0 for details. +until its own PEP is created. See :pep:`0` for details. This PEP was created to allow us to close bug reports that are really feature requests. Marked as Open, they distract from the list of real @@ -150,7 +149,7 @@ Standard Library https://bugs.python.org/issue210849 -* urlparse should be updated to comply with RFC 2396, which defines +* urlparse should be updated to comply with :rfc:`2396`, which defines optional parameters for each segment of the path. https://bugs.python.org/issue210834 @@ -335,6 +334,8 @@ Building and Installing .. _`Python bug tracker`: https://bugs.python.org .. _`python-ideas`: https://mail.python.org/mailman/listinfo/python-ideas +.. _`withdrawn as obsolete`: https://github.com/python/peps/pull/108#issuecomment-249603204 + .. Local Variables: diff --git a/pep-0100.txt b/peps/pep-0100.rst similarity index 99% rename from pep-0100.txt rename to peps/pep-0100.rst index f93b78292..77b74e8e3 100644 --- a/pep-0100.txt +++ b/peps/pep-0100.rst @@ -1,8 +1,6 @@ PEP: 100 Title: Python Unicode Integration -Version: $Revision$ -Last-Modified: $Date$ -Author: mal@lemburg.com (Marc-AndrĂ© Lemburg) +Author: Marc-AndrĂ© Lemburg Status: Final Type: Standards Track Content-Type: text/x-rst @@ -1083,11 +1081,9 @@ References * UCS-2: http://www.uazone.org/multiling/unicode/ucs2.html - * UTF-7: Defined in RFC2152, e.g. - http://www.uazone.org/multiling/ml-docs/rfc2152.txt + * UTF-7: Defined in :rfc:`2152` - * UTF-8: Defined in RFC2279, e.g. - https://tools.ietf.org/html/rfc2279 + * UTF-8: Defined in :rfc:`2279` * UTF-16: http://www.uazone.org/multiling/unicode/wg2n1035.html @@ -1248,10 +1244,3 @@ file.] --- * first version - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0101.txt b/peps/pep-0101.rst similarity index 91% rename from pep-0101.txt rename to peps/pep-0101.rst index 7b8528e10..e877f8958 100644 --- a/pep-0101.txt +++ b/peps/pep-0101.rst @@ -78,6 +78,11 @@ Here's a hopefully-complete list. * A subscription to the super secret release manager mailing list, which may or may not be called ``python-cabal``. Bug Barry about this. +* A ``@python.org`` email address that you will use to sign your releases + with. Ask ``postmaster@`` for an address; you can either get a full + account, or a redirecting alias + SMTP credentials to send email from + this address that looks legit to major email providers. + Types of Releases ================= @@ -121,9 +126,10 @@ release. The roles and their current experts are: * RM = Release Manager - - Ɓukasz Langa (Central Europe) - - Ned Deily (US) + - Thomas Wouters (NL) - Pablo Galindo Salgado (UK) + - Ɓukasz Langa (PL) + - Ned Deily (US) * WE = Windows - Steve Dower * ME = Mac - Ned Deily (US) @@ -206,7 +212,7 @@ to perform some manual editing steps. within it (called the "release clone" from now on). You can use the same GitHub fork you use for cpython development. Using the standard setup recommended in the Python Developer's Guide, your fork would be referred - to as `origin` and the standard cpython repo as `upstream`. You will + to as ``origin`` and the standard cpython repo as ``upstream``. You will use the branch on your fork to do the release engineering work, including tagging the release, and you will use it to share with the other experts for making the binaries. @@ -296,7 +302,7 @@ to perform some manual editing steps. $ .../release-tools/release.py --tag X.Y.ZaN - This executes a `git tag` command with the `-s` option so that the + This executes a ``git tag`` command with the ``-s`` option so that the release tag in the repo is signed with your gpg key. When prompted choose the private key you use for signing release tarballs etc. @@ -321,6 +327,10 @@ to perform some manual editing steps. tarballs and signatures in a subdirectory called ``X.Y.ZaN/src``, and the built docs in ``X.Y.ZaN/docs`` (for **final** releases). + Note that the script will sign your release with Sigstore. Please use + your **@python.org** email address for this. See here for more information: + https://www.python.org/download/sigstore/. + - Now you want to perform the very important step of checking the tarball you just created, to make sure a completely clean, virgin build passes the regression test. Here are the best @@ -528,7 +538,7 @@ the main repo. do some post-merge cleanup. Check the top-level ``README.rst`` and ``include/patchlevel.h`` files to ensure they now reflect the desired post-release values for on-going development. - The patchlevel should be the release tag with a `+`. + The patchlevel should be the release tag with a ``+``. Also, if you cherry-picked changes from the standard release branch into the release engineering branch for this release, you will now need to manual remove each blurb entry from @@ -536,8 +546,8 @@ the main repo. into the release you are working on since that blurb entry is now captured in the merged x.y.z.rst file for the new release. Otherwise, the blurb entry will appear twice in - the `changelog.html` file, once under `Python next` and again - under `x.y.z`. + the ``changelog.html`` file, once under ``Python next`` and again + under ``x.y.z``. - Review and commit these changes:: @@ -548,52 +558,52 @@ the main repo. following feature release. When finished, the ``main`` branch will now build Python ``X.Y+1``. - - First, set main up to be the next release, i.e.X.Y+1.a0:: + - First, set main up to be the next release, i.e.X.Y+1.a0:: - $ git checkout main - $ .../release-tools/release.py --bump 3.9.0a0 + $ git checkout main + $ .../release-tools/release.py --bump 3.9.0a0 - - Edit all version references in README.rst + - Edit all version references in README.rst - - Move any historical "what's new" entries from ``Misc/NEWS`` to - ``Misc/HISTORY``. + - Move any historical "what's new" entries from ``Misc/NEWS`` to + ``Misc/HISTORY``. - - Edit ``Doc/tutorial/interpreter.rst`` (2 references to '[Pp]ython3x', - one to 'Python 3.x', also make the date in the banner consistent). + - Edit ``Doc/tutorial/interpreter.rst`` (2 references to '[Pp]ython3x', + one to 'Python 3.x', also make the date in the banner consistent). - - Edit ``Doc/tutorial/stdlib.rst`` and ``Doc/tutorial/stdlib2.rst``, which - have each one reference to '[Pp]ython3x'. + - Edit ``Doc/tutorial/stdlib.rst`` and ``Doc/tutorial/stdlib2.rst``, which + have each one reference to '[Pp]ython3x'. - - Add a new ``whatsnew/3.x.rst`` file (with the comment near the top - and the toplevel sections copied from the previous file) and - add it to the toctree in ``whatsnew/index.rst``. But beware that - the initial ``whatsnew/3.x.rst`` checkin from previous releases - may be incorrect due to the initial midstream change to ``blurb`` - that propagates from release to release! Help break the cycle: if - necessary make the following change:: + - Add a new ``whatsnew/3.x.rst`` file (with the comment near the top + and the toplevel sections copied from the previous file) and + add it to the toctree in ``whatsnew/index.rst``. But beware that + the initial ``whatsnew/3.x.rst`` checkin from previous releases + may be incorrect due to the initial midstream change to ``blurb`` + that propagates from release to release! Help break the cycle: if + necessary make the following change:: - - For full details, see the :source:`Misc/NEWS` file. - + For full details, see the :ref:`changelog `. + - For full details, see the :source:`Misc/NEWS` file. + + For full details, see the :ref:`changelog `. - - Update the version number in ``configure.ac`` and re-run ``autoconf``. + - Update the version number in ``configure.ac`` and re-run ``autoconf``. - - Make sure the ``SOURCE_URI`` in ``Doc/tools/extensions/pyspecific.py`` - points to ``main``. + - Make sure the ``SOURCE_URI`` in ``Doc/tools/extensions/pyspecific.py`` + points to ``main``. - - Update the version numbers for the Windows builds in PC/ and - PCbuild/, which have references to python38. - NOTE, check with Steve Dower about this step, it is probably obsolete.:: + - Update the version numbers for the Windows builds in PC/ and + PCbuild/, which have references to python38. + NOTE, check with Steve Dower about this step, it is probably obsolete.:: - $ find PC/ PCbuild/ -type f | xargs sed -i 's/python38/python39/g' - $ git mv -f PC/os2emx/python38.def PC/os2emx/python39.def - $ git mv -f PC/python38stub.def PC/python39stub.def - $ git mv -f PC/python38gen.py PC/python39gen.py + $ find PC/ PCbuild/ -type f | xargs sed -i 's/python38/python39/g' + $ git mv -f PC/os2emx/python38.def PC/os2emx/python39.def + $ git mv -f PC/python38stub.def PC/python39stub.def + $ git mv -f PC/python38gen.py PC/python39gen.py - - Commit these changes to the main branch:: + - Commit these changes to the main branch:: - $ git status - $ git add ... - $ git commit -m 'Bump to 3.9.0a0' + $ git status + $ git add ... + $ git commit -m 'Bump to 3.9.0a0' - Do another ``git status`` in this directory. @@ -689,6 +699,11 @@ with RevSys.) (It's best to update add-to-pydotorg.py when file types are removed, too.) + The script will also sign any remaining files that were not + signed with Sigstore until this point. Again, if this happens, + do use your @python.org address for this process. More info: + https://www.python.org/download/sigstore/ + - In case the CDN already cached a version of the Downloads page without the files present, you can invalidate the cache using:: @@ -697,19 +712,19 @@ with RevSys.) - If this is a **final** release: - Add the new version to the *Python Documentation by Version* - page `https://www.python.org/doc/versions/` and + page ``https://www.python.org/doc/versions/`` and remove the current version from any 'in development' section. - For X.Y.Z, edit all the previous X.Y releases' page(s) to point to the new release. This includes the content field of the - `Downloads -> Releases` entry for the release:: + ``Downloads -> Releases`` entry for the release:: Note: Python x.y.m has been superseded by `Python x.y.n `_. And, for those releases having separate release page entries (phasing these out?), update those pages as well, - e.g. `download/releases/x.y.z`:: + e.g. ``download/releases/x.y.z``:: Note: Python x.y.m has been superseded by `Python x.y.n `_. @@ -893,8 +908,8 @@ else does them. Some of those tasks include: - Remove the release from the list of "Active Python Releases" on the Downloads page. To do this, log in to the admin page for python.org, navigate to Boxes, - and edit the `downloads-active-releases` entry. Simply strip out the relevant - paragraph of HTML for your release. (You'll probably have to do the `curl -X PURGE` + and edit the ``downloads-active-releases`` entry. Simply strip out the relevant + paragraph of HTML for your release. (You'll probably have to do the ``curl -X PURGE`` trick to purge the cache if you want to confirm you made the change correctly.) - Add retired notice to each release page on python.org for the retired branch. @@ -931,7 +946,7 @@ else does them. Some of those tasks include: * mailing lists (python-committers, python-dev, python-list, python-announcements) - * discuss.python.org and Zulip + * discuss.python.org * Python Dev blog diff --git a/pep-0102.txt b/peps/pep-0102.rst similarity index 96% rename from pep-0102.txt rename to peps/pep-0102.rst index 5f7cff22a..458e16274 100644 --- a/pep-0102.txt +++ b/peps/pep-0102.rst @@ -1,14 +1,12 @@ PEP: 102 Title: Doing Python Micro Releases -Version: $Revision$ -Last-Modified: $Date$ -Author: anthony@interlink.com.au (Anthony Baxter), - barry@python.org (Barry Warsaw), - guido@python.org (Guido van Rossum) +Author: Anthony Baxter , + Barry Warsaw , + Guido van Rossum Status: Superseded Type: Informational Content-Type: text/x-rst -Created: 22-Aug-2001 (edited down on 9-Jan-2002 to become PEP 102) +Created: 09-Jan-2002 Post-History: Superseded-By: 101 @@ -17,10 +15,10 @@ Replacement Note ================ Although the size of the to-do list in this PEP is much less scary -than that in PEP 101, it turns out not to be enough justification +than that in :pep:`101`, it turns out not to be enough justification for the duplication of information, and with it, the danger of one of the copies to become out of date. Therefore, this PEP is not -maintained anymore, and micro releases are fully covered by PEP 101. +maintained anymore, and micro releases are fully covered by :pep:`101`. Abstract @@ -33,8 +31,8 @@ Guido himself. But several recent releases have been performed by other folks, so this PEP attempts to collect, in one place, all the steps needed to make a Python bugfix release. -The major Python release process is covered in PEP 101 - this PEP -is just PEP 101, trimmed down to only include the bits that are +The major Python release process is covered in :pep:`101` - this PEP +is just :pep:`101`, trimmed down to only include the bits that are relevant for micro releases, a.k.a. patch, or bug fix releases. It is organized as a recipe and you can actually print this out and @@ -160,7 +158,7 @@ from the release21-maint branch. 11. Once the release process has started, the documentation needs to be built and posted on python.org according to the instructions - in PEP 101. + in :pep:`101`. Note that Fred is responsible both for merging doc changes from the trunk to the branch AND for merging any branch changes from @@ -512,10 +510,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0103.txt b/peps/pep-0103.rst similarity index 99% rename from pep-0103.txt rename to peps/pep-0103.rst index ff3586e47..c92469adc 100644 --- a/pep-0103.txt +++ b/peps/pep-0103.rst @@ -137,7 +137,7 @@ you have done something like that:: $ git branch v1 origin/v1 The first command clones remote repository into local directory -`python``, creates a new local branch master, sets +``python``, creates a new local branch master, sets remotes/origin/master as its upstream remote-tracking branch and checks it out into the working directory. diff --git a/pep-0160.txt b/peps/pep-0160.rst similarity index 99% rename from pep-0160.txt rename to peps/pep-0160.rst index ca43b29c7..7e88df993 100644 --- a/pep-0160.txt +++ b/peps/pep-0160.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Fred L. Drake, Jr. Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 25-Jul-2000 Python-Version: 1.6 diff --git a/pep-0200.txt b/peps/pep-0200.rst similarity index 98% rename from pep-0200.txt rename to peps/pep-0200.rst index eb6297d46..e1b156a64 100644 --- a/pep-0200.txt +++ b/peps/pep-0200.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Jeremy Hylton Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 12-Jul-2000 Python-Version: 2.0 @@ -298,7 +299,7 @@ Open items -- completed/fixed Accepted and completed ====================== -* Change meaning of \x escapes - PEP 223 - Fredrik Lundh +* Change meaning of \x escapes - :pep:`223` - Fredrik Lundh * Add \U1234678 escapes in u"" strings - Fredrik Lundh @@ -330,8 +331,8 @@ Accepted and completed doing. It also makes the xrange objects obvious when working in the interactive interpreter. -* Extended print statement - Barry Warsaw PEP 214 - http://www.python.org/dev/peps/pep-0214/ SF Patch #100970 +* Extended print statement - Barry Warsaw :pep:`214` + SF Patch #100970 http://sourceforge.net/patch/?func=detailpatch&patch_id=100970&group_id=5470 * interface to poll system call - Andrew Kuchling SF Patch 100852 diff --git a/pep-0201.txt b/peps/pep-0201.rst similarity index 97% rename from pep-0201.txt rename to peps/pep-0201.rst index 9675bc662..7b1c77af3 100644 --- a/pep-0201.txt +++ b/peps/pep-0201.rst @@ -1,8 +1,6 @@ PEP: 201 Title: Lockstep Iteration -Version: $Revision$ -Last-Modified: $Date$ -Author: barry@python.org (Barry Warsaw) +Author: Barry Warsaw Status: Final Type: Standards Track Content-Type: text/x-rst @@ -48,8 +46,8 @@ Lockstep For-Loops Lockstep for-loops are non-nested iterations over two or more sequences, such that at each pass through the loop, one element from each sequence is taken to compose the target. This behavior can -already be accomplished in Python through the use of the map() built- -in function:: +already be accomplished in Python through the use of the map() built-in +function:: >>> a = (1, 2, 3) >>> b = (4, 5, 6) @@ -95,7 +93,7 @@ Neither of these forms would work, since they both already mean something in Python and changing the meanings would break existing code. All other suggestions for new syntax suffered the same problem, or were in conflict with other another proposed feature called 'list -comprehensions' (see PEP 202). +comprehensions' (see :pep:`202`). The Proposed Solution ===================== @@ -289,11 +287,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0202.txt b/peps/pep-0202.rst similarity index 93% rename from pep-0202.txt rename to peps/pep-0202.rst index 11704eca6..25c5bc12e 100644 --- a/pep-0202.txt +++ b/peps/pep-0202.rst @@ -1,8 +1,6 @@ PEP: 202 Title: List Comprehensions -Version: $Revision$ -Last-Modified: $Date$ -Author: barry@python.org (Barry Warsaw) +Author: Barry Warsaw Status: Final Type: Standards Track Content-Type: text/x-rst @@ -84,11 +82,3 @@ References ========== .. [1] http://docs.python.org/reference/expressions.html#list-displays - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0203.txt b/peps/pep-0203.rst similarity index 97% rename from pep-0203.txt rename to peps/pep-0203.rst index 032ef0cc4..c110ca2e0 100644 --- a/pep-0203.txt +++ b/peps/pep-0203.rst @@ -1,8 +1,6 @@ PEP: 203 Title: Augmented Assignments -Version: $Revision$ -Last-Modified: $Date$ -Author: thomas@python.org (Thomas Wouters) +Author: Thomas Wouters Status: Final Type: Standards Track Content-Type: text/x-rst @@ -171,8 +169,8 @@ To work around this problem, the packages currently have to use methods or functions to modify an object in-place, which is definitely less readable than an augmented assignment expression. Augmented assignment won't solve all the problems for these packages, since some operations cannot be expressed in the -limited set of binary operators to start with, but it is a start. A -different PEP [3]_ is looking at adding new operators. +limited set of binary operators to start with, but it is a start. :pep:`211` +is looking at adding new operators. New methods @@ -323,13 +321,3 @@ References .. [1] http://www.python.org/pipermail/python-list/2000-June/059556.html .. [2] http://sourceforge.net/patch?func=detailpatch&patch_id=100699&group_id=5470 - -.. [3] PEP 211, Adding A New Outer Product Operator, Wilson - http://www.python.org/dev/peps/pep-0211/ - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0204.txt b/peps/pep-0204.rst similarity index 97% rename from pep-0204.txt rename to peps/pep-0204.rst index 1b3da1699..cd7a7efcf 100644 --- a/pep-0204.txt +++ b/peps/pep-0204.rst @@ -1,8 +1,6 @@ PEP: 204 Title: Range Literals -Version: $Revision$ -Last-Modified: $Date$ -Author: thomas@python.org (Thomas Wouters) +Author: Thomas Wouters Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -230,7 +228,7 @@ Open issues [5, 6, 1, 2, 3, 4, 5, 7, 9] - How should range literals interact with another proposed new - feature, "list comprehensions" [2]_? Specifically, should it be + feature, :pep:`"list comprehensions" <202>`? Specifically, should it be possible to create lists in list comprehensions? E.g.:: >>> [x:y for x in (1, 2) y in (3, 4)] @@ -241,7 +239,7 @@ Open issues Or a list of lists, like so:: - [[1, 2], [1, 2, 3], [2]_, [2, 3]] + [[1, 2], [1, 2, 3], [2], [2, 3]] However, as the syntax and semantics of list comprehensions are still subject of hot debate, these issues are probably best @@ -297,12 +295,3 @@ References ========== .. [1] http://sourceforge.net/patch/?func=detailpatch&patch_id=100902&group_id=5470 -.. [2] PEP 202, List Comprehensions - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0205.txt b/peps/pep-0205.rst similarity index 99% rename from pep-0205.txt rename to peps/pep-0205.rst index 39aebeac6..e16c2d3dc 100644 --- a/pep-0205.txt +++ b/peps/pep-0205.rst @@ -185,8 +185,8 @@ Implementation Strategy ======================= The implementation of weak references will include a list of -reference containers that must be cleared for each weakly- -referencable object. If the reference is from a weak dictionary, +reference containers that must be cleared for each weakly-referencable +object. If the reference is from a weak dictionary, the dictionary entry is cleared first. Then, any associated callback is called with the object passed as a parameter. Once all callbacks have been called, the object is finalized and @@ -335,7 +335,7 @@ handle on an object that does not increment the object's reference count. This means that holding a vref on an object will not keep the object from being destroyed. This would allow the Python programmer, for example, to create the aforementioned tree -structure tree structure, which is automatically destroyed when it +structure, which is automatically destroyed when it is no longer in use -- by making all of the parent back-references into vrefs, they no longer create reference cycles which keep the tree from being destroyed. diff --git a/pep-0206.txt b/peps/pep-0206.rst similarity index 100% rename from pep-0206.txt rename to peps/pep-0206.rst diff --git a/pep-0207.txt b/peps/pep-0207.rst similarity index 98% rename from pep-0207.txt rename to peps/pep-0207.rst index ce80b420a..39a49ae6a 100644 --- a/pep-0207.txt +++ b/peps/pep-0207.rst @@ -1,8 +1,6 @@ PEP: 207 Title: Rich Comparisons -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum), DavidA@ActiveState.com (David Ascher) +Author: Guido van Rossum , David Ascher Status: Final Type: Standards Track Content-Type: text/x-rst @@ -475,9 +473,3 @@ testing "if a>b:" is massively ambiguous. The inlining already present which deals with integer comparisons would still apply, resulting in no performance cost for the most common cases. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0208.txt b/peps/pep-0208.rst similarity index 97% rename from pep-0208.txt rename to peps/pep-0208.rst index b89e11cc6..a70b3c935 100644 --- a/pep-0208.txt +++ b/peps/pep-0208.rst @@ -1,8 +1,6 @@ PEP: 208 Title: Reworking the Coercion Model -Version: $Revision$ -Last-Modified: $Date$ -Author: nas@arctrix.com (Neil Schemenauer), mal@lemburg.com (Marc-AndrĂ© Lemburg) +Author: Neil Schemenauer , Marc-AndrĂ© Lemburg Status: Final Type: Standards Track Content-Type: text/x-rst @@ -185,7 +183,7 @@ stating the result. Currently, this result integer may only be -1, 0, 1. If the slot cannot handle the type combination, it may return a reference to ``Py_NotImplemented``. [XXX Note that this slot is still in flux since it should take into account rich comparisons -(i.e. PEP 207).] +(i.e. :pep:`207`).] Numeric comparisons are handled by a new numeric protocol API:: @@ -232,15 +230,3 @@ References .. [1] http://www.lemburg.com/files/python/mxDateTime.html .. [2] http://sourceforge.net/patch/?func=detailpatch&patch_id=102652&group_id=5470 .. [3] http://www.lemburg.com/files/python/CoercionProposal.html - - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0209.txt b/peps/pep-0209.rst similarity index 96% rename from pep-0209.txt rename to peps/pep-0209.rst index 9358d6af9..c91a15f0a 100644 --- a/pep-0209.txt +++ b/peps/pep-0209.rst @@ -1,8 +1,6 @@ PEP: 209 Title: Multi-dimensional Arrays -Version: $Revision$ -Last-Modified: $Date$ -Author: barrett@stsci.edu (Paul Barrett), oliphant@ee.byu.edu (Travis Oliphant) +Author: Paul Barrett , Travis Oliphant Status: Withdrawn Type: Standards Track Content-Type: text/x-rst @@ -14,9 +12,9 @@ Post-History: Abstract ======== -This PEP proposes a redesign and re-implementation of the multi- -dimensional array module, Numeric, to make it easier to add new -features and functionality to the module. Aspects of Numeric 2 +This PEP proposes a redesign and re-implementation of the +multi-dimensional array module, Numeric, to make it easier to add +new features and functionality to the module. Aspects of Numeric 2 that will receive special attention are efficient access to arrays exceeding a gigabyte in size and composed of inhomogeneous data structures or records. The proposed design uses four Python @@ -80,7 +78,7 @@ Some planned features are: Instead of using Python's global coercion model which creates temporary arrays, Numeric 2, like Numeric 1, will implement a - local coercion model as described in PEP 208 which defers the + local coercion model as described in :pep:`208` which defers the responsibility of coercion to the operator. By using internal buffers, a coercion operation can be done for each array (including output arrays), if necessary, at the time of the @@ -130,8 +128,8 @@ Some planned features are: automatically handle alignment and representational issues when data is accessed or operated on. There are two approaches to implementing records; as either a derived array - class or a special array type, depending on your point-of- - view. We defer this discussion to the Open Issues section. + class or a special array type, depending on your point-of-view. + We defer this discussion to the Open Issues section. 2. Additional array types @@ -167,7 +165,7 @@ Some planned features are: 4. Rich comparisons - The implementation of PEP 207: Rich Comparisons in Python 2.1 + The implementation of :pep:`207`: Rich Comparisons in Python 2.1 provides additional flexibility when manipulating arrays. We intend to implement this feature in Numeric 2. @@ -267,8 +265,8 @@ The design of Numeric 2 has four primary classes: _ufunc.compute(slice, data, func, swap, conv) The 'func' argument is a CFuncObject, while the 'swap' and 'conv' - arguments are lists of CFuncObjects for those arrays needing pre- - or post-processing, otherwise None is used. The data argument is + arguments are lists of CFuncObjects for those arrays needing pre- or + post-processing, otherwise None is used. The data argument is a list of buffer objects, and the slice argument gives the number of iterations for each dimension along with the buffer offset and step size for each array and each dimension. @@ -684,19 +682,19 @@ This document is placed in the public domain. Related PEPs ============ -* PEP 207: Rich Comparisons +* :pep:`207`: Rich Comparisons by Guido van Rossum and David Ascher -* PEP 208: Reworking the Coercion Model +* :pep:`208`: Reworking the Coercion Model by Neil Schemenauer and Marc-Andre' Lemburg -* PEP 211: Adding New Linear Algebra Operators to Python +* :pep:`211`: Adding New Linear Algebra Operators to Python by Greg Wilson -* PEP 225: Elementwise/Objectwise Operators +* :pep:`225`: Elementwise/Objectwise Operators by Huaiyu Zhu -* PEP 228: Reworking Python's Numeric Model +* :pep:`228`: Reworking Python's Numeric Model by Moshe Zadka @@ -704,10 +702,3 @@ References ========== .. [1] P. Greenfield 2000. private communication. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0210.txt b/peps/pep-0210.rst similarity index 73% rename from pep-0210.txt rename to peps/pep-0210.rst index 986a429ae..68d1b49a9 100644 --- a/pep-0210.txt +++ b/peps/pep-0210.rst @@ -1,8 +1,6 @@ PEP: 210 Title: Decoupling the Interpreter Loop -Version: $Revision$ -Last-Modified: $Date$ -Author: davida@activestate.com (David Ascher) +Author: David Ascher Status: Rejected Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0211.txt b/peps/pep-0211.rst similarity index 81% rename from pep-0211.txt rename to peps/pep-0211.rst index d145d6de0..da2358ba2 100644 --- a/pep-0211.txt +++ b/peps/pep-0211.rst @@ -1,15 +1,20 @@ PEP: 211 Title: Adding A New Outer Product Operator -Version: $Revision$ -Last-Modified: $Date$ -Author: gvwilson@ddj.com (Greg Wilson) +Author: Greg Wilson Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 15-Jul-2000 Python-Version: 2.1 Post-History: -Resolution: https://www.python.org/dev/peps/pep-0465/#rejected-alternatives-to-adding-a-new-operator + + +.. note:: + + The approach in the later :pep:`465` was eventually accepted + in lieu of this PEP. The :pep:`Rejected Ideas + <465#rejected-alternatives-to-adding-a-new-operator>` + of that PEP explains the rationale in more detail. Introduction @@ -31,7 +36,7 @@ will be equivalent to:: Classes will be able to overload this operator using the special methods ``__across__``, ``__racross__``, and ``__iacross__``. In -particular, the new Numeric module (PEP 209) will overload this +particular, the new Numeric module (:pep:`209`) will overload this operator for multi-dimensional arrays to implement matrix multiplication. @@ -49,12 +54,12 @@ elements of its matrix arguments. The other implements the "mathematical" definition of that operation, e.g. performs row-column matrix multiplication. -Zhu and Lielens have proposed doubling up Python's operators in -this way [1]_. Their proposal would create six new binary infix +Zhu and Lielens have :pep:`proposed <225>` doubling up Python's operators in +this way. Their proposal would create six new binary infix operators, and six new in-place operators. The original version of this proposal was much more conservative. -The author consulted the developers of GNU Octave [2]_, an open +The author consulted the developers of GNU Octave [1]_, an open source clone of MATLAB. Its developers agreed that providing an infix operator for matrix multiplication was important: numerical programmers really do care whether they have to write ``mmul(A,B)`` @@ -62,7 +67,7 @@ instead of ``A op B``. On the other hand, when asked how important it was to have infix operators for matrix solution and other operations, Prof. James -Rawlings replied [3]_: +Rawlings replied [2]_: I DON'T think it's a must have, and I do a lot of matrix inversion. I cannot remember if its ``A\b`` or ``b\A`` so I always @@ -77,9 +82,9 @@ Iterators ========= The planned addition of iterators to Python 2.2 opens up a broader -scope for this proposal. As part of the discussion of PEP 201, -Lockstep Iteration [4]_, the author of this proposal conducted an -informal usability experiment [5]_. The results showed that users +scope for this proposal. As part of the discussion of :pep:`201`, +Lockstep Iteration, the author of this proposal conducted an +informal usability experiment [3]_. The results showed that users are psychologically receptive to "cross-product" loop syntax. For example, most users expected:: @@ -163,10 +168,10 @@ Alternatives expression of a very common idiom (nested loops). 3. Introduce prefixed forms of all existing operators, such as - ``~*`` and ``~+``, as proposed in PEP 225 [1]_. + ``~*`` and ``~+``, as proposed in :pep:`225`. Our objections to this are that there isn't enough demand to - justify the additional complexity (see Rawlings' comments [3]_), + justify the additional complexity (see Rawlings' comments [2]_), and that the proposed syntax fails the "low toner" readability test. @@ -182,22 +187,8 @@ discussions of what numerical programmers really care about. References ========== -.. [1] PEP 225, Elementwise/Objectwise Operators, Zhu, Lielens - http://www.python.org/dev/peps/pep-0225/ +.. [1] http://bevo.che.wisc.edu/octave/ -.. [2] http://bevo.che.wisc.edu/octave/ +.. [2] http://www.egroups.com/message/python-numeric/4 -.. [3] http://www.egroups.com/message/python-numeric/4 - -.. [4] PEP 201, Lockstep Iteration, Warsaw - http://www.python.org/dev/peps/pep-0201/ - -.. [5] https://mail.python.org/pipermail/python-dev/2000-July/006427.html - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: +.. [3] https://mail.python.org/pipermail/python-dev/2000-July/006427.html diff --git a/pep-0212.txt b/peps/pep-0212.rst similarity index 92% rename from pep-0212.txt rename to peps/pep-0212.rst index cf7ab8a89..dcaedeec0 100644 --- a/pep-0212.txt +++ b/peps/pep-0212.rst @@ -1,8 +1,6 @@ PEP: 212 Title: Loop Counter Iteration -Version: $Revision$ -Last-Modified: $Date$ -Author: nowonder@nowonder.de (Peter Schneider-Kamp) +Author: Peter Schneider-Kamp Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -14,7 +12,7 @@ Post-History: Rejection Notice ================ -This PEP has been rejected. ``enumerate()``, introduced in PEP 279 [6]_, +This PEP has been rejected. ``enumerate()``, introduced in :pep:`279`, covers the use-case proposed in this PEP, and the PEP author has been unreachable. @@ -175,20 +173,10 @@ References .. [1] http://docs.python.org/reference/compound_stmts.html#for -.. [2] Lockstep Iteration, PEP 201 +.. [2] Lockstep Iteration, :pep:`201` .. [3] http://sourceforge.net/patch/download.php?id=101138 .. [4] http://sourceforge.net/patch/download.php?id=101129 .. [5] http://sourceforge.net/patch/download.php?id=101178 - -.. [6] PEP 279, The enumerate() built-in function, Hettinger - https://www.python.org/dev/peps/pep-0279/ - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0213.txt b/peps/pep-0213.rst similarity index 97% rename from pep-0213.txt rename to peps/pep-0213.rst index adb484f33..8ab4f0bac 100644 --- a/pep-0213.txt +++ b/peps/pep-0213.rst @@ -1,8 +1,6 @@ PEP: 213 Title: Attribute Access Handlers -Version: $Revision$ -Last-Modified: $Date$ -Author: paul@prescod.net (Paul Prescod) +Author: Paul Prescod Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -229,7 +227,7 @@ Caveats Note ==== -The descriptor mechanism described in PEP 252 is powerful enough +The descriptor mechanism described in :pep:`252` is powerful enough to support this more directly. A 'getset' constructor may be added to the language making this possible:: @@ -242,11 +240,3 @@ added to the language making this possible:: Additional syntactic sugar might be added, or a naming convention could be recognized. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0214.txt b/peps/pep-0214.rst similarity index 98% rename from pep-0214.txt rename to peps/pep-0214.rst index 05050689b..6c6e7530f 100644 --- a/pep-0214.txt +++ b/peps/pep-0214.rst @@ -1,8 +1,6 @@ PEP: 214 Title: Extended Print Statement -Version: $Revision$ -Last-Modified: $Date$ -Author: barry@python.org (Barry Warsaw) +Author: Barry Warsaw Status: Final Type: Standards Track Content-Type: text/x-rst @@ -381,10 +379,3 @@ References .. [1] http://docs.python.org/reference/simple_stmts.html#print .. [2] http://sourceforge.net/patch/download.php?id=100970 - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0215.txt b/peps/pep-0215.rst similarity index 96% rename from pep-0215.txt rename to peps/pep-0215.rst index 5fe4d6ab3..bdcdcf11f 100644 --- a/pep-0215.txt +++ b/peps/pep-0215.rst @@ -1,8 +1,6 @@ PEP: 215 Title: String Interpolation -Version: $Revision$ -Last-Modified: $Date$ -Author: ping@zesty.ca (Ka-Ping Yee) +Author: Ka-Ping Yee Status: Superseded Type: Standards Track Content-Type: text/x-rst @@ -149,10 +147,3 @@ References ========== .. [1] http://www.lfw.org/python/Itpl.py - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0216.txt b/peps/pep-0216.rst similarity index 96% rename from pep-0216.txt rename to peps/pep-0216.rst index 31bace5a0..497fc135c 100644 --- a/pep-0216.txt +++ b/peps/pep-0216.rst @@ -1,8 +1,6 @@ PEP: 216 Title: Docstring Format -Version: $Revision$ -Last-Modified: $Date$ -Author: moshez@zadka.site.co.il (Moshe Zadka) +Author: Moshe Zadka Status: Rejected Type: Informational Content-Type: text/x-rst @@ -158,11 +156,3 @@ Rejected Suggestions ==================== XML -- it's very hard to type, and too cluttered to read it comfortably. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0217.txt b/peps/pep-0217.rst similarity index 90% rename from pep-0217.txt rename to peps/pep-0217.rst index 13f2227e0..f1a564148 100644 --- a/pep-0217.txt +++ b/peps/pep-0217.rst @@ -1,8 +1,6 @@ PEP: 217 Title: Display Hook for Interactive Use -Version: $Revision$ -Last-Modified: $Date$ -Author: moshez@zadka.site.co.il (Moshe Zadka) +Author: Moshe Zadka Status: Final Type: Standards Track Content-Type: text/x-rst @@ -57,10 +55,3 @@ Jython Issues ============= The method ``Py.printResult`` will be similarly changed. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0218.txt b/peps/pep-0218.rst similarity index 95% rename from pep-0218.txt rename to peps/pep-0218.rst index 27fe3ba6a..9e8f61a24 100644 --- a/pep-0218.txt +++ b/peps/pep-0218.rst @@ -1,8 +1,6 @@ PEP: 218 Title: Adding a Built-In Set Object Type -Version: $Revision$ -Last-Modified: $Date$ -Author: gvwilson@ddj.com (Greg Wilson), python@rcn.com (Raymond Hettinger) +Author: Greg Wilson , Raymond Hettinger Status: Final Type: Standards Track Content-Type: text/x-rst @@ -140,10 +138,10 @@ dictionaries less instantly recognizable. It was also contemplated that the braced notation would support set comprehensions; however, Python 2.4 provided generator expressions which fully met that need and did so it a more general way. -(See PEP 289 for details on generator expressions). +(See :pep:`289` for details on generator expressions). So, Guido ruled that there would not be a set syntax; however, the -issue could be revisited for Python 3000 (see PEP 3000). +issue could be revisited for Python 3000 (see :pep:`3000`). History @@ -195,7 +193,7 @@ reliable lookup. While it would be easy to require set elements to be immutable, this would preclude sets of sets (which are widely used in graph algorithms and other applications). -Earlier drafts of PEP 218 had only a single set type, but the +Earlier drafts of :pep:`218` had only a single set type, but the ``sets.py`` implementation in Python 2.3 has two, Set and ImmutableSet. For Python 2.4, the new built-in types were named ``set`` and ``frozenset`` which are slightly less cumbersome. @@ -214,11 +212,3 @@ Copyright ========= This document has been placed in the Public Domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0219.txt b/peps/pep-0219.rst similarity index 96% rename from pep-0219.txt rename to peps/pep-0219.rst index 48c9560cd..c13f27c4a 100644 --- a/pep-0219.txt +++ b/peps/pep-0219.rst @@ -1,8 +1,6 @@ PEP: 219 Title: Stackless Python -Version: $Revision$ -Last-Modified: $Date$ -Author: gmcm@hypernet.com (Gordon McMillan) +Author: Gordon McMillan Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -16,7 +14,7 @@ Introduction This PEP discusses changes required to core Python in order to efficiently support generators, microthreads and coroutines. It is -related to PEP 220, which describes how Python should be extended +related to :pep:`220`, which describes how Python should be extended to support these facilities. The focus of this PEP is strictly on the changes required to allow these extensions to work. @@ -59,7 +57,7 @@ way for making Python able to realistically manage thousands of separate "threads" of activity (vs. today's limit of perhaps dozens of separate threads of activity). -Another justification for this PEP (explored in PEP 220) is that +Another justification for this PEP (explored in :pep:`220`) is that coroutines and generators often allow a more direct expression of an algorithm than is possible in today's Python. @@ -189,11 +187,3 @@ References .. [2] http://web.archive.org/web/20000815070602/http://world.std.com/~wware/uthread.html .. [3] Demo/threads/Generator.py in the source distribution .. [4] http://www.stackless.com/coroutines.tim.peters.html - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0220.txt b/peps/pep-0220.rst similarity index 78% rename from pep-0220.txt rename to peps/pep-0220.rst index 597cc4715..9d9a47609 100644 --- a/pep-0220.txt +++ b/peps/pep-0220.rst @@ -1,8 +1,6 @@ PEP: 220 Title: Coroutines, Generators, Continuations -Version: $Revision$ -Last-Modified: $Date$ -Author: gmcm@hypernet.com (Gordon McMillan) +Author: Gordon McMillan Status: Rejected Type: Informational Content-Type: text/x-rst @@ -20,11 +18,3 @@ higher level module that makes coroutines and generators easy to create is desirable (and being worked on). The focus of this PEP is on showing how coroutines, generators, and green threads can simplify common programming problems. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0221.txt b/peps/pep-0221.rst similarity index 95% rename from pep-0221.txt rename to peps/pep-0221.rst index 34efd28de..0b92c64c0 100644 --- a/pep-0221.txt +++ b/peps/pep-0221.rst @@ -1,8 +1,6 @@ PEP: 221 Title: Import As -Version: $Revision$ -Last-Modified: $Date$ -Author: thomas@python.org (Thomas Wouters) +Author: Thomas Wouters Status: Final Type: Standards Track Content-Type: text/x-rst @@ -114,11 +112,3 @@ References .. [1] https://hg.python.org/cpython/rev/18385172fac0 .. [2] http://sourceforge.net/patch/?func=detailpatch&patch_id=101234&group_id=5470 - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0222.txt b/peps/pep-0222.rst similarity index 100% rename from pep-0222.txt rename to peps/pep-0222.rst diff --git a/pep-0223.txt b/peps/pep-0223.rst similarity index 97% rename from pep-0223.txt rename to peps/pep-0223.rst index ef240517d..e7238cf0a 100644 --- a/pep-0223.txt +++ b/peps/pep-0223.rst @@ -1,8 +1,6 @@ PEP: 223 Title: Change the Meaning of ``\x`` Escapes -Version: $Revision$ -Last-Modified: $Date$ -Author: tim.peters@gmail.com (Tim Peters) +Author: Tim Peters Status: Final Type: Standards Track Content-Type: text/x-rst @@ -224,11 +222,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0224.txt b/peps/pep-0224.rst similarity index 98% rename from pep-0224.txt rename to peps/pep-0224.rst index a208b002a..2c7cd0a1f 100644 --- a/pep-0224.txt +++ b/peps/pep-0224.rst @@ -1,8 +1,6 @@ PEP: 224 Title: Attribute Docstrings -Version: $Revision$ -Last-Modified: $Date$ -Author: mal@lemburg.com (Marc-AndrĂ© Lemburg) +Author: Marc-AndrĂ© Lemburg Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -266,11 +264,3 @@ References ========== .. [1] http://sourceforge.net/patch/?func=detailpatch&patch_id=101264&group_id=5470 - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0225.txt b/peps/pep-0225.rst similarity index 98% rename from pep-0225.txt rename to peps/pep-0225.rst index 23e8bb625..b66aadee8 100644 --- a/pep-0225.txt +++ b/peps/pep-0225.rst @@ -1,16 +1,21 @@ PEP: 225 Title: Elementwise/Objectwise Operators -Version: $Revision$ -Last-Modified: $Date$ -Author: hzhu@users.sourceforge.net (Huaiyu Zhu), - gregory.lielens@fft.be (Gregory Lielens) +Author: Huaiyu Zhu , + Gregory Lielens Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 19-Sep-2000 Python-Version: 2.1 Post-History: -Resolution: https://www.python.org/dev/peps/pep-0465/#rejected-alternatives-to-adding-a-new-operator + + +.. note:: + + The approach in the later :pep:`465` was eventually accepted + in lieu of this PEP. The :pep:`Rejected Ideas + <465#rejected-alternatives-to-adding-a-new-operator>` + of that PEP explains the rationale in more detail. Introduction @@ -751,24 +756,16 @@ These are earlier drafts of this PEP: http://www.python.org/pipermail/python-list/2000-August/112529.html http://www.python.org/pipermail/python-dev/2000-August/014906.html -There is an alternative PEP (officially, PEP 211) by Greg Wilson, titled +There is an alternative PEP (officially, :pep:`211`) by Greg Wilson, titled "Adding New Linear Algebra Operators to Python". Its first (and current) version is at: http://www.python.org/pipermail/python-dev/2000-August/014876.html - http://www.python.org/dev/peps/pep-0211/ + :pep:`211` Additional References ===================== .. [1] http://MatPy.sourceforge.net/Misc/index.html - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0226.txt b/peps/pep-0226.rst similarity index 98% rename from pep-0226.txt rename to peps/pep-0226.rst index a1a030c02..67ebac990 100644 --- a/pep-0226.txt +++ b/peps/pep-0226.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Jeremy Hylton Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 16-Oct-2000 Python-Version: 2.1 @@ -53,7 +54,7 @@ The guidelines and schedule will be revised based on discussion in the python-dev@python.org mailing list. The PEP system was instituted late in the Python 2.0 development -cycle and many changes did not follow the process described in PEP 1. +cycle and many changes did not follow the process described in :pep:`1`. The development process for 2.1, however, will follow the PEP process as documented. diff --git a/pep-0227.txt b/peps/pep-0227.rst similarity index 98% rename from pep-0227.txt rename to peps/pep-0227.rst index cca2defbf..4bd514b9b 100644 --- a/pep-0227.txt +++ b/peps/pep-0227.rst @@ -1,8 +1,6 @@ PEP: 227 Title: Statically Nested Scopes -Version: $Revision$ -Last-Modified: $Date$ -Author: jeremy@alum.mit.edu (Jeremy Hylton) +Author: Jeremy Hylton Status: Final Type: Standards Track Content-Type: text/x-rst @@ -37,7 +35,7 @@ This proposal changes the rules for resolving free variables in Python functions. The new name resolution semantics will take effect with Python 2.2. These semantics will also be available in Python 2.1 by adding "from __future__ import nested_scopes" to the -top of a module. (See PEP 236.) +top of a module. (See :pep:`236`.) The Python 2.0 definition specifies exactly three namespaces to check for each name -- the local namespace, the global namespace, @@ -502,14 +500,7 @@ References .. [1] Luca Cardelli. Compiling a functional language. In Proc. of the 1984 ACM Conference on Lisp and Functional Programming, pp. 208-217, Aug. 1984 - http://citeseer.ist.psu.edu/cardelli84compiling.html + https://dl.acm.org/doi/10.1145/800055.802037 Copyright ========= - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0228.txt b/peps/pep-0228.rst similarity index 94% rename from pep-0228.txt rename to peps/pep-0228.rst index 7469200ab..4b708fc1e 100644 --- a/pep-0228.txt +++ b/peps/pep-0228.rst @@ -1,8 +1,6 @@ PEP: 228 Title: Reworking Python's Numeric Model -Version: $Revision$ -Last-Modified: $Date$ -Author: moshez@zadka.site.co.il (Moshe Zadka), guido@python.org (Guido van Rossum) +Author: Moshe Zadka , Guido van Rossum Status: Withdrawn Type: Standards Track Content-Type: text/x-rst @@ -13,7 +11,7 @@ Post-History: Withdrawal ========== -This PEP has been withdrawn in favor of PEP 3141. +This PEP has been withdrawn in favor of :pep:`3141`. Abstract @@ -146,11 +144,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0229.txt b/peps/pep-0229.rst similarity index 99% rename from pep-0229.txt rename to peps/pep-0229.rst index ca7581968..f538d1677 100644 --- a/pep-0229.txt +++ b/peps/pep-0229.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 16-Nov-2000 +Python-Version: 2.1 Post-History: diff --git a/pep-0230.txt b/peps/pep-0230.rst similarity index 98% rename from pep-0230.txt rename to peps/pep-0230.rst index eb404ae4a..d521a6938 100644 --- a/pep-0230.txt +++ b/peps/pep-0230.rst @@ -1,8 +1,6 @@ PEP: 230 Title: Warning Framework -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Final Type: Standards Track Content-Type: text/x-rst @@ -410,8 +408,3 @@ Implementation Here's a prototype implementation: http://sourceforge.net/patch/?func=detailpatch&patch_id=102715&group_id=5470 -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0231.txt b/peps/pep-0231.rst similarity index 97% rename from pep-0231.txt rename to peps/pep-0231.rst index 2600bf82d..3ff93081c 100644 --- a/pep-0231.txt +++ b/peps/pep-0231.rst @@ -1,8 +1,6 @@ PEP: 231 Title: __findattr__() -Version: $Revision$ -Last-Modified: $Date$ -Author: barry@python.org (Barry Warsaw) +Author: Barry Warsaw Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -120,9 +118,9 @@ is used. Related Work ============ -PEP 213 [9]_ describes a different approach to hooking into -attribute access and modification. The semantics proposed in PEP -213 can be implemented using the ``__findattr__()`` hook described +:pep:`213` describes a different approach to hooking into +attribute access and modification. The semantics proposed in :pep:`213` +can be implemented using the ``__findattr__()`` hook described here, with one caveat. The current reference implementation of ``__findattr__()`` does not support hooking on attribute deletion. This could be added if it's found desirable. See example below. @@ -613,9 +611,8 @@ References .. [5] http://www.digicool.com/releases/ExtensionClass .. [6] http://www.python.org/doc/essays/metaclasses/ .. [7] http://www.foretec.com/python/workshops/1998-11/dd-ascher-sum.html -.. [8] http://docs.python.org/howto/regex.html -.. [9] PEP 213, Attribute Access Handlers, Prescod - http://www.python.org/dev/peps/pep-0213/ + +* http://docs.python.org/howto/regex.html Rejection @@ -643,10 +640,3 @@ Copyright ========= This document has been placed in the Public Domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0232.txt b/peps/pep-0232.rst similarity index 94% rename from pep-0232.txt rename to peps/pep-0232.rst index c146391ed..0f65294c6 100644 --- a/pep-0232.txt +++ b/peps/pep-0232.rst @@ -174,7 +174,9 @@ the 2.1 release. syntactic support for conveniently setting. It may be worthwhile to eventually enhance the language for supporting easy function attribute setting. Here are some syntaxes - suggested by PEP reviewers:: + suggested by PEP reviewers: [3]_ + + .. code:: python def a { 'publish' : 1, @@ -243,25 +245,15 @@ References ========== .. [1] Aycock, "Compiling Little Languages in Python", - http://www.foretec.com/python/workshops/1998-11/proceedings/papers/aycock-little/aycock-little.html + https://legacy.python.org/workshops/1998-11/proceedings/papers/aycock-little/aycock-little.html -.. [2] http://classic.zope.org:8080/Documentation/Reference/ORB +.. [2] https://web.archive.org/web/20010307022153/http://classic.zope.org:8080/Documentation/Reference/ORB .. [3] Hudson, Michael, SourceForge patch implementing this syntax, - http://sourceforge.net/tracker/index.php?func=detail&aid=403441&group_id=5470&atid=305470 + https://web.archive.org/web/20010901050535/http://sourceforge.net/tracker/index.php?func=detail&aid=403441&group_id=5470&atid=305470 Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0233.txt b/peps/pep-0233.rst similarity index 94% rename from pep-0233.txt rename to peps/pep-0233.rst index 02393efcc..313e199bd 100644 --- a/pep-0233.txt +++ b/peps/pep-0233.rst @@ -1,8 +1,6 @@ PEP: 233 Title: Python Online Help -Version: $Revision$ -Last-Modified: $Date$ -Author: paul@prescod.net (Paul Prescod) +Author: Paul Prescod Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -108,11 +106,3 @@ Security Issues This module will attempt to import modules with the same names as requested topics. Don't use the modules if you are not confident that everything in your ``PYTHONPATH`` is from a trusted source. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0234.txt b/peps/pep-0234.rst similarity index 99% rename from pep-0234.txt rename to peps/pep-0234.rst index 7ce5d44f6..74640a9ea 100644 --- a/pep-0234.txt +++ b/peps/pep-0234.rst @@ -1,8 +1,6 @@ PEP: 234 Title: Iterators -Version: $Revision$ -Last-Modified: $Date$ -Author: ping@zesty.ca (Ka-Ping Yee), guido@python.org (Guido van Rossum) +Author: Ka-Ping Yee , Guido van Rossum Status: Final Type: Standards Track Content-Type: text/x-rst @@ -483,11 +481,3 @@ Copyright ========= This document is in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0235.txt b/peps/pep-0235.rst similarity index 97% rename from pep-0235.txt rename to peps/pep-0235.rst index 41b96a718..d5a374afa 100644 --- a/pep-0235.txt +++ b/peps/pep-0235.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 21-Feb-2001 Python-Version: 2.1 -Post-History: 16 February 2001 +Post-History: 16-Feb-2001 Note @@ -125,8 +125,8 @@ A. If the ``PYTHONCASEOK`` environment variable exists, same as B. Else search ``sys.path`` for the first case-sensitive match; raise ``ImportError`` if none found. -#B is the same rule as is used on Unix, so this will improve cross- -platform portability. That's good. #B is also the rule the Mac +#B is the same rule as is used on Unix, so this will improve +cross-platform portability. That's good. #B is also the rule the Mac and Cygwin folks want (and wanted enough to implement themselves, multiple times, which is a powerful argument in PythonLand). It can't cause any existing non-exceptional Windows import to fail, diff --git a/pep-0236.txt b/peps/pep-0236.rst similarity index 92% rename from pep-0236.txt rename to peps/pep-0236.rst index 9cc4d3b44..f1043a3c0 100644 --- a/pep-0236.txt +++ b/peps/pep-0236.rst @@ -20,17 +20,17 @@ semantics of core language constructs, or changes their accidental capriciously, and is always done with the aim of improving the language over the long term, over the short term it's contentious and disrupting. -PEP 5, Guidelines for Language Evolution [1]_ suggests ways to ease the pain, +:pep:`5`, Guidelines for Language Evolution suggests ways to ease the pain, and this PEP introduces some machinery in support of that. -PEP 227, Statically Nested Scopes [2]_ is the first application, and will be +:pep:`227`, Statically Nested Scopes is the first application, and will be used as an example here. Intent ====== -[Note: This is policy, and so should eventually move into PEP 5 [1]_] +[Note: This is policy, and so should eventually move into :pep:`5`] When an incompatible change to core language syntax or semantics is being made: @@ -41,9 +41,9 @@ made: 2. A future release R is identified in which the new syntax or semantics will be enforced. -3. The mechanisms described in PEP 3, Warning Framework [3]_ are used to +3. The mechanisms described in :pep:`230`, Warning Framework are used to generate warnings, whenever possible, about constructs or operations whose - meaning may [4]_ change in release R. + meaning may [1]_ change in release R. 4. The new future_statement (see below) can be explicitly included in a module M to request that the code in module M use the new syntax or semantics in @@ -106,7 +106,7 @@ cannot be pushed off until runtime. For any given release, the compiler knows which feature names have been defined, and raises a compile-time error if a future_statement contains a -feature not known to it [5]_. +feature not known to it [2]_. The direct runtime semantics are the same as for any ``import`` statement: there is a standard module ``__future__.py``, described later, and it will be @@ -140,7 +140,7 @@ Under 2.0, it prints:: x is 42 -Nested scopes [2]_ are being introduced in 2.1. But under 2.1, it still +Nested scopes (:pep:`227`) are being introduced in 2.1. But under 2.1, it still prints:: x is 42 @@ -241,7 +241,7 @@ This isn't always desired, though. For example, ``doctest.testmod(M)`` compiles examples taken from strings in M, and those examples should use M's choices, not necessarily the doctest module's choices. In the 2.1 release, this isn't possible, and no scheme has yet been suggested for working around -this. NOTE: PEP 264 later addressed this in a flexible way, by adding +this. NOTE: :pep:`264` later addressed this in a flexible way, by adding optional arguments to ``compile()``. In any case, a future_statement appearing "near the top" (see Syntax above) of @@ -279,10 +279,10 @@ However, the machinery used internally by native interactive shells has not been exposed, and there isn't a clear way for tools building their own interactive shells to achieve the desired behavior. -NOTE: PEP 264 later addressed this, by adding intelligence to the standard +NOTE: :pep:`264` later addressed this, by adding intelligence to the standard ``codeop.py``. Simulated shells that don't use the standard library shell helpers can get a similar effect by exploiting the new optional arguments to -``compile()`` added by PEP 264. +``compile()`` added by :pep:`264`. Questions and Answers @@ -297,7 +297,7 @@ PEP if you want to pursue it. What about incompatibilities due to changes in the Python virtual machine? -------------------------------------------------------------------------- -Outside the scope of this PEP, although PEP 5 [1]_ suggests a grace period +Outside the scope of this PEP, although :pep:`5` suggests a grace period there too, and the future_statement may also have a role to play there. What about incompatibilities due to changes in Python's C API? @@ -350,19 +350,10 @@ This document has been placed in the public domain. References and Footnotes ======================== -.. [1] PEP 5, Guidelines for Language Evolution, Prescod - http://www.python.org/dev/peps/pep-0005/ - -.. [2] PEP 227, Statically Nested Scopes, Hylton - http://www.python.org/dev/peps/pep-0227/ - -.. [3] PEP 230, Warning Framework, Van Rossum - http://www.python.org/dev/peps/pep-0230/ - -.. [4] Note that this is *may* and not *will*: better safe than sorry. Of course +.. [1] Note that this is *may* and not *will*: better safe than sorry. Of course spurious warnings won't be generated when avoidable with reasonable cost. -.. [5] This ensures that a future_statement run under a release prior to the +.. [2] This ensures that a future_statement run under a release prior to the first one in which a given feature is known (but >= 2.1) will raise a compile-time error rather than silently do a wrong thing. If transported to a release prior to 2.1, a runtime error will be raised because of the diff --git a/pep-0237.txt b/peps/pep-0237.rst similarity index 100% rename from pep-0237.txt rename to peps/pep-0237.rst diff --git a/pep-0238.txt b/peps/pep-0238.rst similarity index 93% rename from pep-0238.txt rename to peps/pep-0238.rst index c415d6a68..a78256f3f 100644 --- a/pep-0238.txt +++ b/peps/pep-0238.rst @@ -1,9 +1,7 @@ PEP: 238 Title: Changing the Division Operator -Version: $Revision$ -Last-Modified: $Date$ -Author: moshez@zadka.site.co.il (Moshe Zadka), - guido@python.org (Guido van Rossum) +Author: Moshe Zadka , + Guido van Rossum Status: Final Type: Standards Track Content-Type: text/x-rst @@ -96,7 +94,7 @@ of code to be fixed, but the amount of code that might be affected by the bug in the future is unbounded. Another reason for this change is the desire to ultimately unify Python's -numeric model. This is the subject of PEP 228 [0]_ (which is currently +numeric model. This is the subject of :pep:`228` (which is currently incomplete). A unified numeric model removes most of the user's need to be aware of different numerical types. This is good for beginners, but also takes away concerns about different numeric behavior for advanced programmers. @@ -156,7 +154,7 @@ discussed on c.l.py that isn't mentioned here, please mail the second author. - Let ``/`` keep its classic semantics; introduce ``//`` for true division. This still leaves a broken operator in the language, and invites to use the broken behavior. It also shuts off the road to a unified numeric model a la - PEP 228 [0]_. + :pep:`228`. - Let int division return a special "portmanteau" type that behaves as an integer in integer context, but like a float in a float context. The @@ -316,7 +314,7 @@ language). Algorithms that consciously use longs should consider using ``//``, as true division of longs retains no more than 53 bits of precision (on most platforms). -If and when a rational type is added to Python (see PEP 239 [2]_), true +If and when a rational type is added to Python (see :pep:`239`), true division for ints and longs should probably return a rational. This avoids the problem with true division of ints and longs losing information. But until then, for consistency, float is the only choice for true division. @@ -333,7 +331,7 @@ Python 3.0 comes along, where they are always translated to true division). The future division statement has no effect on the recognition or translation of ``//`` and ``//=``. -See PEP 236 [4]_ for the general rules for future statements. +See :pep:`236` for the general rules for future statements. (It has been proposed to use a longer phrase, like *true_division* or *modern_division*. These don't seem to add much information.) @@ -416,7 +414,7 @@ Why isn't true division called float division? ---------------------------------------------- Because I want to keep the door open to *possibly* introducing rationals - and making 1/2 return a rational rather than a float. See PEP 239 [2]_. + and making 1/2 return a rational rather than a float. See :pep:`239`. Why is there a need for ``__truediv__`` and ``__itruediv__``? ------------------------------------------------------------- @@ -427,7 +425,7 @@ Why is there a need for ``__truediv__`` and ``__itruediv__``? How do I write code that works under the classic rules as well as under the new rules without using ``//`` or a future division statement? ------------------------------------------------------------------------------------------------------------------------------------------ - Use ``x*1.0/y`` for true division, ``divmod(x, y)`` [0]_ for int + Use ``x*1.0/y`` for true division, ``divmod(x, y)`` (:pep:`228`) for int division. Especially the latter is best hidden inside a function. You may also write ``float(x)/y`` for true division if you are sure that you don't expect complex numbers. If you know your integers are never @@ -442,13 +440,13 @@ How do I write code that works under the classic rules as well as under the new How do I specify the division semantics for ``input()``, ``compile()``, ``execfile()``, ``eval()`` and ``exec``? ---------------------------------------------------------------------------------------------------------------- - They inherit the choice from the invoking module. PEP 236 [4]_ now lists - this as a resolved problem, referring to PEP 264 [5]_. + They inherit the choice from the invoking module. :pep:`236` now lists + this as a resolved problem, referring to :pep:`264`. What about code compiled by the codeop module? ---------------------------------------------- - This is dealt with properly; see PEP 264 [5]_. + This is dealt with properly; see :pep:`264`. Will there be conversion tools or aids? --------------------------------------- @@ -476,37 +474,7 @@ Essentially everything mentioned here is implemented in CVS and will be released with Python 2.2a3; most of it was already released with Python 2.2a2. -References -========== - -.. [0] PEP 228, Reworking Python's Numeric Model - http://www.python.org/dev/peps/pep-0228/ - -.. [1] PEP 237, Unifying Long Integers and Integers, Zadka, - http://www.python.org/dev/peps/pep-0237/ - -.. [2] PEP 239, Adding a Rational Type to Python, Zadka, - http://www.python.org/dev/peps/pep-0239/ - -.. [3] PEP 240, Adding a Rational Literal to Python, Zadka, - http://www.python.org/dev/peps/pep-0240/ - -.. [4] PEP 236, Back to the __future__, Peters, - http://www.python.org/dev/peps/pep-0236/ - -.. [5] PEP 264, Future statements in simulated shells - http://www.python.org/dev/peps/pep-0236/ - - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0239.txt b/peps/pep-0239.rst similarity index 91% rename from pep-0239.txt rename to peps/pep-0239.rst index 0ccafaed8..b56f4d89b 100644 --- a/pep-0239.txt +++ b/peps/pep-0239.rst @@ -18,19 +18,19 @@ Python has no numeric type with the semantics of an unboundedly precise rational number. This proposal explains the semantics of such a type, and suggests builtin functions and literals to support such a type. This PEP suggests no literals for rational -numbers; that is left for another PEP [1]_. +numbers; that is left for :pep:`another PEP <240>`. BDFL Pronouncement ================== This PEP is rejected. The needs outlined in the rationale section -have been addressed to some extent by the acceptance of PEP 327 +have been addressed to some extent by the acceptance of :pep:`327` for decimal arithmetic. Guido also noted, "Rational arithmetic was the default 'exact' arithmetic in ABC and it did not work out as -expected". See the python-dev discussion on 17 June 2005 [2]_. +expected". See the python-dev discussion on 17 June 2005 [1]_. -*Postscript:* With the acceptance of PEP 3141, "A Type Hierarchy +*Postscript:* With the acceptance of :pep:`3141`, "A Type Hierarchy for Numbers", a 'Rational' numeric abstract base class was added with a concrete implementation in the 'fractions' module. @@ -127,10 +127,7 @@ Open Issues References ========== -.. [1] PEP 240, Adding a Rational Literal to Python, Zadka, - http://www.python.org/dev/peps/pep-0240/ - -.. [2] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin +.. [1] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin rational type and rational literals https://mail.python.org/pipermail/python-dev/2005-June/054281.html diff --git a/pep-0240.txt b/peps/pep-0240.rst similarity index 83% rename from pep-0240.txt rename to peps/pep-0240.rst index 3d5c40f5f..6ee7e0345 100644 --- a/pep-0240.txt +++ b/peps/pep-0240.rst @@ -1,7 +1,5 @@ PEP: 240 Title: Adding a Rational Literal to Python -Version: $Revision$ -Last-Modified: $Date$ Author: Christopher A. Craig , Moshe Zadka Status: Rejected Type: Standards Track @@ -14,7 +12,7 @@ Post-History: 16-Mar-2001 Abstract ======== -A different PEP [1]_ suggests adding a builtin rational type to +A :pep:`different PEP <239>` suggests adding a builtin rational type to Python. This PEP suggests changing the ddd.ddd float literal to a rational in Python, and modifying non-integer division to return it. @@ -24,10 +22,10 @@ BDFL Pronouncement ================== This PEP is rejected. The needs outlined in the rationale section -have been addressed to some extent by the acceptance of PEP 327 +have been addressed to some extent by the acceptance of :pep:`327` for decimal arithmetic. Guido also noted, "Rational arithmetic was the default 'exact' arithmetic in ABC and it did not work out as -expected". See the python-dev discussion on 17 June 2005 [2]_. +expected". See the python-dev discussion on 17 June 2005 [1]_. Rationale @@ -61,7 +59,7 @@ mentioned above. The following migration is suggested: to cause all such literals to be treated as rational numbers. 2. Python 3.0 will have a warning, turned on by default, about - such literals in the absence of a `` __future__`` statement. The + such literals in the absence of a ``__future__`` statement. The warning message will contain information about the ``__future__`` statement, and indicate that to get floating point literals, they should be suffixed with "e0". @@ -88,9 +86,7 @@ point, which gives us a new loss of precision. References ========== -.. [1] PEP 239, Adding a Rational Type to Python, Zadka, - http://www.python.org/dev/peps/pep-0239/ -.. [2] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin +.. [1] Raymond Hettinger, Propose rejection of PEPs 239 and 240 -- a builtin rational type and rational literals https://mail.python.org/pipermail/python-dev/2005-June/054281.html @@ -99,11 +95,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0241.txt b/peps/pep-0241.rst similarity index 95% rename from pep-0241.txt rename to peps/pep-0241.rst index 9a99650ce..e82b2267b 100644 --- a/pep-0241.txt +++ b/peps/pep-0241.rst @@ -1,13 +1,14 @@ PEP: 241 Title: Metadata for Python Software Packages -Version: $Revision$ -Last-Modified: $Date$ Author: A.M. Kuchling -Status: Final +Discussions-To: distutils-sig@python.org +Status: Superseded Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 12-Mar-2001 -Post-History: 19-Mar-2001 +Post-History: `19-Mar-2001 `__ +Superseded-By: 314 Introduction @@ -33,7 +34,7 @@ command will, if it detects an existing PKG-INFO file, terminate with an appropriate error message. This should prevent confusion caused by the PKG-INFO and setup.py files being out of sync. -The PKG-INFO file format is a single set of RFC-822 headers +The PKG-INFO file format is a single set of :rfc:`822` headers parseable by the rfc822.py module. The field names listed in the following section are used as the header names. There's no extension mechanism in this simple format; the Catalog and Distutils @@ -165,7 +166,7 @@ Author-email ------------ A string containing the author's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` 'From:' header. It's not optional because cataloging systems can use the e-mail portion of this field as a unique key representing the author. A catalog might provide authors the diff --git a/pep-0242.txt b/peps/pep-0242.rst similarity index 98% rename from pep-0242.txt rename to peps/pep-0242.rst index e3cebbd12..92f383470 100644 --- a/pep-0242.txt +++ b/peps/pep-0242.rst @@ -1,8 +1,6 @@ PEP: 242 Title: Numeric Kinds -Version: $Revision$ -Last-Modified: $Date$ -Author: paul@pfdubois.com (Paul F. Dubois) +Author: Paul F. Dubois Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -237,10 +235,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0243.txt b/peps/pep-0243.rst similarity index 87% rename from pep-0243.txt rename to peps/pep-0243.rst index 8352846a4..8d38d8b70 100644 --- a/pep-0243.txt +++ b/peps/pep-0243.rst @@ -1,11 +1,10 @@ PEP: 243 Title: Module Repository Upload Mechanism -Version: $Revision$ -Last-Modified: $Date$ -Author: jafo-pep@tummy.com (Sean Reifschneider) +Author: Sean Reifschneider Discussions-To: distutils-sig@python.org Status: Withdrawn Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 18-Mar-2001 Python-Version: 2.1 @@ -34,11 +33,11 @@ Upload Process ============== The upload will include the Distutils ``PKG-INFO`` meta-data -information (as specified in PEP 241 [1]_), the actual software +information (as specified in :pep:`241`), the actual software distribution, and other optional information. This information will be uploaded as a multi-part form encoded the same as a regular HTML file upload request. This form is posted using -``ENCTYPE="multipart/form-data"`` encoding [2]_. +``ENCTYPE="multipart/form-data"`` encoding (:rfc:`1867`). The upload will be made to the host "www.python.org" on port 80/tcp (``POST http://www.python.org:80/pypi``). The form @@ -53,7 +52,7 @@ will consist of the following fields: ord(byte))``). - ``pkginfo`` (optional) -- The file containing the distribution - meta-data (as specified in PEP 241 [1]_). Note that if this is + meta-data (as specified in :pep:`241`). Note that if this is not included, the distribution file is expected to be in ``.tar`` format (gzipped and bzipped compressed are allowed) or ``.zip`` format, with a ``PKG-INFO`` file in the top-level directory it @@ -70,7 +69,7 @@ will consist of the following fields: ``---``. -- ``signature`` (optional) -- A OpenPGP-compatible signature [3]_ of +- ``signature`` (optional) -- A :rfc:`OpenPGP-compatible <2440>` signature of the uploaded distribution as signed by the author. This may be used by the cataloging system to automate acceptance of uploads. @@ -161,34 +160,13 @@ Status I currently have a proof-of-concept client and server implemented. I plan to have the Distutils patches ready for the 2.1 release. -Combined with Andrew's PEP 241 [1]_ for specifying distribution +Combined with Andrew's :pep:`241` for specifying distribution meta-data, I hope to have a platform which will allow us to gather real-world data for finalizing the catalog system for the 2.2 release. -References -========== - -.. [1] Metadata for Python Software Package, Kuchling, - http://www.python.org/dev/peps/pep-0241/ - -.. [2] RFC 1867, Form-based File Upload in HTML - http://www.faqs.org/rfcs/rfc1867.html - -.. [3] RFC 2440, OpenPGP Message Format - http://www.faqs.org/rfcs/rfc2440.html - - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0244.txt b/peps/pep-0244.rst similarity index 84% rename from pep-0244.txt rename to peps/pep-0244.rst index c54924afe..59b0759fe 100644 --- a/pep-0244.txt +++ b/peps/pep-0244.rst @@ -1,8 +1,6 @@ PEP: 244 Title: The ``directive`` statement -Version: $Revision$ -Last-Modified: $Date$ -Author: martin@v.loewis.de (Martin von Löwis) +Author: Martin von Löwis Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -21,11 +19,11 @@ this is never done capriciously, and is always done with the aim of improving the language over the long term, over the short term it's contentious and disrupting. -PEP 1, Guidelines for Language Evolution [1]_ suggests ways to ease +:pep:`5`, Guidelines for Language Evolution suggests ways to ease the pain, and this PEP introduces some machinery in support of that. -PEP 2, Statically Nested Scopes [2]_ is the first application, and +:pep:`227`, Statically Nested Scopes is the first application, and will be used as an example here. When a new, potentially incompatible language feature is added, @@ -40,7 +38,7 @@ kinds of "settable" language features: - those that are designed to eventually become the only option, at which time specifying use of them is not necessary anymore. The features for which the syntax of the "Back to the ``__future__``" - PEP 236, Back to the ``__future__`` [3]_ was proposed fall into this + :pep:`236`, Back to the ``__future__`` was proposed fall into this category. This PEP supports declaring such features, and supports phasing out the "old" meaning of constructs whose semantics has changed under the new feature. However, it @@ -75,7 +73,7 @@ placed on the directive (e.g. placement of the directive in the module may be restricted to the top of the module). In the directive_statement, ``directive`` is a new -keyword. According to [1]_, this keyword is initially considered as +keyword. According to :pep:`5`, this keyword is initially considered as a keyword only when used in a directive statement, see "Backwards Compatibility" below. @@ -94,7 +92,7 @@ Specific Directives: transitional ================================= If a syntactical or semantical change is added to Python which is -incompatible, [1]_ mandates a transitional evolution of the +incompatible, :pep:`5` mandates a transitional evolution of the language, where the new feature is initially available alongside with the old one. Such a transition is possible by means of the transitional directive. @@ -117,7 +115,7 @@ Backwards Compatibility Introducing ``directive`` as a new keyword might cause incompatibilities with existing code. Following the guideline in -[1]_, in the initial implementation of this specification, +:pep:`5`, in the initial implementation of this specification, directive is a new keyword only if it was used in a valid directive_statement (i.e. if it appeared as the first non-string token in a module). @@ -147,7 +145,7 @@ allow source code encodings, no specific directive is proposed. **Q:** Then why was this PEP written at all? -**A:** It acts as a counter-proposal to [3]_, which proposes to +**A:** It acts as a counter-proposal to :pep:`236`, which proposes to overload the import statement with a new meaning. This PEP allows to solve the problem in a more general way. @@ -158,28 +156,7 @@ mixing apples and oranges? "transitional" directive has been defined. -References and Footnotes -======================== - -.. [1] PEP 5, Guidelines for Language Evolution, Prescod - http://www.python.org/dev/peps/pep-0005/ - -.. [2] PEP 227, Statically Nested Scopes, Hylton - http://www.python.org/dev/peps/pep-0227/ - -.. [3] PEP 236, Back to the ``__future__``, Peters - http://www.python.org/dev/peps/pep-0236/ - - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0245.txt b/peps/pep-0245.rst similarity index 94% rename from pep-0245.txt rename to peps/pep-0245.rst index 84fac96ed..ae79afb96 100644 --- a/pep-0245.txt +++ b/peps/pep-0245.rst @@ -3,7 +3,6 @@ Title: Python Interface Syntax Version: $Revision$ Last-Modified: $Date$ Author: Michel Pelletier -Discussions-To: http://www.zope.org/Wikis/Interfaces Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -12,12 +11,24 @@ Python-Version: 2.2 Post-History: 21-Mar-2001 +.. note:: + + The no-longer-available Zope interfaces wiki page + (``https://www.zope.org/Wikis/Interfaces``) originally linked here, + containing links to further resources for this PEP, + can be `found on the Wayback Machine archive + `__. + Also, the Interface-Dev Zope mailing list on which this PEP was discussed + was shut down, but `its archives remain available + `__. + + Rejection Notice ================ I'm rejecting this PEP. It's been five years now. While at some point I expect that Python will have interfaces, it would be naive -to expect it to resemble the syntax in this PEP. Also, PEP 246 is +to expect it to resemble the syntax in this PEP. Also, :pep:`246` is being rejected in favor of something completely different; interfaces won't play a role in adaptation or whatever will replace it. GvR. @@ -48,10 +59,10 @@ standard software. Zope's Interface package is used as the reference implementation for this PEP. The syntax proposed by this PEP relies on syntax enhancements -describe in PEP 232 [3]_ and describes an underlying framework -which PEP 233 [4]_ could be based upon. There is some work being +describe in :pep:`232` and describes an underlying framework +which :pep:`233` could be based upon. There is some work being done with regard to interface objects and Proxy objects, so for -those optional parts of this PEP you may want to see [5]_. +those optional parts of this PEP you may want to see [3]_. The Problem @@ -328,7 +339,7 @@ Formal Interface Syntax ======================= Python syntax is defined in a modified BNF grammar notation -described in the Python Reference Manual [8]_. This section +described in the Python Reference Manual [4]_. This section describes the proposed interface syntax using this grammar:: interfacedef: "interface" interfacename [extends] ":" suite @@ -437,8 +448,8 @@ defining ``interface`` as a new keyword will introduce. This extension to Python's syntax does not change any existing syntax in any backward incompatible way. -The new ``from __future__`` Python syntax [6]_, and the new warning -framework [7]_ is ideal for resolving this backward +The new ``from __future__`` Python syntax (:pep:`236`), and the new warning +framework (:pep:`230`) is ideal for resolving this backward incompatibility. To use interface syntax now, a developer could use the statement:: @@ -496,21 +507,9 @@ References .. [2] http://www.zope.org -.. [3] PEP 232, Function Attributes, Warsaw - http://www.python.org/dev/peps/pep-0232/ +.. [3] http://www.lemburg.com/files/python/mxProxy.html -.. [4] PEP 233, Python Online Help, Prescod - http://www.python.org/dev/peps/pep-0233/ - -.. [5] http://www.lemburg.com/files/python/mxProxy.html - -.. [6] PEP 236, Back to the __future__, Peters - http://www.python.org/dev/peps/pep-0236/ - -.. [7] PEP 230, Warning Framework, van Rossum - http://www.python.org/dev/peps/pep-0236/ - -.. [8] Python Reference Manual +.. [4] Python Reference Manual http://docs.python.org/reference/ diff --git a/pep-0246.txt b/peps/pep-0246.rst similarity index 98% rename from pep-0246.txt rename to peps/pep-0246.rst index 7609d4a66..8a10d3ba3 100644 --- a/pep-0246.txt +++ b/peps/pep-0246.rst @@ -1,9 +1,7 @@ PEP: 246 Title: Object Adaptation -Version: $Revision$ -Last-Modified: $Date$ -Author: aleaxit@gmail.com (Alex Martelli), - cce@clarkevans.com (Clark C. Evans) +Author: Alex Martelli , + Clark C. Evans Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -61,7 +59,7 @@ intended to leave this proposal compatible with both existing categories of protocols, such as the existing system of type and classes, as well as the many concepts for "interfaces" as such which have been proposed or implemented for Python, such as the -one in PEP 245 [1]_, the one in Zope3 [2]_, or the ones discussed in +one in :pep:`245`, the one in Zope3 [2]_, or the ones discussed in the BDFL's Artima blog in late 2004 and early 2005 [3]_. However, some reflections on these subjects, intended to be suggestive and not normative, are also included. @@ -737,9 +735,6 @@ adaptation of interfaces and protocols in Python. References and Footnotes ======================== -.. [1] PEP 245, Python Interface Syntax, Pelletier - http://www.python.org/dev/peps/pep-0245/ - .. [2] http://www.zope.org/Wikis/Interfaces/FrontPage .. [3] http://www.artima.com/weblogs/index.jsp?blogger=guido @@ -753,12 +748,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0247.txt b/peps/pep-0247.rst similarity index 100% rename from pep-0247.txt rename to peps/pep-0247.rst diff --git a/pep-0248.txt b/peps/pep-0248.rst similarity index 97% rename from pep-0248.txt rename to peps/pep-0248.rst index 06f11e228..887970388 100644 --- a/pep-0248.txt +++ b/peps/pep-0248.rst @@ -1,13 +1,11 @@ PEP: 248 Title: Python Database API Specification v1.0 -Version: $Revision$ -Last-Modified: $Date$ -Author: mal@lemburg.com (Marc-AndrĂ© Lemburg) +Author: Greg Stein , Marc-AndrĂ© Lemburg Discussions-To: db-sig@python.org Status: Final Type: Informational Content-Type: text/x-rst -Created: 29-Mar-2001 +Created: 08-May-1996 Post-History: Superseded-By: 249 @@ -296,18 +294,13 @@ Acknowledgements Many thanks go to Andrew Kuchling who converted the Python Database API Specification 1.0 from the original HTML format into -the PEP format. +the PEP format in 2001. +Greg Stein is the original author of the Python Database API +Specification 1.0. Marc-AndrĂ© later continued maintenance of the API as +an editor. Copyright ========= This document has been placed in the Public Domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0249.txt b/peps/pep-0249.rst similarity index 90% rename from pep-0249.txt rename to peps/pep-0249.rst index cce6a3502..d8a409cc5 100644 --- a/pep-0249.txt +++ b/peps/pep-0249.rst @@ -1,19 +1,17 @@ PEP: 249 Title: Python Database API Specification v2.0 -Version: $Revision$ -Last-Modified: $Date$ -Author: mal@lemburg.com (Marc-AndrĂ© Lemburg) +Author: Marc-AndrĂ© Lemburg Discussions-To: db-sig@python.org Status: Final Type: Informational Content-Type: text/x-rst -Created: 29-Mar-2001 +Created: 12-Apr-1999 Post-History: Replaces: 248 -`Introduction`_ -=============== +Introduction +============ This API has been defined to encourage similarity between the Python modules that are used to access databases. By doing this, we hope to @@ -35,11 +33,11 @@ encouraged to use this version of the specification as basis for new interfaces. -`Module Interface`_ -=================== +Module Interface +================= -`Constructors`_ ---------------- +Constructors +------------ Access to the database is made available through connection objects. The module must provide the following constructor for these: @@ -53,8 +51,8 @@ objects. The module must provide the following constructor for these: which are database dependent. [1]_ -`Globals`_ ----------- +Globals +------- These module globals must be defined: @@ -107,8 +105,8 @@ These module globals must be defined: ============ ============================================================== -`Exceptions`_ -------------- +Exceptions +---------- The module should make all error information available through these exceptions or subclasses thereof: @@ -118,7 +116,7 @@ exceptions or subclasses thereof: `Warning`_ Exception raised for important warnings like data truncations while inserting, etc. It must be a subclass of the Python - ``StandardError`` (defined in the module exceptions). + ``Exception`` class [10]_ [11]_. .. _Error: @@ -128,7 +126,7 @@ exceptions or subclasses thereof: exceptions. You can use this to catch all errors with one single ``except`` statement. Warnings are not considered errors and thus should not use this class as base. It must be a subclass of the - Python ``StandardError`` (defined in the module exceptions). + Python ``Exception`` class [10]_. .. _InterfaceError: @@ -199,9 +197,11 @@ exceptions or subclasses thereof: or has transactions turned off. It must be a subclass of DatabaseError_. -This is the exception inheritance layout:: +This is the exception inheritance layout [10]_ [11]_: - StandardError +.. code-block:: text + + Exception |__Warning |__Error |__InterfaceError @@ -220,14 +220,14 @@ This is the exception inheritance layout:: .. _Connection: -`Connection Objects`_ -===================== +Connection Objects +================== Connection objects should respond to the following methods. -`Connection methods`_ ---------------------- +Connection methods +------------------ .. .close(): .. _Connection.close: @@ -283,8 +283,8 @@ Connection objects should respond to the following methods. .. _Cursor: -`Cursor Objects`_ -================= +Cursor Objects +============== These objects represent a database cursor, which is used to manage the context of a fetch operation. Cursors created from the same connection @@ -297,8 +297,8 @@ transaction support is implemented (see also the connection's Cursor Objects should respond to the following methods and attributes. -`Cursor attributes`_ --------------------- +Cursor attributes +----------------- .. _.description: @@ -344,8 +344,8 @@ Cursor Objects should respond to the following methods and attributes. latter case to have the object return ``None`` instead of -1. -`Cursor methods`_ ------------------ +Cursor methods +-------------- .. _.callproc: .. _.callproc(): @@ -561,8 +561,8 @@ Cursor Objects should respond to the following methods and attributes. .. _Type Objects: -`Type Objects and Constructors`_ -================================ +Type Objects and Constructors +============================= Many databases need to have the input in a particular format for binding to an operation's input parameters. For example, if an input @@ -682,8 +682,8 @@ on input and output. .. _Implementation Hints: -`Implementation Hints for Module Authors`_ -========================================== +Implementation Hints for Module Authors +======================================= * Date/time objects can be implemented as `Python datetime module `__ objects (available @@ -731,14 +731,12 @@ on input and output. constructor. * Here is a snippet of Python code that implements the exception - hierarchy defined above:: + hierarchy defined above [10]_:: - import exceptions - - class Error(exceptions.StandardError): + class Error(Exception): pass - class Warning(exceptions.StandardError): + class Warning(Exception): pass class InterfaceError(Error): @@ -769,8 +767,8 @@ on input and output. API to create the exception objects. -`Optional DB API Extensions`_ -============================= +Optional DB API Extensions +========================== During the lifetime of DB API 2.0, module authors have often extended their implementations beyond what is required by this DB API @@ -934,8 +932,36 @@ Cursor\ `.lastrowid`_ *Warning Message:* "DB-API extension cursor.lastrowid used" -`Optional Error Handling Extensions`_ -===================================== +.. _Connection.autocommit: +.. _.autocommit: + +Connection\ `.autocommit`_ + Attribute to query and set the autocommit mode of the connection. + + Return ``True`` if the connection is operating in autocommit + (non-transactional) mode. Return ``False`` if the connection is + operating in manual commit (transactional) mode. + + Setting the attribute to ``True`` or ``False`` adjusts the + connection's mode accordingly. + + Changing the setting from ``True`` to ``False`` (disabling + autocommit) will have the database leave autocommit mode and start + a new transaction. Changing from ``False`` to ``True`` (enabling + autocommit) has database dependent semantics with respect to how + pending transactions are handled. [12]_ + + *Deprecation notice*: Even though several database modules implement + both the read and write nature of this attribute, setting the + autocommit mode by writing to the attribute is deprecated, since + this may result in I/O and related exceptions, making it difficult + to implement in an async context. [13]_ + + *Warning Message:* "DB-API extension connection.autocommit used" + + +Optional Error Handling Extensions +================================== The core DB API specification only introduces a set of exceptions which can be raised to report errors to the user. In some cases, @@ -981,8 +1007,8 @@ Cursors should inherit the ``.errorhandler`` setting from their connection objects at cursor creation time. -`Optional Two-Phase Commit Extensions`_ -======================================= +Optional Two-Phase Commit Extensions +==================================== Many databases have support for two-phase commit (TPC) which allows managing transactions across multiple database connections and other @@ -994,8 +1020,8 @@ API should be implemented. NotSupportedError_ should be raised, if the database backend support for two-phase commit can only be checked at run-time. -`TPC Transaction IDs`_ ----------------------- +TPC Transaction IDs +------------------- As many databases follow the XA specification, transaction IDs are formed from three components: @@ -1034,8 +1060,8 @@ Transaction IDs are created with the `.xid()`_ Connection method: represent transaction IDs with tuples rather than a custom object. -`TPC Connection Methods`_ -------------------------- +TPC Connection Methods +---------------------- .. _.tpc_*: .. _.tpc_*(): @@ -1117,8 +1143,8 @@ Transaction IDs are created with the `.xid()`_ Connection method: -`Frequently Asked Questions`_ -============================= +Frequently Asked Questions +========================== The database SIG often sees reoccurring questions about the DB API specification. This section covers some of the issues people sometimes @@ -1153,8 +1179,8 @@ between databases and makes writing portable code impossible. -`Major Changes from Version 1.0 to Version 2.0`_ -================================================ +Major Changes from Version 1.0 to Version 2.0 +============================================= The Python Database API 2.0 introduces a few major changes compared to the 1.0 version. Because some of these changes will cause existing DB @@ -1197,8 +1223,8 @@ Post-publishing additions to the DB API 2.0 specification: functionality were specified. -`Open Issues`_ -============== +Open Issues +=========== Although the version 2.0 specification clarifies a lot of questions that were left open in the 1.0 version, there are still some remaining @@ -1213,8 +1239,8 @@ issues which should be addressed in future versions: -`Footnotes`_ -============ +Footnotes +========= .. [1] As a guideline the connection constructor parameters should be implemented as keyword parameters for more intuitive use and @@ -1234,6 +1260,8 @@ issues which should be addressed in future versions: connect(dsn='myhost:MYDB', user='guido', password='234$') + Also see [13]_ regarding planned future additions to this list. + .. [2] Module implementors should prefer ``numeric``, ``named`` or ``pyformat`` over the other formats because these offer more clarity and flexibility. @@ -1243,10 +1271,10 @@ issues which should be addressed in future versions: the method, the interface should throw an exception in case the method is used. - The preferred approach is to not implement the method and thus - have Python generate an ``AttributeError`` in case the method is requested. This - allows the programmer to check for database capabilities using the - standard ``hasattr()`` function. + The preferred approach is to not implement the method and thus have + Python generate an ``AttributeError`` in case the method is + requested. This allows the programmer to check for database + capabilities using the standard ``hasattr()`` function. For some dynamically configured interfaces it may not be appropriate to require dynamically making the method @@ -1297,22 +1325,49 @@ issues which should be addressed in future versions: ``WHERE`` clause, or clearly document a different interpretation of the ``.rowcount`` attribute. +.. [10] In Python 2 and earlier versions of this PEP, ``StandardError`` + was used as the base class for all DB-API exceptions. Since + ``StandardError`` was removed in Python 3, database modules + targeting Python 3 should use ``Exception`` as base class instead. + The PEP was updated to use ``Exception`` throughout the text, to + avoid confusion. The change should not affect existing modules or + uses of those modules, since all DB-API error exception classes are + still rooted at the ``Error`` or ``Warning`` classes. -`Acknowledgements`_ -=================== +.. [11] In a future revision of the DB-API, the base class for + ``Warning`` will likely change to the builtin ``Warning`` class. At + the time of writing of the DB-API 2.0 in 1999, the warning framework + in Python did not yet exist. + +.. [12] Many database modules implementing the autocommit attribute will + automatically commit any pending transaction and then enter + autocommit mode. It is generally recommended to explicitly + `.commit()`_ or `.rollback()`_ transactions prior to changing the + autocommit setting, since this is portable across database modules. + +.. [13] In a future revision of the DB-API, we are going to introduce a + new method ``.setautocommit(value)``, which will allow setting the + autocommit mode, and make ``.autocommit`` a read-only attribute. + Additionally, we are considering to add a new standard keyword + parameter ``autocommit`` to the Connection constructor. Modules + authors are encouraged to add these changes in preparation for this + change. + +Acknowledgements +================ Many thanks go to Andrew Kuchling who converted the Python Database API Specification 2.0 from the original HTML format into the PEP -format. +format in 2001. -Many thanks to James Henstridge for leading the discussion which led -to the standardization of the two-phase commit API extensions. +Many thanks to James Henstridge for leading the discussion which led to +the standardization of the two-phase commit API extensions in 2008. Many thanks to Daniele Varrazzo for converting the specification from text PEP format to ReST PEP format, which allows linking to various -parts. +parts in 2012. -`Copyright`_ -============ +Copyright +========= This document has been placed in the Public Domain. diff --git a/pep-0250.txt b/peps/pep-0250.rst similarity index 96% rename from pep-0250.txt rename to peps/pep-0250.rst index 688c5ee41..d6fdc1d56 100644 --- a/pep-0250.txt +++ b/peps/pep-0250.rst @@ -1,8 +1,6 @@ PEP: 250 Title: Using site-packages on Windows -Version: $Revision$ -Last-Modified: $Date$ -Author: p.f.moore@gmail.com (Paul Moore) +Author: Paul Moore Status: Final Type: Standards Track Content-Type: text/x-rst @@ -136,11 +134,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0251.txt b/peps/pep-0251.rst similarity index 80% rename from pep-0251.txt rename to peps/pep-0251.rst index c7de8ad4f..89302cc1d 100644 --- a/pep-0251.txt +++ b/peps/pep-0251.rst @@ -1,10 +1,9 @@ PEP: 251 Title: Python 2.2 Release Schedule -Version: $Revision$ -Last-Modified: $Date$ -Author: barry@python.org (Barry Warsaw), guido@python.org (Guido van Rossum) +Author: Barry Warsaw , Guido van Rossum Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 17-Apr-2001 Python-Version: 2.2 @@ -54,7 +53,7 @@ every alpha, beta or other release, we forked off a branch which became the release. Changes to the branch are limited to the release manager and his designated 'bots. This experiment was deemed a success and should be observed for future releases. See -PEP 101 for the actual release mechanics [1]_. +:pep:`101` for the actual release mechanics. New features for Python 2.2 @@ -65,19 +64,16 @@ more detailed account, see Misc/NEWS [2]_ in the Python distribution, or Andrew Kuchling's "What's New in Python 2.2" document [3]_. -- iterators (PEP 234) -- generators (PEP 255) -- unifying long ints and plain ints (PEP 237) -- division (PEP 238) -- unification of types and classes (PEP 252, PEP 253) +- iterators (:pep:`234`) +- generators (:pep:`255`) +- unifying long ints and plain ints (:pep:`237`) +- division (:pep:`238`) +- unification of types and classes (:pep:`252`, :pep:`253`) References ========== -.. [1] PEP 101, Doing Python Releases 101 - http://www.python.org/dev/peps/pep-0101/ - .. [2] Misc/NEWS file from CVS http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/python/python/dist/src/Misc/NEWS?rev=1.337.2.4&content-type=text/vnd.viewcvs-markup @@ -89,11 +85,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0252.txt b/peps/pep-0252.rst similarity index 98% rename from pep-0252.txt rename to peps/pep-0252.rst index ed250ec03..671d3736e 100644 --- a/pep-0252.txt +++ b/peps/pep-0252.rst @@ -1,8 +1,6 @@ PEP: 252 Title: Making Types Look More Like Classes -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Final Type: Standards Track Content-Type: text/x-rst @@ -84,7 +82,7 @@ are: (The last two rules together are often summarized as the left-to-right, depth-first rule for attribute search. This is the -classic Python attribute lookup rule. Note that PEP 253 will +classic Python attribute lookup rule. Note that :pep:`253` will propose to change the attribute lookup order, and if accepted, this PEP will follow suit.) @@ -257,7 +255,7 @@ specific attribute of a given object. orders. In particular, classic classes use the old left-to-right depth-first rule, while new-style classes use a more advanced rule (see the section on method resolution order - in PEP 253). + in :pep:`253`). When a dynamic attribute (one defined in a regular object's ``__dict__``) has the same name as a static attribute (one defined @@ -466,7 +464,7 @@ explicit first argument. (If you don't understand this, don't worry, you're not alone.) Note that calling ``cls.foo(y)`` would be a mistake -- it would cause infinite recursion. Also note that you can't specify an explicit 'cls' argument to a class method. If -you want this (e.g. the ``__new__`` method in PEP 253 requires this), +you want this (e.g. the ``__new__`` method in :pep:`253` requires this), use a static method with a class as its explicit first argument instead. @@ -530,7 +528,7 @@ this situation substantially. field tells what kind of descriptor it is (method, member, or getset). -- As explained in PEP 252, descriptors have a ``get()`` method that +- As explained in :pep:`252`, descriptors have a ``get()`` method that takes an object argument and returns that object's attribute; descriptors for writable attributes also have a ``set()`` method that takes an object and a value and set that object's @@ -683,7 +681,7 @@ You can invoke any method from this list directly:: This is just like it is for user-defined classes. Notice a familiar yet surprising name in the list: ``__init__``. This -is the domain of PEP 253. +is the domain of :pep:`253`. Backwards compatibility @@ -711,7 +709,7 @@ add the arguments "-r descr-branch" to the cvs checkout command. here, see the file Lib/test/test_descr.py. Note: the code in this branch goes way beyond this PEP; it is also -the experimentation area for PEP 253 (Subtyping Built-in Types). +the experimentation area for :pep:`253` (Subtyping Built-in Types). References @@ -724,9 +722,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0253.txt b/peps/pep-0253.rst similarity index 98% rename from pep-0253.txt rename to peps/pep-0253.rst index a3fb28e5e..cf165a543 100644 --- a/pep-0253.txt +++ b/peps/pep-0253.rst @@ -1,8 +1,6 @@ PEP: 253 Title: Subtyping Built-in Types -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Final Type: Standards Track Content-Type: text/x-rst @@ -67,10 +65,10 @@ This PEP will introduce the following features: specifying the specific names of the instance variables supported -This PEP builds on PEP 252, which adds standard introspection to +This PEP builds on :pep:`252`, which adds standard introspection to types; for example, when a particular type object initializes the ``tp_hash`` slot, that type object has a ``__hash__`` method when -introspected. PEP 252 also adds a dictionary to type objects +introspected. :pep:`252` also adds a dictionary to type objects which contains all methods. At the Python level, this dictionary is read-only for built-in types; at the C level, it is accessible directly (but it should not be modified except as part of @@ -86,9 +84,9 @@ checking of this flag bit. This should be fixed before the final release.) In current Python, a distinction is made between types and -classes. This PEP together with PEP 254 will remove that +classes. This PEP together with :pep:`254` will remove that distinction. However, for backwards compatibility the distinction -will probably remain for years to come, and without PEP 254, the +will probably remain for years to come, and without :pep:`254`, the distinction is still large: types ultimately have a built-in type as a base class, while classes ultimately derive from a user-defined class. Therefore, in the rest of this PEP, I will @@ -550,7 +548,7 @@ Assume B is a type object. Since type objects are objects, and every object has a type, B has a type. Since B is itself a type, we also call its type its metatype. B's metatype is accessible via ``type(B)`` or ``B.__class__`` (the latter notation is new for types; -it is introduced in PEP 252). Let's say this metatype is M (for +it is introduced in :pep:`252`). Let's say this metatype is M (for Metatype). The class statement will create a new type, C. Since C will be a type object just like B, we view the creation of C as an instantiation of the metatype, M. The information that needs @@ -902,7 +900,7 @@ Additional topics to be discussed in this PEP: - cooperative methods and ``super()`` - mapping between type object slots (tp_foo) and special methods - (``__foo__``) (actually, this may belong in PEP 252) + (``__foo__``) (actually, this may belong in :pep:`252`) - built-in names for built-in types (object, int, str, list etc.) @@ -942,7 +940,7 @@ open issues Implementation ============== -A prototype implementation of this PEP (and for PEP 252) is +A prototype implementation of this PEP (and for :pep:`252`) is available from CVS, and in the series of Python 2.2 alpha and beta releases. For some examples of the features described here, see the file Lib/test/test_descr.py and the extension module @@ -961,9 +959,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0254.txt b/peps/pep-0254.rst similarity index 76% rename from pep-0254.txt rename to peps/pep-0254.rst index f769a5b85..0d1f64081 100644 --- a/pep-0254.txt +++ b/peps/pep-0254.rst @@ -1,8 +1,6 @@ PEP: 254 Title: Making Classes Look More Like Types -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -29,10 +27,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0255.txt b/peps/pep-0255.rst similarity index 91% rename from pep-0255.txt rename to peps/pep-0255.rst index 9add0610e..104935fcd 100644 --- a/pep-0255.txt +++ b/peps/pep-0255.rst @@ -1,11 +1,8 @@ PEP: 255 Title: Simple Generators -Version: $Revision$ -Last-Modified: $Date$ -Author: nas@arctrix.com (Neil Schemenauer), - tim.peters@gmail.com (Tim Peters), - magnus@hetland.org (Magnus Lie Hetland) -Discussions-To: python-iterators@lists.sourceforge.net +Author: Neil Schemenauer , + Tim Peters , + Magnus Lie Hetland Status: Final Type: Standards Track Content-Type: text/x-rst @@ -51,7 +48,8 @@ only want to see whether something specific appears early in the program (e.g., a future statement, or, as is done in IDLE, just the first indented statement), and then parsing the whole program first is a severe waste of time. -Another alternative would be to make tokenize an iterator [1], delivering the +Another alternative would be to make tokenize an :pep:`iterator <234>`, +delivering the next token whenever its ``.next()`` method is invoked. This is pleasant for the caller in the same way a large list of results would be, but without the memory and "what if I want to get out early?" drawbacks. However, this shifts the @@ -69,7 +67,7 @@ provides a usable synchronized-communication class for doing that in a general way. This doesn't work on platforms without threads, though, and is very slow on platforms that do (compared to what is achievable without threads). -A final option is to use the Stackless [2] [3] variant implementation of Python +A final option is to use the Stackless [1]_ (:pep:`219`) variant implementation of Python instead, which supports lightweight coroutines. This has much the same programmatic benefits as the thread option, but is much more efficient. However, Stackless is a controversial rethinking of the Python core, and it may @@ -80,8 +78,8 @@ current CPython implementation, and is believed to be relatively straightforward for other Python implementations. That exhausts the current alternatives. Some other high-level languages -provide pleasant solutions, notably iterators in Sather [4], which were -inspired by iterators in CLU; and generators in Icon [5], a novel language +provide pleasant solutions, notably iterators in Sather [2]_, which were +inspired by iterators in CLU; and generators in Icon [3]_, a novel language where every expression *is a generator*. There are differences among these, but the basic idea is the same: provide a kind of function that can return an intermediate result ("the next value") to its caller, but maintaining the @@ -111,7 +109,7 @@ The same kind of approach applies to many producer/consumer functions. For example, ``tokenize.py`` could yield the next token instead of invoking a callback function with it as argument, and tokenize clients could iterate over the tokens in a natural way: a Python generator is a kind of Python -iterator [1]_, but of an especially powerful kind. +:pep:`iterator <234>`, but of an especially powerful kind. Specification: Yield @@ -121,13 +119,13 @@ A new statement is introduced:: yield_stmt: "yield" expression_list -``yield`` is a new keyword, so a ``future`` statement [8]_ is needed to phase +``yield`` is a new keyword, so a ``future`` statement (:pep:`236`) is needed to phase this in: in the initial release, a module desiring to use generators must include the line:: from __future__ import generators -near the top (see PEP 236 [8]_) for details). Modules using the identifier +near the top (see :pep:`236`) for details). Modules using the identifier ``yield`` without a ``future`` statement will trigger warnings. In the following release, ``yield`` will be a language keyword and the ``future`` statement will no longer be needed. @@ -140,7 +138,7 @@ function is an ordinary function object in all respects, but has the new When a generator function is called, the actual arguments are bound to function-local formal argument names in the usual way, but no code in the body of the function is executed. Instead a generator-iterator object is returned; -this conforms to the iterator protocol [6]_, so in particular can be used in +this conforms to the :pep:`iterator protocol <234>`, so in particular can be used in for-loops in a natural way. Note that when the intent is clear from context, the unqualified name "generator" may be used to refer either to a generator-function or a generator-iterator. @@ -474,37 +472,26 @@ Reference Implementation ======================== The current implementation, in a preliminary state (no docs, but well tested -and solid), is part of Python's CVS development tree [9]_. Using this requires +and solid), is part of Python's CVS development tree [5]_. Using this requires that you build Python from source. -This was derived from an earlier patch by Neil Schemenauer [7]_. +This was derived from an earlier patch by Neil Schemenauer [4]_. Footnotes and References ======================== -.. [1] PEP 234, Iterators, Yee, Van Rossum - http://www.python.org/dev/peps/pep-0234/ +.. [1] http://www.stackless.com/ -.. [2] http://www.stackless.com/ - -.. [3] PEP 219, Stackless Python, McMillan - http://www.python.org/dev/peps/pep-0219/ - -.. [4] "Iteration Abstraction in Sather" +.. [2] "Iteration Abstraction in Sather" Murer, Omohundro, Stoutamire and Szyperski http://www.icsi.berkeley.edu/~sather/Publications/toplas.html -.. [5] http://www.cs.arizona.edu/icon/ +.. [3] http://www.cs.arizona.edu/icon/ -.. [6] The concept of iterators is described in PEP 234. See [1] above. +.. [4] http://python.ca/nas/python/generator.diff -.. [7] http://python.ca/nas/python/generator.diff - -.. [8] PEP 236, Back to the __future__, Peters - http://www.python.org/dev/peps/pep-0236/ - -.. [9] To experiment with this implementation, check out Python from CVS +.. [5] To experiment with this implementation, check out Python from CVS according to the instructions at http://sf.net/cvs/?group_id=5470 Note that the std test ``Lib/test/test_generators.py`` contains many examples, including all those in this PEP. @@ -514,11 +501,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0256.txt b/peps/pep-0256.rst similarity index 90% rename from pep-0256.txt rename to peps/pep-0256.rst index ccb5b715e..9e9a66e27 100644 --- a/pep-0256.txt +++ b/peps/pep-0256.rst @@ -3,7 +3,7 @@ Title: Docstring Processing System Framework Version: $Revision$ Last-Modified: $Date$ Author: David Goodger -Discussions-To: +Discussions-To: doc-sig@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -48,27 +48,27 @@ have broken up the issues in order to deal with each of them in isolation, or as close as possible. The individual aspects and associated PEPs are as follows: -* Docstring syntax. PEP 287, "reStructuredText Docstring Format" - [#PEP-287]_, proposes a syntax for Python docstrings, PEPs, and +* Docstring syntax. :pep:`287`, "reStructuredText Docstring Format", + proposes a syntax for Python docstrings, PEPs, and other uses. * Docstring semantics consist of at least two aspects: - Conventions: the high-level structure of docstrings. Dealt with - in PEP 257, "Docstring Conventions" [#PEP-257]_. + in :pep:`257`, "Docstring Conventions". - Methodology: rules for the informational content of docstrings. Not addressed. -* Processing mechanisms. This PEP (PEP 256) outlines the high-level +* Processing mechanisms. This PEP (:pep:`256`) outlines the high-level issues and specification of an abstract docstring processing system - (DPS). PEP 258, "Docutils Design Specification" [#PEP-258]_, is an + (DPS). :pep:`258`, "Docutils Design Specification", is an overview of the design and implementation of one DPS under development. * Output styles: developers want the documentation generated from their source code to look good, and there are many different ideas - about what that means. PEP 258 touches on "Stylist Transforms". + about what that means. :pep:`258` touches on "Stylist Transforms". This aspect of docstring processing has yet to be fully explored. By separating out the issues, we can form consensus more easily @@ -149,7 +149,7 @@ deficiencies, including: There are security issues involved with importing untrusted code. Also, information from the source is lost when importing, such as comments, "additional docstrings" (string literals in non-docstring - contexts; see PEP 258 [#PEP-258]_), and the order of definitions. + contexts; see :pep:`258`), and the order of definitions. The functionality proposed in this PEP could be added to or used by PyDoc when serving HTML pages. The proposed docstring processing @@ -177,7 +177,7 @@ The docstring processing system framework is broken up as follows: - First line is a one-line synopsis. - PEP 257 [#PEP-257]_ documents some of these issues. + :pep:`257` documents some of these issues. 2. Docstring processing system design specification. Documents issues such as: @@ -206,11 +206,11 @@ The docstring processing system framework is broken up as follows: files, or objects in memory). These issues are applicable to any docstring processing system - implementation. PEP 258 [#PEP-258]_ documents these issues. + implementation. :pep:`258` documents these issues. 3. Docstring processing system implementation. -4. Input markup specifications: docstring syntax. PEP 287 [#PEP-287]_ +4. Input markup specifications: docstring syntax. :pep:`287` proposes a standard syntax. 5. Input parser implementations. @@ -241,15 +241,6 @@ http://docutils.sourceforge.net/. References and Footnotes ======================== -.. [#PEP-287] PEP 287, reStructuredText Docstring Format, Goodger - (http://www.python.org/dev/peps/pep-0287/) - -.. [#PEP-257] PEP 257, Docstring Conventions, Goodger, Van Rossum - (http://www.python.org/dev/peps/pep-0257/) - -.. [#PEP-258] PEP 258, Docutils Design Specification, Goodger - (http://www.python.org/dev/peps/pep-0258/) - .. _Literate Programming: http://www.literateprogramming.com/ .. _POD: http://www.perldoc.com/perl5.6/pod/perlpod.html diff --git a/pep-0257.txt b/peps/pep-0257.rst similarity index 94% rename from pep-0257.txt rename to peps/pep-0257.rst index c70aaa359..0074cd05b 100644 --- a/pep-0257.txt +++ b/peps/pep-0257.rst @@ -36,7 +36,7 @@ conventions, not laws or syntax. If you violate these conventions, the worst you'll get is some dirty looks. But some software (such as the Docutils_ docstring processing -system [1]_ [2]_) will be aware of the conventions, so following them +system :pep:`256`, :pep:`258`) will be aware of the conventions, so following them will get you the best results. @@ -69,7 +69,7 @@ extracted by software tools: 2. String literals occurring immediately after another docstring are called "additional docstrings". -Please see PEP 258, "Docutils Design Specification" [2]_, for a +Please see :pep:`258`, "Docutils Design Specification", for a detailed description of attribute and additional docstrings. For consistency, always use ``"""triple double quotes"""`` around @@ -224,14 +224,14 @@ of the algorithm:: # and split into a list of lines: lines = docstring.expandtabs().splitlines() # Determine minimum indentation (first line doesn't count): - indent = sys.maxint + indent = sys.maxsize for line in lines[1:]: stripped = line.lstrip() if stripped: indent = min(indent, len(line) - len(stripped)) # Remove indentation (first line is special): trimmed = [lines[0].strip()] - if indent < sys.maxint: + if indent < sys.maxsize: for line in lines[1:]: trimmed.append(line[indent:].rstrip()) # Strip off trailing and leading blank lines: @@ -276,17 +276,8 @@ Once trimmed, these docstrings are equivalent:: References and Footnotes ======================== -.. [1] PEP 256, Docstring Processing System Framework, Goodger - (http://www.python.org/dev/peps/pep-0256/) - -.. [2] PEP 258, Docutils Design Specification, Goodger - (http://www.python.org/dev/peps/pep-0258/) - .. _Docutils: http://docutils.sourceforge.net/ -.. _Python Style Guide: - http://www.python.org/dev/peps/pep-0008/ - .. _Doc-SIG: http://www.python.org/sigs/doc-sig/ @@ -299,8 +290,8 @@ This document has been placed in the public domain. Acknowledgements ================ -The "Specification" text comes mostly verbatim from the `Python Style -Guide`_ essay by Guido van Rossum. +The "Specification" text comes mostly verbatim from :pep:`8` +by Guido van Rossum. This document borrows ideas from the archives of the Python Doc-SIG_. Thanks to all members past and present. diff --git a/pep-0258.txt b/peps/pep-0258.rst similarity index 97% rename from pep-0258.txt rename to peps/pep-0258.rst index c6dc60099..4f793a016 100644 --- a/pep-0258.txt +++ b/peps/pep-0258.rst @@ -3,7 +3,7 @@ Title: Docutils Design Specification Version: $Revision$ Last-Modified: $Date$ Author: David Goodger -Discussions-To: +Discussions-To: doc-sig@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -27,8 +27,8 @@ standard library. This PEP documents design issues and implementation details for Docutils, a Python Docstring Processing System (DPS). The rationale -and high-level concepts of a DPS are documented in PEP 256, "Docstring -Processing System Framework" [#PEP-256]_. Also see PEP 256 for a +and high-level concepts of a DPS are documented in :pep:`256`, "Docstring +Processing System Framework". Also see :pep:`256` for a "Road Map to the Docstring PEPs". Docutils is being designed modularly so that any of its components can @@ -142,11 +142,11 @@ Examples: * Python Source: See `Python Source Reader`_ below. This Reader is currently in development in the Docutils sandbox. -* Email: RFC-822 headers, quoted excerpts, signatures, MIME parts. +* Email: :rfc:`822` headers, quoted excerpts, signatures, MIME parts. -* PEP: RFC-822 headers, "PEP xxxx" and "RFC xxxx" conversion to URIs. +* PEP: :rfc:`822` headers, "PEP xxxx" and "RFC xxxx" conversion to URIs. The "PEP Reader" has been implemented in module - ``docutils.readers.pep``; see PEP 287 and PEP 12. + ``docutils.readers.pep``; see :pep:`287` and :pep:`12`. * Wiki: Global reference lookups of "wiki links" incorporated into transforms. (CamelCase only or unrestricted?) Lazy @@ -711,7 +711,7 @@ slight performance hit. Attribute Docstrings '''''''''''''''''''' -(This is a simplified version of PEP 224 [#PEP-224]_.) +(This is a simplified version of :pep:`224`.) A string literal immediately following an assignment statement is interpreted by the docstring extraction machinery as the docstring of @@ -787,7 +787,7 @@ Examples:: Additional Docstrings ''''''''''''''''''''' -(This idea was adapted from PEP 216 [#PEP-216]_.) +(This idea was adapted from :pep:`216`.) Many programmers would like to make extensive use of docstrings for API documentation. However, docstrings do take up space in the @@ -865,10 +865,10 @@ established. The ``__docformat__`` string may contain an optional second field, separated from the format name (first field) by a single space: a -case-insensitive language identifier as defined in RFC 1766. A +case-insensitive language identifier as defined in :rfc:`1766`. A typical language identifier consists of a 2-letter language code from -`ISO 639`_ (3-letter codes used only if no 2-letter code exists; RFC -1766 is currently being revised to allow 3-letter codes). If no +`ISO 639`_ (3-letter codes used only if no 2-letter code exists; +:rfc:`1766` is currently being revised to allow 3-letter codes). If no language identifier is specified, the default is "en" for English. The language identifier is passed to the parser and can be used for language-dependent markup features. @@ -950,15 +950,6 @@ stylist code will lower the barrier considerably. References and Footnotes ========================== -.. [#PEP-256] PEP 256, Docstring Processing System Framework, Goodger - (http://www.python.org/dev/peps/pep-0256/) - -.. [#PEP-224] PEP 224, Attribute Docstrings, Lemburg - (http://www.python.org/dev/peps/pep-0224/) - -.. [#PEP-216] PEP 216, Docstring Format, Zadka - (http://www.python.org/dev/peps/pep-0216/) - .. _docutils.dtd: http://docutils.sourceforge.net/docs/ref/docutils.dtd diff --git a/pep-0259.txt b/peps/pep-0259.rst similarity index 95% rename from pep-0259.txt rename to peps/pep-0259.rst index a7c0e6c12..1041acd9c 100644 --- a/pep-0259.txt +++ b/peps/pep-0259.rst @@ -1,8 +1,6 @@ PEP: 259 Title: Omit printing newline after newline -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -135,10 +133,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0260.txt b/peps/pep-0260.rst similarity index 92% rename from pep-0260.txt rename to peps/pep-0260.rst index 9542bd20d..06b018b79 100644 --- a/pep-0260.txt +++ b/peps/pep-0260.rst @@ -1,8 +1,6 @@ PEP: 260 Title: Simplify xrange() -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Final Type: Standards Track Content-Type: text/x-rst @@ -86,10 +84,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0261.txt b/peps/pep-0261.rst similarity index 99% rename from pep-0261.txt rename to peps/pep-0261.rst index 1614364e1..14ef780d4 100644 --- a/pep-0261.txt +++ b/peps/pep-0261.rst @@ -201,7 +201,7 @@ There is a new configure option: ===================== ============================================ --enable-unicode=ucs2 configures a narrow ``Py_UNICODE``, and uses wchar_t if it fits ---enable-unicode=ucs4 configures a wide `Py_UNICODE``, and uses +--enable-unicode=ucs4 configures a wide ``Py_UNICODE``, and uses wchar_t if it fits --enable-unicode same as "=ucs2" --disable-unicode entirely remove the Unicode functionality. diff --git a/pep-0262.txt b/peps/pep-0262.rst similarity index 96% rename from pep-0262.txt rename to peps/pep-0262.rst index d427e80c3..90c89d189 100644 --- a/pep-0262.txt +++ b/peps/pep-0262.rst @@ -1,14 +1,19 @@ PEP: 262 Title: A Database of Installed Python Packages -Version: $Revision$ -Last-Modified: $Date$ Author: A.M. Kuchling -Status: Deferred +Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 08-Jul-2001 Post-History: 27-Mar-2002 + +.. note:: + This PEP was superseded by :pep:`345` and :pep:`376`, which were accepted. + Therefore, this PEP is (by implication) rejected. + + Introduction ============ @@ -88,8 +93,8 @@ Each section of the file is used for a different purpose. PKG-INFO section ---------------- -An initial set of RFC-822 headers containing the distribution -information for a file, as described in PEP 241, "Metadata for +An initial set of :rfc:`822` headers containing the distribution +information for a file, as described in :pep:`241`, "Metadata for Python Software Packages". FILES section @@ -314,8 +319,8 @@ for Windows. References ========== -.. [1] Michael Muller's patch (posted to the Distutils-SIG around 28 - Dec 1999) generates a list of installed files. +[1] Michael Muller's patch (posted to the Distutils-SIG around 28 +\ Dec 1999) generates a list of installed files. .. [2] A patch to implement this PEP will be tracked as patch #562100 on SourceForge. @@ -339,9 +344,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0263.txt b/peps/pep-0263.rst similarity index 96% rename from pep-0263.txt rename to peps/pep-0263.rst index 7aff27ffd..f55230676 100644 --- a/pep-0263.txt +++ b/peps/pep-0263.rst @@ -1,9 +1,7 @@ PEP: 263 Title: Defining Python Source Code Encodings -Version: $Revision$ -Last-Modified: $Date$ -Author: mal@lemburg.com (Marc-AndrĂ© Lemburg), - martin@v.loewis.de (Martin von Löwis) +Author: Marc-AndrĂ© Lemburg , + Martin von Löwis Status: Final Type: Standards Track Content-Type: text/x-rst @@ -282,13 +280,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0264.txt b/peps/pep-0264.rst similarity index 97% rename from pep-0264.txt rename to peps/pep-0264.rst index 77d94d26e..bb53e97a5 100644 --- a/pep-0264.txt +++ b/peps/pep-0264.rst @@ -15,13 +15,13 @@ Post-History: 30-Jul-2001 Abstract ======== -As noted in PEP 236, there is no clear way for "simulated +As noted in :pep:`236`, there is no clear way for "simulated interactive shells" to simulate the behaviour of ``__future__`` statements in "real" interactive shells, i.e. have ``__future__`` statements' effects last the life of the shell. The PEP also takes the opportunity to clean up the other -unresolved issue mentioned in PEP 236, the inability to stop +unresolved issue mentioned in :pep:`236`, the inability to stop ``compile()`` inheriting the effect of future statements affecting the code calling ``compile()``. diff --git a/pep-0265.txt b/peps/pep-0265.rst similarity index 97% rename from pep-0265.txt rename to peps/pep-0265.rst index 5039a8916..af4a55841 100644 --- a/pep-0265.txt +++ b/peps/pep-0265.rst @@ -1,8 +1,6 @@ PEP: 265 Title: Sorting Dictionaries by Value -Version: $Revision$ -Last-Modified: $Date$ -Author: g2@iowegian.com (Grant Griffin) +Author: Grant Griffin Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -207,11 +205,3 @@ Copyright This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: - diff --git a/pep-0266.txt b/peps/pep-0266.rst similarity index 99% rename from pep-0266.txt rename to peps/pep-0266.rst index 7e4901437..a57911f4b 100644 --- a/pep-0266.txt +++ b/peps/pep-0266.rst @@ -1,8 +1,6 @@ PEP: 266 Title: Optimizing Global Variable/Attribute Access -Version: $Revision$ -Last-Modified: $Date$ -Author: skip@pobox.com (Skip Montanaro) +Author: Skip Montanaro Status: Withdrawn Type: Standards Track Content-Type: text/x-rst @@ -433,12 +431,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0267.txt b/peps/pep-0267.rst similarity index 97% rename from pep-0267.txt rename to peps/pep-0267.rst index 364eb0c1d..32bdb3a32 100644 --- a/pep-0267.txt +++ b/peps/pep-0267.rst @@ -1,8 +1,6 @@ PEP: 267 Title: Optimized Access to Module Namespaces -Version: $Revision$ -Last-Modified: $Date$ -Author: jeremy@alum.mit.edu (Jeremy Hylton) +Author: Jeremy Hylton Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -15,7 +13,7 @@ Deferral ======== While this PEP is a nice idea, no-one has yet emerged to do the work of -hashing out the differences between this PEP, PEP 266 and PEP 280. +hashing out the differences between this PEP, :pep:`266` and :pep:`280`. Hence, it is being deferred. @@ -258,7 +256,7 @@ implementation could provide warnings. Related PEPs ============ -PEP 266, Optimizing Global Variable/Attribute Access, proposes a +:pep:`266`, Optimizing Global Variable/Attribute Access, proposes a different mechanism for optimizing access to global variables as well as attributes of objects. The mechanism uses two new opcodes ``TRACK_OBJECT`` and ``UNTRACK_OBJECT`` to create a slot in the local @@ -286,11 +284,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0268.txt b/peps/pep-0268.rst similarity index 94% rename from pep-0268.txt rename to peps/pep-0268.rst index a27d7e517..5d1b9ecee 100644 --- a/pep-0268.txt +++ b/peps/pep-0268.rst @@ -1,8 +1,6 @@ PEP: 268 Title: Extended HTTP functionality and WebDAV -Version: $Revision$ -Last-Modified: $Date$ -Author: gstein@lyra.org (Greg Stein) +Author: Greg Stein Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -30,8 +28,8 @@ Rationale ========= Python has been quite popular as a result of its "batteries included" -positioning. One of the most heavily used protocols, HTTP (see RFC -2616), has been included with Python for years (``httplib``). However, +positioning. One of the most heavily used protocols, HTTP (see +:rfc:`2616`), has been included with Python for years (``httplib``). However, this support has not kept up with the full needs and requirements of many HTTP-based applications and systems. In addition, new protocols based on HTTP, such as WebDAV and XML-RPC, are becoming useful and are @@ -79,7 +77,7 @@ The mixin will delegate the authentication process to one or more "authenticator" objects, allowing multiple connections to share authenticators. The use of a separate object allows for a long term connection to an authentication system (e.g. LDAP). An authenticator -for the Basic and Digest mechanisms (see RFC 2617) will be +for the Basic and Digest mechanisms (see :rfc:`2617`) will be provided. User-supplied authenticator subclasses can be registered and used by the connections. @@ -118,7 +116,7 @@ cached (into the Credentials object; see below), the caller can simply regenerate the request. The mixin will attach the appropriate credentials. -A "protection space" (see RFC 2617, section 1.2) is defined as a tuple +A "protection space" (see :rfc:`2617`, section 1.2) is defined as a tuple of the host, port, and authentication realm. When a request is initially sent to an HTTP server, we do not know the authentication realm (the realm is only returned when authentication fails). However, @@ -180,7 +178,7 @@ classes (the mixin may possibly work with the HTTP and HTTPS compatibility classes, but that is not a requirement). The mixin provides methods to perform the various HTTP methods defined -by HTTP in RFC 2616, and by WebDAV in RFC 2518. +by HTTP in :rfc:`2616`, and by WebDAV in :rfc:`2518`. A custom response object is used to decode ``207 (Multi-Status)`` responses. The response object will use the standard library's xml @@ -207,13 +205,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - sentence-end-double-space: t - End: diff --git a/pep-0269.txt b/peps/pep-0269.rst similarity index 95% rename from pep-0269.txt rename to peps/pep-0269.rst index 10ddf6463..cd2f42e28 100644 --- a/pep-0269.txt +++ b/peps/pep-0269.rst @@ -1,8 +1,6 @@ PEP: 269 Title: Pgen Module for Python -Version: $Revision$ -Last-Modified: $Date$ -Author: jriehl@spaceship.com (Jonathan Riehl) +Author: Jonathan Riehl Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -198,8 +196,7 @@ References .. [3] Hylton, Jeremy. http://docs.python.org/library/compiler.html -.. [4] Pelletier, Michel. "Python Interface Syntax", PEP-245. - http://www.python.org/dev/peps/pep-0245/ +.. [4] Pelletier, Michel. "Python Interface Syntax", :pep:`245` .. [5] The Python Types-SIG http://www.python.org/sigs/types-sig/ @@ -209,12 +206,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0270.txt b/peps/pep-0270.rst similarity index 89% rename from pep-0270.txt rename to peps/pep-0270.rst index ad8315718..2d1ec09ff 100644 --- a/pep-0270.txt +++ b/peps/pep-0270.rst @@ -1,8 +1,6 @@ PEP: 270 Title: uniq method for list objects -Version: $Revision$ -Last-Modified: $Date$ -Author: jp@demonseed.net (Jason Petrone) +Author: Jason Petrone Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -25,7 +23,7 @@ This PEP is withdrawn by the author. He writes: a matter of choosing a different data structure: a set instead of a list. -As described in PEP 218, sets are being added to the standard +As described in :pep:`218`, sets are being added to the standard library for Python 2.3. @@ -81,12 +79,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0271.txt b/peps/pep-0271.rst similarity index 85% rename from pep-0271.txt rename to peps/pep-0271.rst index 0cfff4172..c6dcf8f8c 100644 --- a/pep-0271.txt +++ b/peps/pep-0271.rst @@ -1,8 +1,6 @@ PEP: 271 Title: Prefixing sys.path by command line option -Version: $Revision$ -Last-Modified: $Date$ -Author: fred@arakne.com (FrĂ©dĂ©ric B. Giacometti) +Author: FrĂ©dĂ©ric B. Giacometti Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -72,14 +70,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0272.txt b/peps/pep-0272.rst similarity index 97% rename from pep-0272.txt rename to peps/pep-0272.rst index d55387239..b7e2df854 100644 --- a/pep-0272.txt +++ b/peps/pep-0272.rst @@ -47,7 +47,7 @@ SP 800-38A [1]_. Descriptions of the first three feedback modes can also be found in Bruce Schneier's book *Applied Cryptography* [2]_. (The numeric value 4 is reserved for MODE_PGP, a variant of CFB -described in RFC 2440: "OpenPGP Message Format" [3]_. This mode +described in :rfc:`2440`: "OpenPGP Message Format". This mode isn't considered important enough to make it worth requiring it for all block encryption ciphers, though supporting it is a nice extra feature.) @@ -209,9 +209,6 @@ References .. [2] Applied Cryptography -.. [3] RFC2440: "OpenPGP Message Format" (http://rfc2440.x42.com, - http://www.faqs.org/rfcs/rfc2440.html) - Changes ======= diff --git a/pep-0273.txt b/peps/pep-0273.rst similarity index 96% rename from pep-0273.txt rename to peps/pep-0273.rst index 5855a4562..be1a66eb6 100644 --- a/pep-0273.txt +++ b/peps/pep-0273.rst @@ -1,8 +1,6 @@ PEP: 273 Title: Import Modules from Zip Archives -Version: $Revision$ -Last-Modified: $Date$ -Author: jim@interet.com (James C. Ahlstrom) +Author: James C. Ahlstrom Status: Final Type: Standards Track Content-Type: text/x-rst @@ -26,7 +24,7 @@ Note Zip imports were added to Python 2.3, but the final implementation uses an approach different from the one described in this PEP. The 2.3 implementation is SourceForge patch #652586 [1]_, which adds -new import hooks described in PEP 302. +new import hooks described in :pep:`302`. The rest of this PEP is therefore only of historical interest. @@ -210,8 +208,8 @@ A newer version (updated for recent CVS by Paul Moore) is 645650. Superseded by patch 652586 and current CVS. [3]_ A competing implementation by Just van Rossum is 652586, which is -the basis for the final implementation of PEP 302. PEP 273 has -been implemented using PEP 302's import hooks. [1]_ +the basis for the final implementation of :pep:`302`. :pep:`273` has +been implemented using :pep:`302`'s import hooks. [1]_ References @@ -233,12 +231,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0274.txt b/peps/pep-0274.rst similarity index 97% rename from pep-0274.txt rename to peps/pep-0274.rst index 10e5639e7..b5deb7601 100644 --- a/pep-0274.txt +++ b/peps/pep-0274.rst @@ -7,14 +7,14 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 25-Oct-2001 -Python-Version: 2.7, 3.0 (originally 2.3) +Python-Version: 2.7, 3.0 Post-History: 29-Oct-2001 Abstract ======== -PEP 202 introduces a syntactical extension to Python called the +:pep:`202` introduces a syntactical extension to Python called the "list comprehension". This PEP proposes a similar syntactical extension called the "dictionary comprehension" or "dict comprehension" for short. You can use dict comprehensions in ways diff --git a/pep-0275.txt b/peps/pep-0275.rst similarity index 96% rename from pep-0275.txt rename to peps/pep-0275.rst index 4922a2131..a64e3e18b 100644 --- a/pep-0275.txt +++ b/peps/pep-0275.rst @@ -1,8 +1,6 @@ PEP: 275 Title: Switching on Multiple Values -Version: $Revision$ -Last-Modified: $Date$ -Author: mal@lemburg.com (Marc-AndrĂ© Lemburg) +Author: Marc-AndrĂ© Lemburg Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -13,7 +11,7 @@ Post-History: Rejection Notice ================ -A similar PEP for Python 3000, PEP 3103 [2]_, was already rejected, +A similar PEP for Python 3000, :pep:`3103`, was already rejected, so this proposal has no chance of being accepted either. Abstract @@ -353,19 +351,8 @@ References .. [1] https://sourceforge.net/tracker/index.php?func=detail&aid=481118&group_id=5470&atid=305470 -.. [2] http://www.python.org/dev/peps/pep-3103 - Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0276.txt b/peps/pep-0276.rst similarity index 94% rename from pep-0276.txt rename to peps/pep-0276.rst index 37fa385a1..835351f30 100644 --- a/pep-0276.txt +++ b/peps/pep-0276.rst @@ -1,8 +1,6 @@ PEP: 276 Title: Simple Iterator for ints -Version: $Revision$ -Last-Modified: $Date$ -Author: james_althoff@i2.com (Jim Althoff) +Author: Jim Althoff Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -14,7 +12,7 @@ Post-History: Abstract ======== -Python 2.1 added new functionality to support iterators [1]_. +Python 2.1 added new functionality to support iterators (:pep:`234`). Iterators have proven to be useful and convenient in many coding situations. It is noted that the implementation of Python's for-loop control structure uses the iterator protocol as of @@ -152,7 +150,7 @@ idiom is: And from time to time proposals are put forth for ways in which Python could provide a better mechanism for this idiom. Recent -examples include PEP 204, "Range Literals", and PEP 212, "Loop +examples include :pep:`204`, "Range Literals", and :pep:`212`, "Loop Counter Iteration". Most often, such proposal include changes to Python's syntax and @@ -263,13 +261,13 @@ Tim Peters has pointed out two such examples: Issues ====== -Extensive discussions concerning PEP 276 on the Python interest +Extensive discussions concerning :pep:`276` on the Python interest mailing list suggests a range of opinions: some in favor, some neutral, some against. Those in favor tend to agree with the claims above of the usefulness, convenience, ease of learning, and simplicity of a simple iterator for integers. -Issues with PEP 276 include: +Issues with :pep:`276` include: - Using range/xrange is fine as is. @@ -325,7 +323,7 @@ Issues with PEP 276 include: noted above is an a case in point. Response: From the author's perspective the examples of the - above that were identified in the PEP 276 discussions did + above that were identified in the :pep:`276` discussions did not appear to be ones that would be accidentally misused in ways that would lead to subtle and hard-to-detect errors. @@ -376,7 +374,7 @@ Issues with PEP 276 include: - Why not propose even bigger changes? -The majority of disagreement with PEP 276 came from those who +The majority of disagreement with :pep:`276` came from those who favor much larger changes to Python to address the more general problem of specifying a sequence of integers where such a specification is general enough to handle the starting value, @@ -403,10 +401,10 @@ These include: It should be noted that there was much debate but not an overwhelming consensus for any of these larger-scale suggestions. -Clearly, PEP 276 does not propose such a large-scale change +Clearly, :pep:`276` does not propose such a large-scale change and instead focuses on a specific problem area. Towards the end of the discussion period, several posters expressed favor -for the narrow focus and simplicity of PEP 276 vis-a-vis the more +for the narrow focus and simplicity of :pep:`276` vis-a-vis the more ambitious suggestions that were advanced. There did appear to be consensus for the need for a PEP for any such larger-scale, alternative suggestion. In light of this recognition, details of @@ -422,28 +420,7 @@ int with an ``__iter__`` method (written in Python) as a means to test out the ideas in this proposal, however. -References -========== - -.. [1] PEP 234, Iterators - http://www.python.org/dev/peps/pep-0234/ - -.. [2] PEP 204, Range Literals - http://www.python.org/dev/peps/pep-0204/ - -.. [3] PEP 212, Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - - Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0277.txt b/peps/pep-0277.rst similarity index 93% rename from pep-0277.txt rename to peps/pep-0277.rst index 07a046d75..dddc72dd5 100644 --- a/pep-0277.txt +++ b/peps/pep-0277.rst @@ -1,8 +1,6 @@ PEP: 277 Title: Unicode file name support for Windows NT -Version: $Revision$ -Last-Modified: $Date$ -Author: neilh@scintilla.org (Neil Hodgson) +Author: Neil Hodgson Status: Final Type: Standards Track Content-Type: text/x-rst @@ -103,23 +101,13 @@ The implementation is available at [2]_. References ========== -.. [1] Microsoft Windows APIs - http://msdn.microsoft.com/ +[1] Microsoft Windows APIs +\ https://msdn.microsoft.com/ -.. [2] https://bugs.python.org/issue594001 +.. [2] https://github.com/python/cpython/issues/37017 Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: - diff --git a/pep-0278.txt b/peps/pep-0278.rst similarity index 98% rename from pep-0278.txt rename to peps/pep-0278.rst index d24408b1f..a2ac61356 100644 --- a/pep-0278.txt +++ b/peps/pep-0278.rst @@ -1,8 +1,6 @@ PEP: 278 Title: Universal Newline Support -Version: $Revision$ -Last-Modified: $Date$ -Author: jack@cwi.nl (Jack Jansen) +Author: Jack Jansen Status: Final Type: Standards Track Content-Type: text/x-rst @@ -205,12 +203,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0279.txt b/peps/pep-0279.rst similarity index 90% rename from pep-0279.txt rename to peps/pep-0279.rst index 8a12fbc41..2f3eeecb8 100644 --- a/pep-0279.txt +++ b/peps/pep-0279.rst @@ -1,8 +1,6 @@ PEP: 279 Title: The enumerate() built-in function -Version: $Revision$ -Last-Modified: $Date$ -Author: python@rcn.com (Raymond Hettinger) +Author: Raymond Hettinger Status: Final Type: Standards Track Content-Type: text/x-rst @@ -24,17 +22,17 @@ Rationale ========= Python 2.2 introduced the concept of an iterable interface as -proposed in PEP 234 [3]_. The ``iter()`` factory function was provided +proposed in :pep:`234`. The ``iter()`` factory function was provided as common calling convention and deep changes were made to use iterators as a unifying theme throughout Python. The unification came in the form of establishing a common iterable interface for mappings, sequences, and file objects. -Generators, as proposed in PEP 255 [1]_, were introduced as a means +Generators, as proposed in :pep:`255`, were introduced as a means for making it easier to create iterators, especially ones with complex internal execution or variable states. The availability of generators makes it possible to improve on the loop counter -ideas in PEP 212 [2]_. Those ideas provided a clean syntax for +ideas in :pep:`212`. Those ideas provided a clean syntax for iteration with indices and values, but did not apply to all iterable objects. Also, that approach did not have the memory friendly benefit provided by generators which do not evaluate the @@ -73,13 +71,13 @@ Specification for a new built-in yield (i, it.next()) i += 1 -Note A: PEP 212 Loop Counter Iteration [2]_ discussed several +Note A: :pep:`212` Loop Counter Iteration discussed several proposals for achieving indexing. Some of the proposals only work for lists unlike the above function which works for any generator, xrange, sequence, or iterable object. Also, those proposals were presented and evaluated in the world prior to Python 2.2 which did not include generators. As a result, the non-generator version in -PEP 212 had the disadvantage of consuming memory with a giant list +:pep:`212` had the disadvantage of consuming memory with a giant list of tuples. The generator version presented here is fast and light, works with all iterables, and allows users to abandon the sequence in mid-stream with no loss of computation effort. @@ -197,31 +195,7 @@ Author response: the ``itertools`` module. -References -========== - -.. [1] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [2] PEP 212 Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - -.. [3] PEP 234 Iterators - http://www.python.org/dev/peps/pep-0234/ - - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: - - diff --git a/pep-0280.txt b/peps/pep-0280.rst similarity index 98% rename from pep-0280.txt rename to peps/pep-0280.rst index 843d6d8fc..9a28db3f1 100644 --- a/pep-0280.txt +++ b/peps/pep-0280.rst @@ -1,8 +1,6 @@ PEP: 280 Title: Optimizing access to globals -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -15,7 +13,7 @@ Deferral ======== While this PEP is a nice idea, no-one has yet emerged to do the work of -hashing out the differences between this PEP, PEP 266 and PEP 267. +hashing out the differences between this PEP, :pep:`266` and :pep:`267`. Hence, it is being deferred. @@ -23,8 +21,8 @@ Abstract ======== This PEP describes yet another approach to optimizing access to -module globals, providing an alternative to PEP 266 (Optimizing -Global Variable/Attribute Access by Skip Montanaro) and PEP 267 +module globals, providing an alternative to :pep:`266` (Optimizing +Global Variable/Attribute Access by Skip Montanaro) and :pep:`267` (Optimized Access to Module Namespaces by Jeremy Hylton). The expectation is that eventually one approach will be picked and @@ -529,10 +527,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - End: diff --git a/pep-0281.txt b/peps/pep-0281.rst similarity index 85% rename from pep-0281.txt rename to peps/pep-0281.rst index 7d35a6ae8..9bf788134 100644 --- a/pep-0281.txt +++ b/peps/pep-0281.rst @@ -1,8 +1,6 @@ PEP: 281 Title: Loop Counter Iteration with range and xrange -Version: $Revision$ -Last-Modified: $Date$ -Author: magnus@hetland.org (Magnus Lie Hetland) +Author: Magnus Lie Hetland Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -16,16 +14,16 @@ Abstract This PEP describes yet another way of exposing the loop counter in for-loops. It basically proposes that the functionality of the -function ``indices()`` from PEP 212 [1]_ be included in the existing +function ``indices()`` from :pep:`212` be included in the existing functions ``range()`` and ``xrange()``. Pronouncement ============= -In commenting on PEP 279's ``enumerate()`` function, this PEP's author -offered, "I'm quite happy to have it make PEP 281 obsolete." -Subsequently, PEP 279 was accepted into Python 2.3. +In commenting on :pep:`279`'s ``enumerate()`` function, this PEP's author +offered, "I'm quite happy to have it make :pep:`281` obsolete." +Subsequently, :pep:`279` was accepted into Python 2.3. On 17 June 2005, the BDFL concurred with it being obsolete and hereby rejected the PEP. For the record, he found some of the @@ -116,7 +114,7 @@ Example:: print num # The line itself is not accessible A more controversial alternative (to deal with this) would be to -let ``range()`` behave like the function ``irange()`` of PEP 212 when +let ``range()`` behave like the function ``irange()`` of :pep:`212` when supplied with a sequence. Example:: @@ -136,23 +134,7 @@ the case of lazy iteration with ``xrange``). The author does not believe that this is a significant problem. -References and Footnotes -======================== - -.. [1] PEP 212, Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0282.txt b/peps/pep-0282.rst similarity index 97% rename from pep-0282.txt rename to peps/pep-0282.rst index aceb57822..a328a01b3 100644 --- a/pep-0282.txt +++ b/peps/pep-0282.rst @@ -1,8 +1,7 @@ PEP: 282 Title: A Logging System -Version: $Revision$ -Last-Modified: $Date$ -Author: vinay_sajip at red-dove.com (Vinay Sajip), trentm@activestate.com (Trent Mick) +Author: Vinay Sajip , + Trent Mick Status: Final Type: Standards Track Content-Type: text/x-rst @@ -63,7 +62,7 @@ Simple Example This shows a very simple example of how the logging package can be used to generate simple logging output on stderr. -:: +.. code-block:: python --------- mymodule.py ------------------------------- import logging @@ -75,6 +74,8 @@ used to generate simple logging output on stderr. raise TypeError, "Bogus type error for testing" ----------------------------------------------------- +.. code-block:: python + --------- myapp.py ---------------------------------- import mymodule, logging @@ -90,7 +91,9 @@ used to generate simple logging output on stderr. log.info("Ending my app") ----------------------------------------------------- - % python myapp.py +.. code-block:: console + + $ python myapp.py INFO:MyApp: Starting my app DEBUG:MyModule: Doin' stuff... @@ -106,7 +109,9 @@ used to generate simple logging output on stderr. The above example shows the default output format. All aspects of the output format should be configurable, so that -you could have output formatted like this:: +you could have output formatted like this: + +.. code-block:: text 2002-04-19 07:56:58,174 MyModule DEBUG - Doin' stuff... @@ -126,11 +131,9 @@ Logger names fit into a "dotted name" namespace, with dots (periods) indicating sub-namespaces. The namespace of logger objects therefore corresponds to a single tree data structure. -:: - - "" is the root of the namespace - "Zope" would be a child node of the root - "Zope.ZODB" would be a child node of "Zope" +* ``""`` is the root of the namespace +* ``"Zope"`` would be a child node of the root +* ``"Zope.ZODB"`` would be a child node of ``"Zope"`` These Logger objects create **LogRecord** objects which are passed to **Handler** objects for output. Both Loggers and Handlers may @@ -169,13 +172,13 @@ This is done through a module-level function:: Levels ====== -The logging levels, in increasing order of importance, are:: +The logging levels, in increasing order of importance, are: - DEBUG - INFO - WARN - ERROR - CRITICAL +* DEBUG +* INFO +* WARN +* ERROR +* CRITICAL The term CRITICAL is used in preference to FATAL, which is used by log4j. The levels are conceptually the same - that of a serious, @@ -623,19 +626,10 @@ References http://starship.python.net/crew/jbauer/creosote/ .. [6] Vinay Sajip's logging module. - http://www.red-dove.com/python_logging.html + https://old.red-dove.com/python_logging.html Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0283.txt b/peps/pep-0283.rst similarity index 90% rename from pep-0283.txt rename to peps/pep-0283.rst index 0ee2c6d75..42910b64f 100644 --- a/pep-0283.txt +++ b/peps/pep-0283.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Guido van Rossum Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 27-Feb-2002 Python-Version: 2.3 @@ -50,13 +51,13 @@ for more, and of course ``Misc/NEWS`` for the full list. - Tk 8.4 update. -- The ``bool`` type and its constants, ``True`` and ``False`` (PEP 285). +- The ``bool`` type and its constants, ``True`` and ``False`` (:pep:`285`). - ``PyMalloc`` was greatly enhanced and is enabled by default. -- Universal newline support (PEP 278). +- Universal newline support (:pep:`278`). -- PEP 263 Defining Python Source Code Encodings, Lemburg +- :pep:`263` Defining Python Source Code Encodings, Lemburg Implemented (at least phase 1, which is all that's planned for 2.3). @@ -70,7 +71,7 @@ for more, and of course ``Misc/NEWS`` for the full list. - Timeout sockets. https://bugs.python.org/issue555085 -- Stage B0 of the ``int``/``long`` integration (PEP 237). This means +- Stage B0 of the ``int``/``long`` integration (:pep:`237`). This means issuing a ``FutureWarning`` about situations where ``hex`` or ``oct`` conversions or left shifts returns a different value for an ``int`` than for a ``long`` with the same value. The semantics do *not* @@ -96,7 +97,7 @@ for more, and of course ``Misc/NEWS`` for the full list. - Warn for assignment to ``None`` (in various forms). -- PEP 218 Adding a Built-In Set Object Type, Wilson +- :pep:`218` Adding a Built-In Set Object Type, Wilson Alex Martelli contributed a new version of Greg Wilson's prototype, and I've reworked that quite a bit. It's in the @@ -104,12 +105,12 @@ for more, and of course ``Misc/NEWS`` for the full list. may still change until the first beta release. (There are no plans to make this a built-in type, for now.) -- PEP 293 Codec error handling callbacks, Dörwald +- :pep:`293` Codec error handling callbacks, Dörwald Fully implemented. Error handling in ``unicode.encode`` or ``str.decode`` can now be customized. -- PEP 282 A Logging System, Mick +- :pep:`282` A Logging System, Mick Vinay Sajip's implementation has been packagized and imported. (Documentation and unit tests still pending.) @@ -130,23 +131,23 @@ for more, and of course ``Misc/NEWS`` for the full list. prototype was coded in ``nondist/sandbox/datetime/``. Tim Peters has finished the C implementation and checked it in. -- PEP 273 Import Modules from Zip Archives, Ahlstrom +- :pep:`273` Import Modules from Zip Archives, Ahlstrom - Implemented as a part of the PEP 302 implementation work. + Implemented as a part of the :pep:`302` implementation work. -- PEP 302 New Import Hooks, JvR +- :pep:`302` New Import Hooks, JvR Implemented (though the 2.3a1 release contained some bugs that have been fixed post-release). -- A new pickling protocol. See PEP 307. +- A new pickling protocol. See :pep:`307`. -- PEP 305 (CSV File API, by Skip Montanaro et al.) is in; this is +- :pep:`305` (CSV File API, by Skip Montanaro et al.) is in; this is the csv module. - Raymond Hettinger's ``itertools`` module is in. -- PEP 311 (Simplified GIL Acquisition for Extensions, by Mark +- :pep:`311` (Simplified GIL Acquisition for Extensions, by Mark Hammond) has been included in beta 1. - Two new ``PyArg_Parse*()`` format codes, 'k' returns an unsigned C @@ -236,7 +237,7 @@ Features that did not make it into Python 2.3 I believe this is dead now. -- PEP 304 (Controlling Generation of Bytecode Files by Montanaro) +- :pep:`304` (Controlling Generation of Bytecode Files by Montanaro) seems to have lost steam. - For a class defined inside another class, the ``__name__`` should be @@ -272,7 +273,7 @@ Features that did not make it into Python 2.3 It seems that this is never going to be resolved. -- PEP 269 Pgen Module for Python, Riehl +- :pep:`269` Pgen Module for Python, Riehl (Some necessary changes are in; the ``pgen`` module itself needs to mature more.) @@ -283,11 +284,11 @@ Features that did not make it into Python 2.3 to champion it. (Some changes to distutils to support this are in, at least.) -- PEP 266 Optimizing Global Variable/Attribute Access, Montanaro +- :pep:`266` Optimizing Global Variable/Attribute Access, Montanaro - PEP 267 Optimized Access to Module Namespaces, Hylton + :pep:`267` Optimized Access to Module Namespaces, Hylton - PEP 280 Optimizing access to globals, van Rossum + :pep:`280` Optimizing access to globals, van Rossum These are basically three friendly competing proposals. Jeremy has made a little progress with a new compiler, but it's going @@ -305,7 +306,7 @@ Features that did not make it into Python 2.3 Not much enthusiasm I believe. -- PEP 286 Enhanced Argument Tuples, von Loewis +- :pep:`286` Enhanced Argument Tuples, von Loewis I haven't had the time to review this thoroughly. It seems a deep optimization hack (also makes better correctness guarantees diff --git a/pep-0284.txt b/peps/pep-0284.rst similarity index 88% rename from pep-0284.txt rename to peps/pep-0284.rst index 2d3355bdd..08fedf5ab 100644 --- a/pep-0284.txt +++ b/peps/pep-0284.rst @@ -1,9 +1,7 @@ PEP: 284 Title: Integer for-loops -Version: $Revision$ -Last-Modified: $Date$ Author: David Eppstein , - Greg Ewing + Gregory Ewing Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -35,7 +33,7 @@ Pronouncement This PEP is rejected. There were a number of fixable issues with the proposal (see the fixups listed in Raymond Hettinger's -python-dev post on 18 June 2005 [5]_). However, even with the fixups the +python-dev post on 18 June 2005 [1]_). However, even with the fixups the proposal did not garner support. Specifically, Guido did not buy the premise that the ``range()`` format needed fixing, "The whole point (15 years ago) of ``range()`` was to *avoid* needing syntax to specify a @@ -59,21 +57,21 @@ code that uses ``range()`` or ``xrange()``. The perceived lack of a natural, intuitive integer iteration syntax has led to heated debate on python-list, and spawned at -least four PEPs before this one. PEP 204 [1]_ (rejected) proposed +least four PEPs before this one. :pep:`204` (rejected) proposed to re-use Python's slice syntax for integer ranges, leading to a terser syntax but not solving the readability problem of -multi-argument ``range()``. PEP 212 [2]_ (deferred) proposed several +multi-argument ``range()``. :pep:`212` (deferred) proposed several syntaxes for directly converting a list to a sequence of integer indices, in place of the current idiom :: range(len(list)) -for such conversion, and PEP 281 [3]_ proposes to simplify the same +for such conversion, and :pep:`281` proposes to simplify the same idiom by allowing it to be written as :: range(list). -PEP 276 [4]_ proposes to allow automatic conversion of integers to +:pep:`276` proposes to allow automatic conversion of integers to iterators, simplifying the most common half-open case but not addressing the complexities of other types of interval. Additional alternatives have been discussed on python-list. @@ -213,13 +211,13 @@ proposals on the Python list. this loss is outweighed by the increase in readability from a natural integer iteration syntax. -- To some extent, this PEP addresses the same issues as PEP 276 - [4]_. We feel that the two PEPs are not in conflict since PEP - 276 is primarily concerned with half-open ranges starting in 0 +- To some extent, this PEP addresses the same issues as :pep:`276`. + We feel that the two PEPs are not in conflict since :pep:`276` + is primarily concerned with half-open ranges starting in 0 (the easy case of ``range()``) while this PEP is primarily concerned with simplifying all other cases. However, if this PEP is approved, its new simpler syntax for integer loops could to some - extent reduce the motivation for PEP 276. + extent reduce the motivation for :pep:`276`. - It is not clear whether it makes sense to allow floating point bounds for an integer loop: if a float represents an inexact @@ -256,19 +254,7 @@ into a loop over the items in a special iterator object. References ========== -.. [1] PEP 204, Range Literals - http://www.python.org/dev/peps/pep-0204/ - -.. [2] PEP 212, Loop Counter Iteration - http://www.python.org/dev/peps/pep-0212/ - -.. [3] PEP 281, Loop Counter Iteration with range and xrange - http://www.python.org/dev/peps/pep-0281/ - -.. [4] PEP 276, Simple Iterator for ints - http://www.python.org/dev/peps/pep-0276/ - -.. [5] Raymond Hettinger, Propose updating PEP 284 -- Integer for-loops +.. [1] Raymond Hettinger, Propose updating PEP 284 -- Integer for-loops https://mail.python.org/pipermail/python-dev/2005-June/054316.html @@ -276,12 +262,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0285.txt b/peps/pep-0285.rst similarity index 98% rename from pep-0285.txt rename to peps/pep-0285.rst index d271305c6..a69d495b2 100644 --- a/pep-0285.txt +++ b/peps/pep-0285.rst @@ -1,14 +1,12 @@ PEP: 285 Title: Adding a bool type -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Final Type: Standards Track Content-Type: text/x-rst Created: 08-Mar-2002 Python-Version: 2.3 -Post-History: 8-Mar-2002, 30-Mar-2002, 3-Apr-2002 +Post-History: 08-Mar-2002, 30-Mar-2002, 03-Apr-2002 Abstract @@ -417,7 +415,7 @@ Resolved Issues since the bool is implicit in the "if". Explicit is **not** better than implicit here, since the added verbiage impairs - redability and there's no other interpretation possible. There + readability and there's no other interpretation possible. There is, however, sometimes a reason to write :: @@ -447,11 +445,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0286.txt b/peps/pep-0286.rst similarity index 94% rename from pep-0286.txt rename to peps/pep-0286.rst index 6010f597b..e6d56af8c 100644 --- a/pep-0286.txt +++ b/peps/pep-0286.rst @@ -1,8 +1,6 @@ PEP: 286 Title: Enhanced Argument Tuples -Version: $Revision$ -Last-Modified: $Date$ -Author: martin@v.loewis.de (Martin von Löwis) +Author: Martin von Löwis Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -28,7 +26,7 @@ PEP and collecting and incorporating feedback, and with sufficient available time to do so effectively. The resolution of this PEP may also be affected by the resolution of -PEP 426, which proposes the use of a preprocessing step to generate +:pep:`426`, which proposes the use of a preprocessing step to generate some aspects of C API interface code. @@ -124,11 +122,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0287.txt b/peps/pep-0287.rst similarity index 96% rename from pep-0287.txt rename to peps/pep-0287.rst index d04c0f232..3d6ddfd36 100644 --- a/pep-0287.txt +++ b/peps/pep-0287.rst @@ -3,7 +3,7 @@ Title: reStructuredText Docstring Format Version: $Revision$ Last-Modified: $Date$ Author: David Goodger -Discussions-To: +Discussions-To: doc-sig@python.org Status: Active Type: Informational Content-Type: text/x-rst @@ -25,7 +25,7 @@ what-you-see-is-what-you-get plaintext markup syntax. Only the low-level syntax of docstrings is addressed here. This PEP is not concerned with docstring semantics or processing at all (see -PEP 256 for a "Road Map to the Docstring PEPs"). Nor is it an attempt +:pep:`256` for a "Road Map to the Docstring PEPs"). Nor is it an attempt to deprecate pure plaintext docstrings, which are always going to be legitimate. The reStructuredText markup is an alternative for those who want more expressive docstrings. @@ -152,7 +152,7 @@ b) Replace the PEP section structure constructs with the Strategy (b) is recommended, and its implementation is complete. -Support for RFC 2822 headers has been added to the reStructuredText +Support for :rfc:`2822` headers has been added to the reStructuredText parser for PEPs (unambiguous given a specific context: the first contiguous block of the document). It may be desired to concretely specify what over/underline styles are allowed for PEP section @@ -406,7 +406,7 @@ Docstring-Significant Features such as identifying parameters, exceptions raised, etc.; such usage is beyond the scope of this PEP. - A modified RFC 2822 syntax is used, with a colon *before* as well as + A modified :rfc:`2822` syntax is used, with a colon *before* as well as *after* the field name. Field bodies are more versatile as well; they may contain multiple field bodies (even nested field lists). For example:: @@ -418,7 +418,7 @@ Docstring-Significant Features - Myself - I - Standard RFC 2822 header syntax cannot be used for this construct + Standard :rfc:`2822` header syntax cannot be used for this construct because it is ambiguous. A word followed by a colon at the beginning of a line is common in written text. @@ -564,7 +564,7 @@ Questions & Answers [1] http://www.example.org/ [2] PEP 9876, Let's Hope We Never Get Here - http://www.python.org/dev/peps/pep-9876/ + http://peps.python.org/pep-9876/ [3] "Bogus Complexity Addition" @@ -624,8 +624,8 @@ Questions & Answers The PEP markup proposal may be removed if it is deemed that there is no need for PEP markup, or it could be made into a separate PEP. - If accepted, PEP 1, PEP Purpose and Guidelines [#PEP-1]_, and PEP - 9, Sample PEP Template [#PEP-9]_ will be updated. + If accepted, :pep:`1`, PEP Purpose and Guidelines, and :pep:`9`, + Sample PEP Template will be updated. It seems natural to adopt a single consistent markup standard for all uses of structured plaintext in Python, and to propose it all @@ -670,8 +670,8 @@ Questions & Answers input and check the output for correctness. In a strict sense, the reStructuredText parser is very unforgiving - (as it should be; "In the face of ambiguity, refuse the temptation - to guess" [#Zen]_ applies to parsing markup as well as computer + (as it should be; :pep:`"In the face of ambiguity, refuse the temptation + to guess" <20>` applies to parsing markup as well as computer languages). Here's design goal 3 from `An Introduction to reStructuredText`_: @@ -727,20 +727,6 @@ Questions & Answers References & Footnotes ====================== -.. [#PEP-1] PEP 1, PEP Guidelines, Warsaw, Hylton - (http://www.python.org/dev/peps/pep-0001/) - -.. [#PEP-9] PEP 9, Sample PEP Template, Warsaw - (http://www.python.org/dev/peps/pep-0009/) - -.. [#Zen] From `The Zen of Python (by Tim Peters)`__ (or just - "``import this``" in Python) - -__ http://www.python.org/doc/Humor.html#zen - -.. [#PEP-216] PEP 216, Docstring Format, Zadka - (http://www.python.org/dev/peps/pep-0216/) - .. _reStructuredText markup: http://docutils.sourceforge.net/rst.html .. _Doc-SIG: http://www.python.org/sigs/doc-sig/ @@ -799,7 +785,7 @@ This document has been placed in the public domain. Acknowledgements ================ -Some text is borrowed from PEP 216, Docstring Format [#PEP-216]_, by +Some text is borrowed from :pep:`216`, Docstring Format, by Moshe Zadka. Special thanks to all members past & present of the Python Doc-SIG_. diff --git a/pep-0288.txt b/peps/pep-0288.rst similarity index 91% rename from pep-0288.txt rename to peps/pep-0288.rst index 2640fde54..0b8a35dff 100644 --- a/pep-0288.txt +++ b/peps/pep-0288.rst @@ -1,8 +1,6 @@ PEP: 288 Title: Generators Attributes and Exceptions -Version: $Revision$ -Last-Modified: $Date$ -Author: python@rcn.com (Raymond Hettinger) +Author: Raymond Hettinger Status: Withdrawn Type: Standards Track Content-Type: text/x-rst @@ -22,7 +20,7 @@ Status ====== This PEP is withdrawn. The exception raising mechanism was extended -and subsumed into PEP 343. The attribute passing capability +and subsumed into :pep:`343`. The attribute passing capability never built a following, did not have a clear implementation, and did not have a clean way for the running generator to access its own namespace. @@ -119,7 +117,7 @@ code cannot be excepted to or through. Generator exception passing also helps address an intrinsic limitation on generators, the prohibition against their using -try/finally to trigger clean-up code [2]_. +try/finally to trigger clean-up code (:pep:`255`). Note A: The name of the throw method was selected for several reasons. Raise is a keyword and so cannot be used as a method @@ -149,24 +147,9 @@ References http://gnosis.cx/publish/programming/charming_python_b5.txt http://gnosis.cx/publish/programming/charming_python_b7.txt -.. [2] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [3] Proof-of-concept recipe - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/164044 - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0289.txt b/peps/pep-0289.rst similarity index 90% rename from pep-0289.txt rename to peps/pep-0289.rst index e1d06fb75..dcb505940 100644 --- a/pep-0289.txt +++ b/peps/pep-0289.rst @@ -1,8 +1,6 @@ PEP: 289 Title: Generator Expressions -Version: $Revision$ -Last-Modified: $Date$ -Author: python@rcn.com (Raymond Hettinger) +Author: Raymond Hettinger Status: Final Type: Standards Track Content-Type: text/x-rst @@ -15,8 +13,8 @@ Abstract ======== This PEP introduces generator expressions as a high performance, -memory efficient generalization of list comprehensions [1]_ and -generators [2]_. +memory efficient generalization of list comprehensions :pep:`202` and +generators :pep:`255`. Rationale @@ -214,7 +212,7 @@ for-expression should be evaluated immediately and that the remaining expressions be evaluated when the generator is executed. Asked to summarize the reasoning for binding the first expression, -Guido offered [5]_:: +Guido offered [1]_:: Consider sum(x for x in foo()). Now suppose there's a bug in foo() that raises an exception, and a bug in sum() that raises an @@ -243,7 +241,7 @@ issues were hard to understand and that users should be strongly encouraged to use generator expressions inside functions that consume their arguments immediately. For more complex applications, full generator definitions are always superior in terms of being obvious -about scope, lifetime, and binding [6]_. +about scope, lifetime, and binding [2]_. Reduction Functions @@ -275,35 +273,22 @@ Acknowledgements * Armin Rigo, Tim Peters, Guido van Rossum, Samuele Pedroni, Hye-Shik Chang and Raymond Hettinger teased out the issues surrounding - early versus late binding [5]_. + early versus late binding [1]_. * Jiwon Seo single-handedly implemented various versions of the proposal including the final version loaded into CVS. Along the way, there were periodic code reviews by Hye-Shik Chang and Raymond Hettinger. Guido van Rossum made the key design decisions after comments from Armin Rigo and newsgroup discussions. Raymond Hettinger provided - the test suite, documentation, tutorial, and examples [6]_. + the test suite, documentation, tutorial, and examples [2]_. References ========== -.. [1] PEP 202 List Comprehensions - http://www.python.org/dev/peps/pep-0202/ - -.. [2] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [3] Peter Norvig's Accumulation Display Proposal - http://www.norvig.com/pyacc.html - -.. [4] Jeff Epler had worked up a patch demonstrating - the previously proposed bracket and yield syntax - https://bugs.python.org/issue795947 - -.. [5] Discussion over the relative merits of early versus late binding +.. [1] Discussion over the relative merits of early versus late binding https://mail.python.org/pipermail/python-dev/2004-April/044555.html -.. [6] Patch discussion and alternative patches on Source Forge +.. [2] Patch discussion and alternative patches on Source Forge https://bugs.python.org/issue872326 @@ -311,12 +296,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0290.txt b/peps/pep-0290.rst similarity index 97% rename from pep-0290.txt rename to peps/pep-0290.rst index b364cf6b3..425d2b8c7 100644 --- a/pep-0290.txt +++ b/peps/pep-0290.rst @@ -29,8 +29,8 @@ This repository of procedures serves as a catalog or checklist of known migration issues and procedures for addressing those issues. Migration issues can arise for several reasons. Some obsolete -features are slowly deprecated according to the guidelines in PEP 4 -[1]_. Also, some code relies on undocumented behaviors which are +features are slowly deprecated according to the guidelines in :pep:`4`. +Also, some code relies on undocumented behaviors which are subject to change between versions. Some code may rely on behavior which was subsequently shown to be a bug and that behavior changes when the bug is fixed. @@ -426,7 +426,7 @@ Pattern:: NewError = 'NewError' --> class NewError(Exception): pass -Locating: Use PyChecker_. +Locating: Use `PyChecker `__. All Python Versions @@ -448,15 +448,6 @@ Pattern:: Locating: ``grep '== None'`` or ``grep '!= None'`` -References -========== - -.. [1] PEP 4, Deprecation of Standard Modules, von Loewis - (http://www.python.org/dev/peps/pep-0004/) - -.. _PyChecker: http://pychecker.sourceforge.net/ - - Copyright ========= diff --git a/pep-0291.txt b/peps/pep-0291.rst similarity index 96% rename from pep-0291.txt rename to peps/pep-0291.rst index 817090367..a535f13f6 100644 --- a/pep-0291.txt +++ b/peps/pep-0291.rst @@ -1,8 +1,6 @@ PEP: 291 Title: Backward Compatibility for the Python 2 Standard Library -Version: $Revision$ -Last-Modified: $Date$ -Author: nnorwitz@gmail.com (Neal Norwitz) +Author: Neal Norwitz Status: Final Type: Informational Content-Type: text/x-rst @@ -153,13 +151,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0292.txt b/peps/pep-0292.rst similarity index 84% rename from pep-0292.txt rename to peps/pep-0292.rst index f4001d112..11660661d 100644 --- a/pep-0292.txt +++ b/peps/pep-0292.rst @@ -1,8 +1,6 @@ PEP: 292 Title: Simpler String Substitutions -Version: $Revision$ -Last-Modified: $Date$ -Author: barry@python.org (Barry Warsaw) +Author: Barry Warsaw Status: Final Type: Standards Track Content-Type: text/x-rst @@ -23,8 +21,8 @@ respects: (i.e. ``%``-substitution) is complicated and error prone. This PEP is simpler at the cost of some expressiveness. -2. PEP 215 proposed an alternative string interpolation feature, - introducing a new ``$`` string prefix. PEP 292 is simpler than +2. :pep:`215` proposed an alternative string interpolation feature, + introducing a new ``$`` string prefix. :pep:`292` is simpler than this because it involves no syntax changes and has much simpler rules for what substitutions can occur in the string. @@ -116,7 +114,7 @@ library documentation. Why ``$`` and Braces? ===================== -The BDFL said it best [4]_: "The ``$`` means "substitution" in so many +The BDFL said it best [3]_: "The ``$`` means "substitution" in so many languages besides Perl that I wonder where you've been. [...] We're copying this from the shell." @@ -128,16 +126,16 @@ easier to teach, learn, and remember. Comparison to PEP 215 ===================== -PEP 215 describes an alternate proposal for string interpolation. +:pep:`215` describes an alternate proposal for string interpolation. Unlike that PEP, this one does not propose any new syntax for Python. All the proposed new features are embodied in a new -library module. PEP 215 proposes a new string prefix +library module. :pep:`215` proposes a new string prefix representation such as ``$""`` which signal to Python that a new type of string is present. ``$``-strings would have to interact with the existing r-prefixes and u-prefixes, essentially doubling the number of string prefix combinations. -PEP 215 also allows for arbitrary Python expressions inside the +:pep:`215` also allows for arbitrary Python expressions inside the ``$``-strings, so that you could do things like:: import sys @@ -147,15 +145,15 @@ which would return:: sys = , sys = -It's generally accepted that the rules in PEP 215 are safe in the -sense that they introduce no new security issues (see PEP 215, +It's generally accepted that the rules in :pep:`215` are safe in the +sense that they introduce no new security issues (see :pep:`215`, "Security Issues" for details). However, the rules are still quite complex, and make it more difficult to see the substitution placeholder in the original ``$``-string. The interesting thing is that the ``Template`` class defined in this PEP is designed for inheritance and, with a little extra work, -it's possible to support PEP 215's functionality using existing +it's possible to support :pep:`215`'s functionality using existing Python syntax. For example, one could define subclasses of ``Template`` and dict that @@ -186,37 +184,23 @@ string/unicode like ``%``-operator substitution syntax. Reference Implementation ======================== -The implementation has been committed to the Python 2.4 source tree. - +The implementation [4]_ has been committed to the Python 2.4 source tree. References ========== .. [1] String Formatting Operations - http://docs.python.org/library/stdtypes.html#string-formatting-operations + https://docs.python.org/release/2.6/library/stdtypes.html#string-formatting-operations .. [2] Identifiers and Keywords - http://docs.python.org/reference/lexical_analysis.html#identifiers-and-keywords + https://docs.python.org/release/2.6/reference/lexical_analysis.html#identifiers-and-keywords -.. [3] Guido's python-dev posting from 21-Jul-2002 - https://mail.python.org/pipermail/python-dev/2002-July/026397.html +.. [3] https://mail.python.org/pipermail/python-dev/2002-June/025652.html -.. [4] https://mail.python.org/pipermail/python-dev/2002-June/025652.html - -.. [5] Reference Implementation +.. [4] Reference Implementation http://sourceforge.net/tracker/index.php?func=detail&aid=1014055&group_id=5470&atid=305470 Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0293.txt b/peps/pep-0293.rst similarity index 100% rename from pep-0293.txt rename to peps/pep-0293.rst diff --git a/pep-0294.txt b/peps/pep-0294.rst similarity index 90% rename from pep-0294.txt rename to peps/pep-0294.rst index 3558005b8..782fce9a6 100644 --- a/pep-0294.txt +++ b/peps/pep-0294.rst @@ -1,8 +1,6 @@ PEP: 294 Title: Type Names in the types Module -Version: $Revision$ -Last-Modified: $Date$ -Author: oren at hishome.net (Oren Tirosh) +Author: Oren Tirosh Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -27,7 +25,7 @@ The long capitalized names currently in the types module will be deprecated. With this change the types module can serve as a replacement for the -new module. The new module shall be deprecated and listed in PEP 4. +new module. The new module shall be deprecated and listed in :pep:`4`. Pronouncement @@ -100,11 +98,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0295.txt b/peps/pep-0295.rst similarity index 92% rename from pep-0295.txt rename to peps/pep-0295.rst index 5396f2393..335fb559b 100644 --- a/pep-0295.txt +++ b/peps/pep-0295.rst @@ -1,8 +1,6 @@ PEP: 295 Title: Interpretation of multiline string constants -Version: $Revision$ -Last-Modified: $Date$ -Author: yozh@mx1.ru (Stepan Koltsov) +Author: Stepan Koltsov Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -119,13 +117,3 @@ Copyright ========= This document has been placed in the Public Domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0296.txt b/peps/pep-0296.rst similarity index 98% rename from pep-0296.txt rename to peps/pep-0296.rst index cd580b6a7..9a769a050 100644 --- a/pep-0296.txt +++ b/peps/pep-0296.rst @@ -1,8 +1,6 @@ PEP: 296 Title: Adding a bytes Object Type -Version: $Revision$ -Last-Modified: $Date$ -Author: xscottg at yahoo.com (Scott Gilbert) +Author: Scott Gilbert Status: Withdrawn Type: Standards Track Content-Type: text/x-rst @@ -14,7 +12,7 @@ Post-History: Notice ======= -This PEP is withdrawn by the author (in favor of PEP 358). +This PEP is withdrawn by the author (in favor of :pep:`358`). Abstract @@ -357,12 +355,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0297.txt b/peps/pep-0297.rst similarity index 93% rename from pep-0297.txt rename to peps/pep-0297.rst index 76a36b71d..96d61a514 100644 --- a/pep-0297.txt +++ b/peps/pep-0297.rst @@ -1,8 +1,6 @@ PEP: 297 Title: Support for System Upgrades -Version: $Revision$ -Last-Modified: $Date$ -Author: mal@lemburg.com (Marc-AndrĂ© Lemburg) +Author: Marc-AndrĂ© Lemburg Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -116,14 +114,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0298.txt b/peps/pep-0298.rst similarity index 97% rename from pep-0298.txt rename to peps/pep-0298.rst index 28cb1b0d9..0414fcb14 100644 --- a/pep-0298.txt +++ b/peps/pep-0298.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 26-Jul-2002 Python-Version: 2.3 -Post-History: 30-Jul-2002, 1-Aug-2002 +Post-History: 30-Jul-2002, 01-Aug-2002 Abstract @@ -193,7 +193,7 @@ Guido recommends For strings that might be impractical because the string object would have to grow 4 bytes to hold the counter; but the - new bytes object (PEP 296) could easily implement the counter, + new bytes object (:pep:`296`) could easily implement the counter, and the array object too -- that way there will be plenty of opportunity to test proper use of the protocol. @@ -224,9 +224,6 @@ References .. [1] The buffer interface https://mail.python.org/pipermail/python-dev/2000-October/009974.html -.. [2] The Buffer Problem - http://www.python.org/dev/peps/pep-0296/ - Copyright ========= diff --git a/pep-0299.txt b/peps/pep-0299.rst similarity index 94% rename from pep-0299.txt rename to peps/pep-0299.rst index fb767450e..f2b93b02c 100644 --- a/pep-0299.txt +++ b/peps/pep-0299.rst @@ -100,7 +100,7 @@ Open Issues Rejection ========= -In a short discussion on python-dev [1], two major backwards +In a short discussion on python-dev [1]_, two major backwards compatibility problems were brought up and Guido pronounced that he doesn't like the idea anyway as it's "not worth the change (in docs, user habits, etc.) and there's nothing particularly broken." @@ -117,12 +117,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0301.txt b/peps/pep-0301.rst similarity index 95% rename from pep-0301.txt rename to peps/pep-0301.rst index a8bb12590..b748e843e 100644 --- a/pep-0301.txt +++ b/peps/pep-0301.rst @@ -5,10 +5,11 @@ Last-Modified: $Date$ Author: Richard Jones Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 24-Oct-2002 Python-Version: 2.3 -Post-History: 8-Nov-2002 +Post-History: 08-Nov-2002 Abstract @@ -20,9 +21,9 @@ tools for submitting package information to the index and extensions to the package metadata to include Trove [2]_ information. This PEP does not address issues of package dependency. It also does -not address storage and download of packages as described in PEP 243 -[6]_. Nor is it proposing a local database of packages as described -in PEP 262 [7]_. +not address storage and download of packages as described in :pep:`243`. +Nor is it proposing a local database of packages as described +in :pep:`262`. Existing package repositories such as the Vaults of Parnassus [3]_, CPAN [4]_ and PAUSE [5]_ will be investigated as prior art in this @@ -52,15 +53,15 @@ The interface for submitting information to the catalog should be as simple as possible - hopefully just a one-line command for most users. Issues of package dependency are not addressed due to the complexity -of such a system. PEP 262 proposes such a system, but as of this +of such a system. :pep:`262` proposes such a system, but as of this writing the PEP is still unfinished. Issues of package dissemination (storage on a central server) are not addressed because they require assumptions about availability of -storage and bandwidth that I am not in a position to make. PEP 243, +storage and bandwidth that I am not in a position to make. :pep:`243`, which is still being developed, is tackling these issues and many more. This proposal is considered compatible with, and adjunct to -the proposal in PEP 243. +the proposal in :pep:`243`. Specification @@ -333,7 +334,7 @@ Rejected Proposals ================== Originally, the index server was to return custom headers (inspired by -PEP 243): +:pep:`243`): **X-Pypi-Status** Either "success" or "fail". @@ -342,7 +343,7 @@ PEP 243): A description of the reason for failure, or additional information in the case of a success. -However, it has been pointed out [8]_ that this is a bad scheme to +However, it has been pointed out [6]_ that this is a bad scheme to use. @@ -364,13 +365,7 @@ References .. [5] PAUSE (http://pause.cpan.org/) -.. [6] PEP 243, Module Repository Upload Mechanism - (http://www.python.org/dev/peps/pep-0243/) - -.. [7] PEP 262, A Database of Installed Python Packages - (http://www.python.org/dev/peps/pep-0262/) - -.. [8] [PEP243] upload status is bogus +.. [6] [PEP243] upload status is bogus (https://mail.python.org/pipermail/distutils-sig/2001-March/002262.html) diff --git a/pep-0302.txt b/peps/pep-0302.rst similarity index 97% rename from pep-0302.txt rename to peps/pep-0302.rst index a99830365..29f9f799d 100644 --- a/pep-0302.txt +++ b/peps/pep-0302.rst @@ -72,13 +72,13 @@ are stored in a non-standard way. Examples include modules that are bundled together in an archive; byte code that is not stored in a ``pyc`` formatted file; modules that are loaded from a database over a network. -The work on this PEP was partly triggered by the implementation of PEP 273, +The work on this PEP was partly triggered by the implementation of :pep:`273`, which adds imports from Zip archives as a built-in feature to Python. While the PEP itself was widely accepted as a must-have feature, the implementation left a few things to desire. For one thing it went through great lengths to integrate itself with ``import.c``, adding lots of code that was either specific for Zip file imports or *not* specific to Zip imports, yet was not -generally useful (or even desirable) either. Yet the PEP 273 implementation +generally useful (or even desirable) either. Yet the :pep:`273` implementation can hardly be blamed for this: it is simply extremely hard to do, given the current state of ``import.c``. @@ -265,7 +265,7 @@ The ``load_module()`` method has a few responsibilities that it must fulfill importer-specific extras, for example getting data associated with an importer. -* The ``__package__`` attribute [8]_ must be set. +* The ``__package__`` attribute must be set (:pep:`366`). If the module is a Python module (as opposed to a built-in module or a dynamically loaded extension), it should execute the module's code in the @@ -398,7 +398,8 @@ the module as a string (using newline characters for line endings) or ``None`` if the source is not available (yet it should still raise ``ImportError`` if the module can't be found by the importer at all). -To support execution of modules as scripts [6]_, the above three methods for +To support execution of modules as scripts (:pep:`338`), +the above three methods for finding the code associated with a module must be implemented. In addition to those methods, the following method may be provided in order to allow the ``runpy`` module to correctly set the ``__file__`` attribute:: @@ -427,7 +428,7 @@ built-in import mechanism". They simply won't invoke any import hooks. A new if loader is not None: loader.load_module(fullname) -In the case of a "basic" import, one the `imp.find_module()` function would +In the case of a "basic" import, one the ``imp.find_module()`` function would handle, the loader object would be a wrapper for the current output of ``imp.find_module()``, and ``loader.load_module()`` would call ``imp.load_module()`` with that output. @@ -519,11 +520,11 @@ user-defined hooks either before or after it. Implementation ============== -The PEP 302 implementation has been integrated with Python as of 2.3a1. An +The :pep:`302` implementation has been integrated with Python as of 2.3a1. An earlier version is available as patch #652586 [9]_, but more interestingly, the issue contains a fairly detailed history of the development and design. -PEP 273 has been implemented using PEP 302's import hooks. +:pep:`273` has been implemented using :pep:`302`'s import hooks. References and Footnotes @@ -546,15 +547,9 @@ References and Footnotes from the actual parent module or be supplied by ``imp.find_module()`` or the proposed ``imp.get_loader()`` function. -.. [6] PEP 338: Executing modules as scripts - http://www.python.org/dev/peps/pep-0338/ - .. [7] Quixote, a framework for developing Web applications http://www.mems-exchange.org/software/quixote/ -.. [8] PEP 366: Main module explicit relative imports - http://www.python.org/dev/peps/pep-0366/ - .. [9] New import hooks + Import from Zip files http://bugs.python.org/issue652586 diff --git a/pep-0303.txt b/peps/pep-0303.rst similarity index 100% rename from pep-0303.txt rename to peps/pep-0303.rst diff --git a/pep-0304.txt b/peps/pep-0304.rst similarity index 97% rename from pep-0304.txt rename to peps/pep-0304.rst index 51979da82..e21ba5703 100644 --- a/pep-0304.txt +++ b/peps/pep-0304.rst @@ -20,7 +20,7 @@ by other changes in the intervening years: - the introduction of isolated mode to handle potential security concerns - the switch to ``importlib``, a fully import-hook based import system implementation -- PEP 3147's change in the bytecode cache layout to use ``__pycache__`` +- :pep:`3147`'s change in the bytecode cache layout to use ``__pycache__`` subdirectories, including the ``source_to_cache(path)`` and ``cache_to_source(path)`` APIs that allow the interpreter to automatically handle the redirection to a separate cache directory @@ -239,11 +239,11 @@ Issues - The interaction of this PEP with import hooks has not been considered yet. In fact, the best way to implement this idea might - be as an import hook. See PEP 302. [5]_ + be as an import hook. See :pep:`302`. -- In the current (pre-PEP 304) environment, it is safe to delete a +- In the current (pre-:pep:`304`) environment, it is safe to delete a source file after the corresponding bytecode file has been created, - since they reside in the same directory. With PEP 304 as currently + since they reside in the same directory. With :pep:`304` as currently defined, this is not the case. A bytecode file in the augmented directory is only considered when the source file is present and it thus never considered when looking for module files ending in @@ -344,9 +344,6 @@ References .. [4] python-dev thread, Parallel pyc construction, Dubois (https://mail.python.org/pipermail/python-dev/2003-January/032060.html) -.. [5] PEP 302, New Import Hooks, van Rossum and Moore - (http://www.python.org/dev/peps/pep-0302) - .. [6] patch 677103, PYTHONBYTECODEBASE patch (PEP 304), Montanaro (https://bugs.python.org/issue677103) diff --git a/pep-0305.txt b/peps/pep-0305.rst similarity index 99% rename from pep-0305.txt rename to peps/pep-0305.rst index c86a68130..939a74ba9 100644 --- a/pep-0305.txt +++ b/peps/pep-0305.rst @@ -7,11 +7,12 @@ Author: Kevin Altis , Andrew McNamara , Skip Montanaro , Cliff Wells -Discussions-To: +Discussions-To: csv@python.org Status: Final Type: Standards Track Content-Type: text/x-rst Created: 26-Jan-2003 +Python-Version: 2.3 Post-History: 31-Jan-2003, 13-Feb-2003 diff --git a/pep-0306.txt b/peps/pep-0306.rst similarity index 100% rename from pep-0306.txt rename to peps/pep-0306.rst diff --git a/pep-0307.txt b/peps/pep-0307.rst similarity index 99% rename from pep-0307.txt rename to peps/pep-0307.rst index 9e552266d..875a6c317 100644 --- a/pep-0307.txt +++ b/peps/pep-0307.rst @@ -7,7 +7,8 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 31-Jan-2003 -Post-History: 7-Feb-2003 +Python-Version: 2.3 +Post-History: 07-Feb-2003 Introduction ============ diff --git a/pep-0308.txt b/peps/pep-0308.rst similarity index 99% rename from pep-0308.txt rename to peps/pep-0308.rst index 940bf3c35..6252e6970 100644 --- a/pep-0308.txt +++ b/peps/pep-0308.rst @@ -7,7 +7,8 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 07-Feb-2003 -Post-History: 7-Feb-2003, 11-Feb-2003 +Python-Version: 2.5 +Post-History: 07-Feb-2003, 11-Feb-2003 Adding a conditional expression diff --git a/pep-0309.txt b/peps/pep-0309.rst similarity index 100% rename from pep-0309.txt rename to peps/pep-0309.rst diff --git a/pep-0310.txt b/peps/pep-0310.rst similarity index 97% rename from pep-0310.txt rename to peps/pep-0310.rst index 29fcf705d..a327def99 100644 --- a/pep-0310.txt +++ b/peps/pep-0310.rst @@ -3,7 +3,7 @@ Title: Reliable Acquisition/Release Pairs Version: $Revision$ Last-Modified: $Date$ Author: Michael Hudson , - Paul Moore + Paul Moore Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -30,7 +30,7 @@ This PEP proposes a piece of syntax (a 'with' block) and a Pronouncement ============= -This PEP is rejected in favor of PEP 343. +This PEP is rejected in favor of :pep:`343`. Rationale @@ -191,7 +191,7 @@ in deep and subtle ways and as such belong to a different PEP. Any Smalltalk/Ruby anonymous block style extension obviously subsumes this one. -PEP 319 is in the same area, but did not win support when aired on +:pep:`319` is in the same area, but did not win support when aired on python-dev. @@ -238,7 +238,7 @@ could be mentioned here. https://mail.python.org/pipermail/python-dev/2003-August/037795.html .. [3] Thread on python-dev with subject - `[Python-Dev] pre-PEP: Resource-Release Support for Generators` + ``[Python-Dev] pre-PEP: Resource-Release Support for Generators`` starting at https://mail.python.org/pipermail/python-dev/2003-August/037803.html diff --git a/pep-0311.txt b/peps/pep-0311.rst similarity index 98% rename from pep-0311.txt rename to peps/pep-0311.rst index 1b7aabc53..7ba46cb69 100644 --- a/pep-0311.txt +++ b/peps/pep-0311.rst @@ -7,7 +7,8 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 05-Feb-2003 -Post-History: 05-Feb-2003 14-Feb-2003 19-Apr-2003 +Python-Version: 2.3 +Post-History: 05-Feb-2003, 14-Feb-2003, 19-Apr-2003 Abstract @@ -102,7 +103,7 @@ must continue to call ``Py_Initialize()``, and for multi-threaded applications, ``PyEval_InitThreads()``. The reason for this is that the first thread to call ``PyEval_InitThreads()`` is nominated as the "main thread" by Python, and so forcing the extension author to -specify the main thread (by forcing her to make this first call) +specify the main thread (by requiring them to make this first call) removes ambiguity. As ``Py_Initialize()`` must be called before ``PyEval_InitThreads()``, and as both of these functions currently support being called multiple times, the burden this places on diff --git a/pep-0312.txt b/peps/pep-0312.rst similarity index 100% rename from pep-0312.txt rename to peps/pep-0312.rst diff --git a/pep-0313.txt b/peps/pep-0313.rst similarity index 88% rename from pep-0313.txt rename to peps/pep-0313.rst index 20b9ea468..c516c6141 100644 --- a/pep-0313.txt +++ b/peps/pep-0313.rst @@ -87,11 +87,11 @@ The new built-in function "roman" will aide the translation from integers to Roman numeral literals. It will accept a single object as an argument, and return a string containing the literal of the same value. If the argument is not an integer or a -rational (see PEP 239 [1]_) it will passed through the existing +rational (see :pep:`239`) it will passed through the existing built-in "int" to obtain the value. This may cause a loss of information if the object was a float. If the object is a rational, then the result will be formatted as a rational literal -(see PEP 240 [2]_) with the integers in the string being Roman +(see :pep:`240`) with the integers in the string being Roman numeral literals. @@ -104,31 +104,17 @@ characters M, D, C, L, X, V and I will be affected by the new literals. These programs will now have syntax errors when those variables are assigned, and either syntax errors or subtle bugs when those variables are referenced in expressions. Since such -variable names violate PEP 8 [3]_, the code is already broken, it +variable names violate :pep:`8`, the code is already broken, it just wasn't generating exceptions. This proposal corrects that oversight in the language. -References -========== - -.. [1] PEP 239, Adding a Rational Type to Python - http://www.python.org/dev/peps/pep-0239/ - -.. [2] PEP 240, Adding a Rational Literal to Python - http://www.python.org/dev/peps/pep-0240/ - -.. [3] PEP 8, Style Guide for Python Code - http://www.python.org/dev/peps/pep-0008/ - - Copyright ========= This document has been placed in the public domain. - .. Local Variables: mode: indented-text diff --git a/pep-0314.txt b/peps/pep-0314.rst similarity index 95% rename from pep-0314.txt rename to peps/pep-0314.rst index 6355b4388..6817cff3e 100644 --- a/pep-0314.txt +++ b/peps/pep-0314.rst @@ -1,15 +1,16 @@ PEP: 314 -Title: Metadata for Python Software Packages v1.1 -Version: $Revision$ -Last-Modified: $Date$ +Title: Metadata for Python Software Packages 1.1 Author: A.M. Kuchling, Richard Jones -Status: Final +Discussions-To: distutils-sig@python.org +Status: Superseded Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 12-Apr-2003 Python-Version: 2.5 Post-History: 29-Apr-2003 Replaces: 241 +Superseded-By: 345 Introduction @@ -20,7 +21,7 @@ packages. It includes specifics of the field names, and their semantics and usage. This document specifies version 1.1 of the metadata format. -Version 1.0 is specified in PEP 241. +Version 1.0 is specified in :pep:`241`. Including Metadata in Packages @@ -37,7 +38,7 @@ command will, if it detects an existing PKG-INFO file, terminate with an appropriate error message. This should prevent confusion caused by the PKG-INFO and setup.py files being out of sync. -The PKG-INFO file format is a single set of RFC-822 headers +The PKG-INFO file format is a single set of :rfc:`822` headers parseable by the rfc822.py module. The field names listed in the following section are used as the header names. @@ -181,7 +182,7 @@ Author-email ------------ A string containing the author's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` 'From:' header. It's not optional because cataloging systems can use the e-mail portion of this field as a unique key representing the author. A catalog might provide authors the @@ -212,7 +213,7 @@ Classifier (multiple use) ------------------------- Each entry is a string giving a single classification value -for the package. Classifiers are described in PEP 301 [2]_. +for the package. Classifiers are described in :pep:`301`. Examples:: @@ -297,7 +298,7 @@ Summary of Differences From PEP 241 * Metadata-Version is now 1.1. -* Added the Classifiers field from PEP 301. +* Added the Classifiers field from :pep:`301`. * The License and Platform files should now only be used if the platform or license can't be handled by an appropriate Classifier @@ -324,9 +325,6 @@ References .. [1] reStructuredText http://docutils.sourceforge.net/ -.. [2] PEP 301 - http://www.python.org/dev/peps/pep-0301/ - Copyright ========= diff --git a/pep-0315.txt b/peps/pep-0315.rst similarity index 100% rename from pep-0315.txt rename to peps/pep-0315.rst diff --git a/pep-0316.txt b/peps/pep-0316.rst similarity index 100% rename from pep-0316.txt rename to peps/pep-0316.rst diff --git a/pep-0317.txt b/peps/pep-0317.rst similarity index 96% rename from pep-0317.txt rename to peps/pep-0317.rst index f2a78ec1b..c2fcd32f2 100644 --- a/pep-0317.txt +++ b/peps/pep-0317.rst @@ -43,7 +43,7 @@ proposed implementation schedule, Python 2.4 will introduce warnings about uses of ``raise`` which will eventually become incorrect, and Python 3.0 will eliminate them entirely. (It is assumed that this transition period -- 2.4 to 3.0 -- will be at least one year long, to -comply with the guidelines of PEP 5 [2]_.) +comply with the guidelines of :pep:`5`.) Motivation @@ -184,7 +184,7 @@ Migration Plan Future Statement '''''''''''''''' -Under the future statement [4]_ :: +Under the :pep:`236` future statement:: from __future__ import raise_with_two_args @@ -200,7 +200,7 @@ simple exception raising does not require it. Warnings '''''''' -Three new warnings [5]_, all of category ``DeprecationWarning``, are +Three new :pep:`warnings <230>`, all of category ``DeprecationWarning``, are to be issued to point out uses of ``raise`` which will become incorrect under the proposed changes. @@ -433,18 +433,9 @@ References .. [1] "Standard Exception Classes in Python 1.5", Guido van Rossum. http://www.python.org/doc/essays/stdexceptions.html -.. [2] "Guidelines for Language Evolution", Paul Prescod. - http://www.python.org/dev/peps/pep-0005/ - .. [3] "Python Language Reference", Guido van Rossum. http://docs.python.org/reference/simple_stmts.html#raise -.. [4] PEP 236 "Back to the __future__", Tim Peters. - http://www.python.org/dev/peps/pep-0236/ - -.. [5] PEP 230 "Warning Framework", Guido van Rossum. - http://www.python.org/dev/peps/pep-0230/ - .. [6] Guido van Rossum, 11 June 2003 post to ``python-dev``. https://mail.python.org/pipermail/python-dev/2003-June/036176.html diff --git a/pep-0318.txt b/peps/pep-0318.rst similarity index 98% rename from pep-0318.txt rename to peps/pep-0318.rst index ea62e16a3..276a9c37a 100644 --- a/pep-0318.txt +++ b/peps/pep-0318.rst @@ -9,7 +9,7 @@ Content-Type: text/x-rst Created: 05-Jun-2003 Python-Version: 2.4 Post-History: 09-Jun-2003, 10-Jun-2003, 27-Feb-2004, 23-Mar-2004, 30-Aug-2004, - 2-Sep-2004 + 02-Sep-2004 WarningWarningWarning @@ -72,7 +72,7 @@ using metaclasses is sufficiently obscure that there is some attraction to having an easier way to make simple modifications to classes. For Python 2.4, only function/method decorators are being added. -PEP 3129 [#PEP-3129] proposes to add class decorators as of Python 2.6. +:pep:`3129` proposes to add class decorators as of Python 2.6. Why Is This So Hard? @@ -110,7 +110,7 @@ seem to be most divisive. * Syntax discussions in general appear to cause more contention than almost anything else. Readers are pointed to the ternary operator - discussions that were associated with PEP 308 for another example of + discussions that were associated with :pep:`308` for another example of this. @@ -836,7 +836,7 @@ syntactic support. .. _strong arguments: https://mail.python.org/pipermail/python-dev/2004-March/thread.html - PEP 3129 [#PEP-3129] proposes to add class decorators as of Python 2.6. + :pep:`3129` proposes to add class decorators as of Python 2.6. 2. The choice of the ``@`` character will be re-examined before Python 2.4b1. @@ -844,13 +844,6 @@ syntactic support. In the end, the ``@`` character was kept. -References -========== - -.. [#PEP-3129] PEP 3129, "Class Decorators", Winter - http://www.python.org/dev/peps/pep-3129 - - Copyright ========= diff --git a/pep-0319.txt b/peps/pep-0319.rst similarity index 96% rename from pep-0319.txt rename to peps/pep-0319.rst index 927e9dfc8..1c257cabe 100644 --- a/pep-0319.txt +++ b/peps/pep-0319.rst @@ -7,7 +7,7 @@ Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 24-Feb-2003 -Python-Version: 2.4? +Python-Version: 2.4 Post-History: @@ -20,7 +20,7 @@ and 'asynchronize'. Pronouncement ============= -This PEP is rejected in favor of PEP 343. +This PEP is rejected in favor of :pep:`343`. The 'synchronize' Keyword The concept of code synchronization in Python is too low-level. @@ -375,7 +375,8 @@ Backward Compatibility ====================== Backward compatibility is solved with the new ``from __future__`` -Python syntax [2]_, and the new warning framework [3]_ to evolve the +Python syntax (:pep:`236`), and the new warning framework (:pep:`230`) +to evolve the Python language into phasing out any conflicting names that use the new keywords 'synchronize' and 'asynchronize'. To use the syntax now, a developer could use the statement:: @@ -393,7 +394,7 @@ an exception. PEP 310 Reliable Acquisition/Release Pairs ========================================== -PEP 310 [4]_ proposes the 'with' keyword that can serve the same +:pep:`310` proposes the 'with' keyword that can serve the same function as 'synchronize' (but no facility for 'asynchronize'). The pattern:: @@ -407,7 +408,7 @@ is equivalent to the proposed:: synchronize the_lock: change_shared_data() -PEP 310 must synchronize on an existing lock, while this PEP +:pep:`310` must synchronize on an existing lock, while this PEP proposes that unqualified 'synchronize' statements synchronize on a global, internal, transparent lock in addition to qualified 'synchronize' statements. The 'with' statement also requires lock @@ -489,15 +490,6 @@ References .. [1] The Python Language Reference http://docs.python.org/reference/ -.. [2] PEP 236, Back to the __future__, Peters - http://www.python.org/dev/peps/pep-0236/ - -.. [3] PEP 230, Warning Framework, van Rossum - http://www.python.org/dev/peps/pep-0230/ - -.. [4] PEP 310, Reliable Acquisition/Release Pairs, Hudson, Moore - http://www.python.org/dev/peps/pep-0310/ - Copyright ========= diff --git a/pep-0320.txt b/peps/pep-0320.rst similarity index 87% rename from pep-0320.txt rename to peps/pep-0320.rst index f46bd47bb..d225ca92d 100644 --- a/pep-0320.txt +++ b/peps/pep-0320.rst @@ -5,10 +5,11 @@ Last-Modified: $Date$ Author: Barry Warsaw, Raymond Hettinger, Anthony Baxter Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 29-Jul-2003 Python-Version: 2.4 -Post-History: 1-Dec-2004 +Post-History: 01-Dec-2004 Abstract @@ -53,19 +54,19 @@ Release Schedule Completed features for 2.4 ========================== -- PEP 218 Builtin Set Objects. +- :pep:`218` Builtin Set Objects. -- PEP 289 Generator expressions. +- :pep:`289` Generator expressions. -- PEP 292 Simpler String Substitutions to be implemented as a module. +- :pep:`292` Simpler String Substitutions to be implemented as a module. -- PEP 318: Function/method decorator syntax, using @syntax +- :pep:`318`: Function/method decorator syntax, using @syntax -- PEP 322 Reverse Iteration. +- :pep:`322` Reverse Iteration. -- PEP 327: A Decimal package for fixed precision arithmetic. +- :pep:`327`: A Decimal package for fixed precision arithmetic. -- PEP 328: Multi-line Imports +- :pep:`328`: Multi-line Imports - Encapsulate the decorate-sort-undecorate pattern in a keyword for ``list.sort()``. @@ -85,10 +86,10 @@ Completed features for 2.4 Deferred until 2.5 ================== -- Deprecate and/or remove the modules listed in PEP 4 (``posixfile``, +- Deprecate and/or remove the modules listed in :pep:`4` (``posixfile``, ``gopherlib``, ``pre``, ``others``) -- Remove support for platforms as described in PEP 11. +- Remove support for platforms as described in :pep:`11`. - Finish implementing the Distutils ``bdist_dpkg`` command. (AMK) @@ -161,7 +162,7 @@ Carryover features from Python 2.3 widgets in Tk 8.4? Note that we've got better Tix support already (though not on Windows yet). -- PEP 304 (Controlling Generation of Bytecode Files by Montanaro) +- :pep:`304` (Controlling Generation of Bytecode Files by Montanaro) seems to have lost steam. - For a class defined inside another class, the ``__name__`` should be @@ -182,22 +183,22 @@ Carryover features from Python 2.3 covered yet (e.g. ``string.whitespace`` and ``types.TracebackType``). It seems we can't get consensus on this. -- PEP 262 Database of Installed Python Packages (Kuchling) +- :pep:`262` Database of Installed Python Packages (Kuchling) This turns out to be useful for Jack Jansen's Python installer, so the database is worth implementing. Code will go in sandbox/pep262. -- PEP 269 Pgen Module for Python (Riehl) +- :pep:`269` Pgen Module for Python (Riehl) (Some necessary changes are in; the ``pgen`` module itself needs to mature more.) -- PEP 266 Optimizing Global Variable/Attribute Access (Montanaro) +- :pep:`266` Optimizing Global Variable/Attribute Access (Montanaro) - PEP 267 Optimized Access to Module Namespaces (Hylton) + :pep:`267` Optimized Access to Module Namespaces (Hylton) - PEP 280 Optimizing access to globals (van Rossum) + :pep:`280` Optimizing access to globals (van Rossum) These are basically three friendly competing proposals. Jeremy has made a little progress with a new compiler, but it's going @@ -208,7 +209,7 @@ Carryover features from Python 2.3 - Lazily tracking tuples? [6]_ [7]_ Not much enthusiasm I believe. -- PEP 286 Enhanced Argument Tuples (von Loewis) +- :pep:`286` Enhanced Argument Tuples (von Loewis) I haven't had the time to review this thoroughly. It seems a deep optimization hack (also makes better correctness guarantees diff --git a/pep-0321.txt b/peps/pep-0321.rst similarity index 98% rename from pep-0321.txt rename to peps/pep-0321.rst index 6ff32db88..28199dde5 100644 --- a/pep-0321.txt +++ b/peps/pep-0321.rst @@ -35,7 +35,7 @@ Input Formats Useful formats to support include: * `ISO8601`_ -* ARPA/`RFC2822`_ +* ARPA/:rfc:`2822` * `ctime`_ * Formats commonly written by humans such as the American "MM/DD/YYYY", the European "YYYY/MM/DD", and variants such as @@ -92,7 +92,7 @@ Output Formats Not all input formats need to be supported as output formats, because it's pretty trivial to get the ``strftime()`` argument right for simple things -such as YYYY/MM/DD. Only complicated formats need to be supported; RFC2822 +such as YYYY/MM/DD. Only complicated formats need to be supported; :rfc:`2822` is currently the only one I can think of. Options: @@ -117,8 +117,6 @@ http://simon.incutio.com/archive/2003/10/07/dateInPython) References ========== -.. _RFC2822: http://rfc2822.x42.com - .. _ISO8601: http://www.cl.cam.ac.uk/~mgk25/iso-time.html .. _ParseDate.pm: http://search.cpan.org/author/MUIR/Time-modules-2003.0211/lib/Time/ParseDate.pm diff --git a/pep-0322.txt b/peps/pep-0322.rst similarity index 100% rename from pep-0322.txt rename to peps/pep-0322.rst diff --git a/pep-0323.txt b/peps/pep-0323.rst similarity index 97% rename from pep-0323.txt rename to peps/pep-0323.rst index 632146be8..5cb2249b0 100644 --- a/pep-0323.txt +++ b/peps/pep-0323.rst @@ -252,7 +252,7 @@ iterkeys, itervalues, and iteritems of built-in type dict. Iterators produced by generator functions will not be copyable. However, iterators produced by the new "generator expressions" of -Python 2.4 (PEP 289 [3]_) should be copyable if their underlying +Python 2.4 (:pep:`289`) should be copyable if their underlying ``iterator[s]`` are; the strict limitations on what is possible in a generator expression, compared to the much vaster generality of a generator, should make that feasible. Similarly, the iterators @@ -475,25 +475,14 @@ specific iterator type at the same time. References ========== -.. [1] Discussion on python-dev starting at post: - https://mail.python.org/pipermail/python-dev/2003-October/038969.html +[1] Discussion on python-dev starting at post: +\ https://mail.python.org/pipermail/python-dev/2003-October/038969.html -.. [2] Online documentation for the copy module of the standard library: - http://docs.python.org/library/copy.html +[2] Online documentation for the copy module of the standard library: +\ https://docs.python.org/release/2.6/library/copy.html -.. [3] PEP 289, Generator Expressions, Hettinger - http://www.python.org/dev/peps/pep-0289/ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0324.txt b/peps/pep-0324.rst similarity index 100% rename from pep-0324.txt rename to peps/pep-0324.rst diff --git a/pep-0325.txt b/peps/pep-0325.rst similarity index 94% rename from pep-0325.txt rename to peps/pep-0325.rst index 7dcb2132b..5bdf8cc78 100644 --- a/pep-0325.txt +++ b/peps/pep-0325.rst @@ -31,7 +31,7 @@ generators. Pronouncement ============= -Rejected in favor of PEP 342 which includes substantially all of +Rejected in favor of :pep:`342` which includes substantially all of the requested behavior in a more refined form. @@ -128,7 +128,7 @@ with such semantics that the example could be rewritten as:: finally: all.close() # close on generator -Currently PEP 255 [1]_ disallows yield inside a try clause of a +Currently :pep:`255` disallows yield inside a try clause of a try-finally statement, because the execution of the finally clause cannot be guaranteed as required by try-finally semantics. @@ -264,24 +264,14 @@ clauses has been proposed more than once. Alone it cannot guarantee that timely release of resources acquired by a generator can be enforced. -PEP 288 [2]_ proposes a more general solution, allowing custom +:pep:`288` proposes a more general solution, allowing custom exception passing to generators. The proposal in this PEP -addresses more directly the problem of resource release. Were PEP -288 implemented, Exceptions Semantics for close could be layered -on top of it, on the other hand PEP 288 should make a separate +addresses more directly the problem of resource release. Were +:pep:`288` implemented, Exceptions Semantics for close could be layered +on top of it, on the other hand :pep:`288` should make a separate case for the more general functionality. -References -========== - -.. [1] PEP 255 Simple Generators - http://www.python.org/dev/peps/pep-0255/ - -.. [2] PEP 288 Generators Attributes and Exceptions - http://www.python.org/dev/peps/pep-0288/ - - Copyright ========= diff --git a/pep-0326.txt b/peps/pep-0326.rst similarity index 92% rename from pep-0326.txt rename to peps/pep-0326.rst index 63445c1a1..20ddc6f12 100644 --- a/pep-0326.txt +++ b/peps/pep-0326.rst @@ -15,10 +15,10 @@ Post-History: 20-Dec-2003, 03-Jan-2004, 05-Jan-2004, 07-Jan-2004, Results ======= -This PEP has been rejected by the BDFL [12]_. As per the -pseudo-sunset clause [13]_, PEP 326 is being updated one last time +This PEP has been rejected by the BDFL [8]_. As per the +pseudo-sunset clause [9]_, :pep:`326` is being updated one last time with the latest suggestions, code modifications, etc., and includes a -link to a module [14]_ that implements the behavior described in the +link to a module [10]_ that implements the behavior described in the PEP. Users who desire the behavior listed in this PEP are encouraged to use the module for the reasons listed in `Independent Implementations?`_. @@ -177,7 +177,7 @@ potential problems with numeric overflows. .. _DijkstraSP_table_node: -Gustavo Niemeyer [9]_ points out that using a more Pythonic data +Gustavo Niemeyer [7]_ points out that using a more Pythonic data structure than tuples, to store information about node distances, increases readability. Two equivalent node structures (one using ``None``, the other using ``Max``) and their use in a suitably @@ -253,7 +253,7 @@ A ``Min`` Example ----------------- An example of usage for ``Min`` is an algorithm that solves the -following problem [6]_: +following problem [5]_: Suppose you are given a directed graph, representing a communication network. The vertices are the nodes in the network, @@ -268,7 +268,7 @@ following problem [6]_: and ``B``. Such an algorithm is a 7 line modification to the `DijkstraSP_table`_ -algorithm given above (modified lines prefixed with `*`):: +algorithm given above (modified lines prefixed with ``*``):: def DijkstraSP_table(graph, S, T): table = {} #3 @@ -305,7 +305,7 @@ translations are left as an exercise to the reader. Other Examples -------------- -Andrew P. Lentvorski, Jr. [7]_ has pointed out that various data +Andrew P. Lentvorski, Jr. [6]_ has pointed out that various data structures involving range searching have immediate use for ``Max`` and ``Min`` values. More specifically; Segment trees, Range trees, k-d trees and database keys: @@ -365,7 +365,7 @@ seek to show how inconsistent they can be. could result in incorrect output by passing in secondary versions of ``Max``. -It has been pointed out [9]_ that the reference implementation given +It has been pointed out [7]_ that the reference implementation given below would be incompatible with independent implementations of ``Max``/``Min``. The point of this PEP is for the introduction of "The One True Implementation" of "The One True Maximum" and "The One @@ -445,36 +445,27 @@ References .. [4] [Python-Dev] Re: PEP 326 now online, Reedy, Terry (https://mail.python.org/pipermail/python-dev/2004-January/041685.html) -.. [5] [Python-Dev] PEP 326 now online, Chermside, Michael - (https://mail.python.org/pipermail/python-dev/2004-January/041704.html) - -.. [6] Homework 6, Problem 7, Dillencourt, Michael +.. [5] Homework 6, Problem 7, Dillencourt, Michael (link may not be valid in the future) (http://www.ics.uci.edu/~dillenco/ics161/hw/hw6.pdf) -.. [7] RE: [Python-Dev] PEP 326 now online, Lentvorski, Andrew P., Jr. +.. [6] RE: [Python-Dev] PEP 326 now online, Lentvorski, Andrew P., Jr. (https://mail.python.org/pipermail/python-dev/2004-January/041727.html) -.. [8] Re: It's not really Some is it?, Ippolito, Bob - (http://www.livejournal.com/users/chouyu_31/138195.html?thread=274643#t274643) - -.. [9] [Python-Dev] Re: PEP 326 now online, Niemeyer, Gustavo +.. [7] [Python-Dev] Re: PEP 326 now online, Niemeyer, Gustavo (https://mail.python.org/pipermail/python-dev/2004-January/042261.html); [Python-Dev] Re: PEP 326 now online, Carlson, Josiah (https://mail.python.org/pipermail/python-dev/2004-January/042272.html) -.. [11] [Python-Dev] PEP 326 (quick location possibility), Carlson, Josiah - (https://mail.python.org/pipermail/python-dev/2004-January/042275.html) - -.. [12] [Python-Dev] PEP 326 (quick location possibility), van Rossum, Guido +.. [8] [Python-Dev] PEP 326 (quick location possibility), van Rossum, Guido (https://mail.python.org/pipermail/python-dev/2004-January/042306.html) -.. [13] [Python-Dev] PEP 326 (quick location possibility), Carlson, Josiah +.. [9] [Python-Dev] PEP 326 (quick location possibility), Carlson, Josiah (https://mail.python.org/pipermail/python-dev/2004-January/042300.html) -.. [14] Recommended standard implementation of PEP 326, extremes.py, +.. [10] Recommended standard implementation of PEP 326, extremes.py, Carlson, Josiah - (http://www.ics.uci.edu/~jcarlson/pep326/extremes.py) + (https://web.archive.org/web/20040410135029/http://www.ics.uci.edu:80/~jcarlson/pep326/extremes.py) Changes @@ -508,19 +499,10 @@ Changes - Added some `References`_. -- BDFL rejects [12]_ PEP 326 +- BDFL rejects [8]_ :pep:`326` Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0327.txt b/peps/pep-0327.rst similarity index 99% rename from pep-0327.txt rename to peps/pep-0327.rst index 7ca5584fb..17c63de1b 100644 --- a/pep-0327.txt +++ b/peps/pep-0327.rst @@ -137,7 +137,7 @@ Quoting Alex Martelli: truly major use case in the real world. Anyway, if you're interested in this data type, you maybe will want to -take a look at PEP 239: Adding a Rational Type to Python. +take a look at :pep:`239`: Adding a Rational Type to Python. So, what do we have? diff --git a/pep-0328.txt b/peps/pep-0328.rst similarity index 99% rename from pep-0328.txt rename to peps/pep-0328.rst index cc3378ef3..c2d4d28ab 100644 --- a/pep-0328.txt +++ b/peps/pep-0328.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 21-Dec-2003 Python-Version: 2.4, 2.5, 2.6 -Post-History: 8-Mar-2004 +Post-History: 08-Mar-2004 Abstract diff --git a/pep-0329.txt b/peps/pep-0329.rst similarity index 86% rename from pep-0329.txt rename to peps/pep-0329.rst index 50d76f479..b59e9c588 100644 --- a/pep-0329.txt +++ b/peps/pep-0329.rst @@ -49,14 +49,14 @@ of performance. There are currently over a hundred instances of ``while 1`` in the library. They were not replaced with the more readable ``while True`` because of performance reasons (the compiler cannot eliminate the test -because `True` is not known to always be a constant). Conversion of +because ``True`` is not known to always be a constant). Conversion of True to a constant will clarify the code while retaining performance. Many other basic Python operations run much slower because of global lookups. In try/except statements, the trapped exceptions are dynamically looked up before testing whether they match. Similarly, simple identity tests such as ``while x is not None`` -require the `None` variable to be re-looked up on every pass. +require the ``None`` variable to be re-looked up on every pass. Builtin lookups are especially egregious because the enclosing global scope must be checked first. These lookup chains devour cache space that is best used elsewhere. @@ -69,7 +69,7 @@ Proposal ======== Add a module called codetweaks.py which contains two functions, -`bind_constants()` and `bind_all()`. The first function performs +``bind_constants()`` and ``bind_all()``. The first function performs constant binding and the second recursively applies it to every function and class in a target module. @@ -80,7 +80,7 @@ the end of the script:: codetweaks.bind_all(sys.modules[__name__]) In addition to binding builtins, there are some modules (like -`sre_compile`) where it also makes sense to bind module variables +``sre_compile``) where it also makes sense to bind module variables as well as builtins into constants. @@ -97,17 +97,17 @@ Questions and Answers Every function has attributes with its bytecodes (the language of the Python virtual machine) and a table of constants. The bind - function scans the bytecodes for a `LOAD_GLOBAL` instruction and + function scans the bytecodes for a ``LOAD_GLOBAL`` instruction and checks to see whether the value is already known. If so, it adds that value to the constants table and replaces the opcode with - `LOAD_CONSTANT`. + ``LOAD_CONSTANT``. 3. When does it work? When a module is imported for the first time, python compiles the bytecode and runs the binding optimization. Subsequent imports just re-use the previous work. Each session repeats this process - (the results are not saved in `pyc` files). + (the results are not saved in ``pyc`` files). 4. How do you know this works? @@ -117,35 +117,35 @@ Questions and Answers 5. What if the module defines a variable shadowing a builtin? This does happen. For instance, True can be redefined at the module - level as `True = (1==1)`. The sample implementation below detects the + level as ``True = (1==1)``. The sample implementation below detects the shadowing and leaves the global lookup unchanged. 6. Are you the first person to recognize that most global lookups are for values that never change? No, this has long been known. Skip Montanaro provides an eloquent - explanation in [1]_. + explanation in :pep:`266`. 7. What if I want to replace the builtins module and supply my own implementations? Either do this before importing a module, or just reload the - module, or disable `codetweaks.py` (it will have a disable flag). + module, or disable ``codetweaks.py`` (it will have a disable flag). 8. How susceptible is this module to changes in Python's byte coding? - It imports `opcode.py` to protect against renumbering. Also, it - uses `LOAD_CONST` and `LOAD_GLOBAL` which are fundamental and have + It imports ``opcode.py`` to protect against renumbering. Also, it + uses ``LOAD_CONST`` and ``LOAD_GLOBAL`` which are fundamental and have been around forever. That notwithstanding, the coding scheme could change and this implementation would have to change along with - modules like `dis` which also rely on the current coding scheme. + modules like ``dis`` which also rely on the current coding scheme. 9. What is the effect on startup time? I could not measure a difference. None of the startup modules are bound except for warnings.py. Also, the binding function is very fast, making just a single pass over the code string in search of - the `LOAD_GLOBAL` opcode. + the ``LOAD_GLOBAL`` opcode. Sample Implementation @@ -244,7 +244,7 @@ Here is a sample implementation for codetweaks.py:: Note the automatic detection of a non-CPython environment that does not -have bytecodes [3]_. In that situation, the bind functions would simply +have bytecodes [2]_. In that situation, the bind functions would simply return the original function unchanged. This assures that the two line additions to library modules do not impact other implementations. @@ -255,26 +255,13 @@ The final code should add a flag to make it easy to disable binding. References ========== -.. [1] Optimizing Global Variable/Attribute Access - http://www.python.org/dev/peps/pep-0266/ +[1] ASPN Recipe for a non-private implementation +\ https://code.activestate.com/recipes/277940/ -.. [2] ASPN Recipe for a non-private implementation - http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/277940 - -.. [3] Differences between CPython and Jython - http://www.jython.org/cgi-bin/faqw.py?req=show&file=faq01.003.htp +.. [2] Differences between CPython and Jython + https://web.archive.org/web/20031018014238/http://www.jython.org/cgi-bin/faqw.py?req=show&file=faq01.003.htp Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0330.txt b/peps/pep-0330.rst similarity index 99% rename from pep-0330.txt rename to peps/pep-0330.rst index d377bbfed..dc5c116da 100644 --- a/pep-0330.txt +++ b/peps/pep-0330.rst @@ -7,7 +7,7 @@ Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 17-Jun-2004 -Python-Version: 2.6? +Python-Version: 2.6 Post-History: diff --git a/pep-0331.txt b/peps/pep-0331.rst similarity index 98% rename from pep-0331.txt rename to peps/pep-0331.rst index 002b34000..4b1ffc990 100644 --- a/pep-0331.txt +++ b/peps/pep-0331.rst @@ -172,9 +172,6 @@ Löwis on 2004-06-08, as stated in the bug report. References ========== -.. [1] PEP 1, PEP Purpose and Guidelines, Warsaw, Hylton - http://www.python.org/dev/peps/pep-0001/ - .. [2] Python locale documentation for embedding, http://docs.python.org/library/locale.html diff --git a/pep-0332.txt b/peps/pep-0332.rst similarity index 100% rename from pep-0332.txt rename to peps/pep-0332.rst diff --git a/pep-0333.txt b/peps/pep-0333.rst similarity index 97% rename from pep-0333.txt rename to peps/pep-0333.rst index 37583fb9d..e57f89254 100644 --- a/pep-0333.txt +++ b/peps/pep-0333.rst @@ -3,7 +3,7 @@ Title: Python Web Server Gateway Interface v1.0 Version: $Revision$ Last-Modified: $Date$ Author: Phillip J. Eby -Discussions-To: Python Web-SIG +Discussions-To: web-sig@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -17,7 +17,7 @@ Preface Note: For an updated version of this spec that supports Python 3.x and includes community errata, addenda, and clarifications, please -see PEP 3333 instead. +see :pep:`3333` instead. Abstract @@ -477,7 +477,7 @@ If the iterable returned by the application has a ``close()`` method, the server or gateway **must** call that method upon completion of the current request, whether the request was completed normally, or terminated early due to an error (this is to support resource release -by the application). This protocol is intended to complement PEP 325's +by the application). This protocol is intended to complement :pep:`325`'s generator support, and other common iterables with ``close()`` methods. (Note: the application **must** invoke the ``start_response()`` @@ -492,7 +492,7 @@ attributes of the iterable returned by the application, unless it is an instance of a type specific to that server or gateway, such as a "file wrapper" returned by ``wsgi.file_wrapper`` (see `Optional Platform-Specific File Handling`_). In the general case, only -attributes specified here, or accessed via e.g. the PEP 234 iteration +attributes specified here, or accessed via e.g. the :pep:`234` iteration APIs are acceptable. @@ -560,7 +560,7 @@ unless their value would be an empty string, in which case they A server or gateway **should** attempt to provide as many other CGI variables as are applicable. In addition, if SSL is in use, the server or gateway **should** also provide as many of the Apache SSL environment -variables [5]_ as are applicable, such as ``HTTPS=on`` and +variables [3]_ as are applicable, such as ``HTTPS=on`` and ``SSL_PROTOCOL``. Note, however, that an application that uses any CGI variables other than the ones listed above are necessarily non-portable to web servers that do not support the relevant extensions. (For @@ -596,9 +596,9 @@ Variable Value ``wsgi.input`` An input stream (file-like object) from which the HTTP request body can be read. (The server or gateway may perform reads on-demand as - requested by the application, or it may pre- - read the client's request body and buffer it - in-memory or on disk, or use any other + requested by the application, or it may + pre-read the client's request body and buffer + it in-memory or on disk, or use any other technique for providing such an input stream, according to its preference.) @@ -713,7 +713,7 @@ The ``status`` argument is an HTTP "status" string like ``"200 OK"`` or ``"404 Not Found"``. That is, it is a string consisting of a Status-Code and a Reason-Phrase, in that order and separated by a single space, with no surrounding whitespace or other characters. -(See RFC 2616, Section 6.1.1 for more information.) The string +(See :rfc:`2616`, Section 6.1.1 for more information.) The string **must not** contain control characters, and must not be terminated with a carriage return, linefeed, or combination thereof. @@ -721,7 +721,7 @@ The ``response_headers`` argument is a list of ``(header_name, header_value)`` tuples. It must be a Python list; i.e. ``type(response_headers) is ListType``, and the server **may** change its contents in any way it desires. Each ``header_name`` must be a -valid HTTP header field-name (as defined by RFC 2616, Section 4.2), +valid HTTP header field-name (as defined by :rfc:`2616`, Section 4.2), without a trailing colon or other punctuation. Each ``header_value`` **must not** include *any* control characters, @@ -832,12 +832,13 @@ whose ``len()`` is 1, then the server can automatically determine ``Content-Length`` by taking the length of the first string yielded by the iterable. -And, if the server and client both support HTTP/1.1 "chunked -encoding" [3]_, then the server **may** use chunked encoding to send +And, if the server and client both support HTTP/1.1 +:rfc:`"chunked encoding"<2616#section-3.6.1>`, +then the server **may** use chunked encoding to send a chunk for each ``write()`` call or string yielded by the iterable, thus generating a ``Content-Length`` header for each chunk. This allows the server to keep the client connection alive, if it wishes -to do so. Note that the server **must** comply fully with RFC 2616 +to do so. Note that the server **must** comply fully with :rfc:`2616` when doing this, or else fall back to one of the other strategies for dealing with the absence of ``Content-Length``. @@ -984,8 +985,8 @@ strings, not Unicode objects. The result of using a Unicode object where a string object is required, is undefined. Note also that strings passed to ``start_response()`` as a status or -as response headers **must** follow RFC 2616 with respect to encoding. -That is, they must either be ISO-8859-1 characters, or use RFC 2047 +as response headers **must** follow :rfc:`2616` with respect to encoding. +That is, they must either be ISO-8859-1 characters, or use :rfc:`2047` MIME encoding. On Python platforms where the ``str`` or ``StringType`` type is in @@ -1086,8 +1087,8 @@ may be done in any of several ways: Note that these behavior restrictions do not apply for HTTP 1.0 requests, or for requests that are not directed to an application -object. For more information on HTTP 1.1 Expect/Continue, see RFC -2616, sections 8.2.3 and 10.1.1. +object. For more information on HTTP 1.1 Expect/Continue, see +:rfc:`2616`, sections 8.2.3 and 10.1.1. Other HTTP Features @@ -1100,13 +1101,14 @@ response. It is always possible for the application developer to add middleware components to supply additional features, so server/gateway developers should be conservative in their implementation. In a sense, a server should consider itself to be like an HTTP "gateway server", -with the application being an HTTP "origin server". (See RFC 2616, +with the application being an HTTP "origin server". (See :rfc:`2616`, section 1.3, for the definition of these terms.) However, because WSGI servers and applications do not communicate via -HTTP, what RFC 2616 calls "hop-by-hop" headers do not apply to WSGI +HTTP, what :rfc:`2616` calls "hop-by-hop" headers do not apply to WSGI internal communications. WSGI applications **must not** generate any -"hop-by-hop" headers [4]_, attempt to use HTTP features that would +:rfc:`"hop-by-hop" headers <2616#section-13.5.1>`, +attempt to use HTTP features that would require them to generate such headers, or rely on the content of any incoming "hop-by-hop" headers in the ``environ`` dictionary. WSGI servers **must** handle any supported inbound "hop-by-hop" headers @@ -1302,7 +1304,7 @@ simply restrict themselves to using only a standard "for" loop to iterate over any iterable returned by an application. This is the only way to ensure source-level compatibility with both the pre-2.2 iterator protocol (discussed further below) and "today's" iterator -protocol (see PEP 234). +protocol (see :pep:`234`). (Note that this technique necessarily applies only to servers, gateways, or middleware that are written in Python. Discussion of @@ -1339,8 +1341,8 @@ use only language features available in the target version, use Optional Platform-Specific File Handling ---------------------------------------- -Some operating environments provide special high-performance file- -transmission facilities, such as the Unix ``sendfile()`` call. +Some operating environments provide special high-performance +file-transmission facilities, such as the Unix ``sendfile()`` call. Servers and gateways **may** expose this functionality via an optional ``wsgi.file_wrapper`` key in the ``environ``. An application **may** use this "file wrapper" to convert a file or file-like object @@ -1479,7 +1481,7 @@ Questions and Answers iterable is garbage collected. The ``close()`` idiom allows an application to release critical resources at the end of a request, and it's forward-compatible with the support for try/finally in - generators that's proposed by PEP 325. + generators that's proposed by :pep:`325`. 4. Why is this interface so low-level? I want feature X! (e.g. cookies, sessions, persistence, ...) @@ -1636,15 +1638,9 @@ References (http://www.python.org/cgi-bin/moinmoin/WebProgramming) .. [2] The Common Gateway Interface Specification, v 1.1, 3rd Draft - (http://ken.coar.org/cgi/draft-coar-cgi-v11-03.txt) + (https://datatracker.ietf.org/doc/html/draft-coar-cgi-v11-03) -.. [3] "Chunked Transfer Coding" -- HTTP/1.1, section 3.6.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1) - -.. [4] "End-to-end and Hop-by-hop Headers" -- HTTP/1.1, Section 13.5.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1) - -.. [5] mod_ssl Reference, "Environment Variables" +.. [3] mod_ssl Reference, "Environment Variables" (http://www.modssl.org/docs/2.8/ssl_reference.html#ToC25) diff --git a/pep-0334.txt b/peps/pep-0334.rst similarity index 97% rename from pep-0334.txt rename to peps/pep-0334.rst index 0fcd2e914..519924a6c 100644 --- a/pep-0334.txt +++ b/peps/pep-0334.rst @@ -25,7 +25,7 @@ help the application developer grapple with this state management difficulty. This PEP proposes a limited approach to coroutines based on an -extension to the iterator protocol [5]_. Currently, an iterator may +extension to the :pep:`iterator protocol <234>`. Currently, an iterator may raise a StopIteration exception to indicate that it is done producing values. This proposal adds another exception to this protocol, SuspendIteration, which indicates that the given iterator may have @@ -62,7 +62,7 @@ portable across runtimes. There are four places where this new exception impacts: -* The simple generator [8]_ mechanism could be extended to safely +* The :pep:`255` simple generator mechanism could be extended to safely 'catch' this SuspendIteration exception, stuff away its current state, and pass the exception on to the caller. @@ -381,18 +381,12 @@ References .. [4] Coroutines (http://c2.com/cgi/wiki?CallWithCurrentContinuation) -.. [5] PEP 234, Iterators - (http://www.python.org/dev/peps/pep-0234/) - .. [6] Stackless Python (http://stackless.com) .. [7] Parrot /w coroutines (http://www.sidhe.org/~dan/blog/archives/000178.html) -.. [8] PEP 255, Simple Generators - (http://www.python.org/dev/peps/pep-0255/) - .. [9] itertools - Functions creating iterators (http://docs.python.org/library/itertools.html) diff --git a/pep-0335.txt b/peps/pep-0335.rst similarity index 100% rename from pep-0335.txt rename to peps/pep-0335.rst diff --git a/pep-0336.txt b/peps/pep-0336.rst similarity index 88% rename from pep-0336.txt rename to peps/pep-0336.rst index e7953211d..c2bfc0d7f 100644 --- a/pep-0336.txt +++ b/peps/pep-0336.rst @@ -24,7 +24,7 @@ This PEP is rejected. It is considered a feature that ``None`` raises an error when called. The proposal falls short in tests for obviousness, clarity, explicitness, and necessity. The provided Switch example is nice but easily handled by a simple lambda definition. -See python-dev discussion on 17 June 2005 [2]_. +See python-dev discussion on 17 June 2005 [1]_. Motivation @@ -119,10 +119,7 @@ After:: References ========== -.. [1] Python Reference Manual, Section 3.2, - http://docs.python.org/reference/ - -.. [2] Raymond Hettinger, Propose to reject PEP 336 -- Make None Callable +.. [1] Raymond Hettinger, Propose to reject PEP 336 -- Make None Callable https://mail.python.org/pipermail/python-dev/2005-June/054280.html @@ -130,13 +127,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0337.txt b/peps/pep-0337.rst similarity index 96% rename from pep-0337.txt rename to peps/pep-0337.rst index 0f05b69ff..8c431d6aa 100644 --- a/pep-0337.txt +++ b/peps/pep-0337.rst @@ -14,7 +14,7 @@ Post-History: 10-Nov-2004 Abstract ======== -This PEP defines a standard for using the logging system (PEP 282 [1]_) in the +This PEP defines a standard for using the logging system (:pep:`282`) in the standard library. Implementing this PEP will simplify development of daemon @@ -186,9 +186,6 @@ name "dummy.junk". References ========== -.. [1] PEP 282, A Logging System, Vinay Sajip, Trent Mick - http://www.python.org/dev/peps/pep-0282/ - .. [2] https://mail.python.org/pipermail/python-dev/2004-October/049282.html diff --git a/pep-0338.txt b/peps/pep-0338.rst similarity index 93% rename from pep-0338.txt rename to peps/pep-0338.rst index 678b94e6b..81a665541 100644 --- a/pep-0338.txt +++ b/peps/pep-0338.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 16-Oct-2004 Python-Version: 2.5 -Post-History: 8-Nov-2004, 11-Feb-2006, 12-Feb-2006, 18-Feb-2006 +Post-History: 08-Nov-2004, 11-Feb-2006, 12-Feb-2006, 18-Feb-2006 Abstract @@ -19,7 +19,7 @@ script, either with the ``-m`` command line switch, or by invoking it via ``runpy.run_module(modulename)``. The ``-m`` switch implemented in Python 2.4 is quite limited. This -PEP proposes making use of the PEP 302 [4]_ import hooks to allow any +PEP proposes making use of the :pep:`302` import hooks to allow any module which provides access to its code object to be executed. Rationale @@ -68,7 +68,7 @@ mechanisms (such as ``zipimport``). Prior discussions suggest it should be noted that this PEP is **not** about changing the idiom for making Python modules also useful as -scripts (see PEP 299 [1]_). That issue is considered orthogonal to the +scripts (see :pep:`299`). That issue is considered orthogonal to the specific feature addressed by this PEP. Current Behaviour @@ -98,7 +98,7 @@ Proposed Semantics ================== The semantics proposed are fairly simple: if ``-m`` is used to execute -a module the PEP 302 import mechanisms are used to locate the module and +a module the :pep:`302` import mechanisms are used to locate the module and retrieve its compiled code, before executing the module in accordance with the semantics for a top-level module. The interpreter does this by invoking a new standard library function ``runpy.run_module``. @@ -109,7 +109,7 @@ variable during initialisation. In addition, paths may be affected by ``*.pth`` files, and some packages will install custom loaders on ``sys.metapath``. Accordingly, the only way for Python to reliably locate the module is by importing the containing package and -using the PEP 302 import hooks to gain access to the Python code. +using the :pep:`302` import hooks to gain access to the Python code. Note that the process of locating the module to be executed may require importing the containing package. The effects of such a package import @@ -140,7 +140,7 @@ The delegation has the form:: Execute the code of the specified module and return the resulting module globals dictionary. The module's code is first located using - the standard import mechanism (refer to PEP 302 for details) and + the standard import mechanism (refer to :pep:`302` for details) and then executed in a fresh module namespace. The optional dictionary argument ``init_globals`` may be used to @@ -156,7 +156,7 @@ The delegation has the form:: ``__name__`` is set to ``run_name`` if this optional argument is supplied, and the original ``mod_name`` argument otherwise. - ``__loader__`` is set to the PEP 302 module loader used to retrieve + ``__loader__`` is set to the :pep:`302` module loader used to retrieve the code for the module (This loader may be a wrapper around the standard import mechanism). @@ -184,7 +184,7 @@ Import Statements and the Main Module ===================================== The release of 2.5b1 showed a surprising (although obvious in -retrospect) interaction between this PEP and PEP 328 - explicit +retrospect) interaction between this PEP and :pep:`328` - explicit relative imports don't work from a main module. This is due to the fact that relative imports rely on ``__name__`` to determine the current module's position in the package hierarchy. In a main @@ -229,7 +229,7 @@ Here's an example file layout:: So long as the current directory is ``devel``, or ``devel`` is already on ``sys.path`` and the test modules use absolute imports (such as -``import pkg moduleA`` to retrieve the module under test, PEP 338 +``import pkg moduleA`` to retrieve the module under test, :pep:`338` allows the tests to be run as:: python -m pkg.test.test_A @@ -276,7 +276,7 @@ Python script with this behaviour can be found in the discussion of the ``execmodule`` cookbook recipe [3]_. The ``execmodule`` cookbook recipe itself was the proposed mechanism in -an earlier version of this PEP (before the PEP's author read PEP 302). +an earlier version of this PEP (before the PEP's author read :pep:`302`). Both approaches were rejected as they do not meet the main goal of the ``-m`` switch -- to allow the full Python namespace to be used to @@ -305,19 +305,13 @@ actual search for the module, and releases it before execution, even if References ========== -.. [1] Special __main__() function in modules - (http://www.python.org/dev/peps/pep-0299/) - -.. [2] PEP 338 implementation (runpy module and ``-m`` update) +.. [2] :pep:`338` implementation (runpy module and ``-m`` update) (https://bugs.python.org/issue1429601) .. [3] execmodule Python Cookbook Recipe (http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307772) -.. [4] New import hooks - (http://www.python.org/dev/peps/pep-0302/) - -.. [5] PEP 338 documentation (for runpy module) +.. [5] :pep:`338` documentation (for runpy module) (https://bugs.python.org/issue1429605) Copyright diff --git a/pep-0339.txt b/peps/pep-0339.rst similarity index 99% rename from pep-0339.txt rename to peps/pep-0339.rst index eefa01ad2..434a159c4 100644 --- a/pep-0339.txt +++ b/peps/pep-0339.rst @@ -83,7 +83,7 @@ To tie all of this example, consider the rule for 'while':: The node representing this will have ``TYPE(node) == while_stmt`` and the number of children can be 4 or 7 depending on if there is an 'else' statement. To access what should be the first ':' and require it be an -actual ':' token, `(REQ(CHILD(node, 2), COLON)``. +actual ':' token, ``(REQ(CHILD(node, 2), COLON)``. Abstract Syntax Trees (AST) @@ -550,7 +550,7 @@ References ---------- .. [Aho86] Alfred V. Aho, Ravi Sethi, Jeffrey D. Ullman. - `Compilers: Principles, Techniques, and Tools`, + ``Compilers: Principles, Techniques, and Tools``, http://www.amazon.com/exec/obidos/tg/detail/-/0201100886/104-0162389-6419108 .. [Wang97] Daniel C. Wang, Andrew W. Appel, Jeff L. Korn, and Chris @@ -564,7 +564,7 @@ References .. _SPARK: http://pages.cpsc.ucalgary.ca/~aycock/spark/ .. [#skip-peephole] Skip Montanaro's Peephole Optimizer Paper - (http://www.foretec.com/python/workshops/1998-11/proceedings/papers/montanaro/montanaro.html) + (https://legacy.python.org/workshops/1998-11/proceedings/papers/montanaro/montanaro.html) .. [#Bytecodehacks] Bytecodehacks Project (http://bytecodehacks.sourceforge.net/bch-docs/bch/index.html) diff --git a/pep-0340.txt b/peps/pep-0340.rst similarity index 95% rename from pep-0340.txt rename to peps/pep-0340.rst index f6739ee56..e6873a204 100644 --- a/pep-0340.txt +++ b/peps/pep-0340.rst @@ -17,9 +17,9 @@ used for resource management purposes. The new statement type is provisionally called the block-statement because the keyword to be used has not yet been chosen. -This PEP competes with several other PEPs: PEP 288 (Generators -Attributes and Exceptions; only the second part), PEP 310 -(Reliable Acquisition/Release Pairs), and PEP 325 +This PEP competes with several other PEPs: :pep:`288` (Generators +Attributes and Exceptions; only the second part), :pep:`310` +(Reliable Acquisition/Release Pairs), and :pep:`325` (Resource-Release Support for Generators). I should clarify that using a generator to "drive" a block @@ -30,14 +30,14 @@ turned into a template). But the key idea is using a generator to drive a block statement; the rest is elaboration, so I'd like to keep these two parts together. -(PEP 342, Enhanced Iterators, was originally a part of this PEP; +(:pep:`342`, Enhanced Iterators, was originally a part of this PEP; but the two proposals are really independent and with Steven Bethard's help I have moved it to a separate PEP.) Rejection Notice ================ -I am rejecting this PEP in favor of PEP 343. See the motivational +I am rejecting this PEP in favor of :pep:`343`. See the motivational section in that PEP for the reasoning behind this rejection. GvR. Motivation and Summary @@ -62,7 +62,7 @@ controlled code more than once (or not at all), catch exceptions, or receive data from the body of the block statement. A convenient way to write block iterators is to write a generator -(PEP 255). A generator looks a lot like a Python function, but +(:pep:`255`). A generator looks a lot like a Python function, but instead of returning a value immediately, generators pause their execution at "yield" statements. When a generator is used as a block iterator, the yield statement tells the Python interpreter @@ -115,7 +115,7 @@ assigned are still evaluated!). The choice of the 'block' keyword is contentious; many alternatives have been proposed, including not to use a keyword at -all (which I actually like). PEP 310 uses 'with' for similar +all (which I actually like). :pep:`310` uses 'with' for similar semantics, but I would like to reserve that for a with-statement similar to the one found in Pascal and VB. (Though I just found that the C# designers don't like 'with' [2]_, and I have to agree @@ -380,7 +380,7 @@ there a use for that capability here?" I believe there are definitely uses for this; several people have already shown how to do asynchronous light-weight threads using -generators (e.g. David Mertz quoted in PEP 288, and Fredrik +generators (e.g. David Mertz quoted in :pep:`288`, and Fredrik Lundh [3]_). And finally, Greg says: "a thunk implementation has the potential @@ -398,7 +398,7 @@ that defeats the purpose of using thunks in the first place.) Examples ======== -(Several of these examples contain "yield None". If PEP 342 is +(Several of these examples contain "yield None". If :pep:`342` is accepted, these can be changed to just "yield" of course.) 1. A template for ensuring that a lock, acquired at the start of a @@ -461,7 +461,7 @@ accepted, these can be changed to just "yield" of course.) Used as follows:: block auto_retry(3, IOError): - f = urllib.urlopen("http://www.python.org/dev/peps/pep-0340/") + f = urllib.urlopen("https://www.example.com/") print f.read() 5. It is possible to nest blocks and combine templates:: @@ -563,11 +563,11 @@ contributions! References ========== -.. [1] https://mail.python.org/pipermail/python-dev/2005-April/052821.html +[1] https://mail.python.org/pipermail/python-dev/2005-April/052821.html -.. [2] http://msdn.microsoft.com/vcsharp/programming/language/ask/withstatement/ +.. [2] https://web.archive.org/web/20060719195933/http://msdn.microsoft.com/vcsharp/programming/language/ask/withstatement/ -.. [3] http://effbot.org/zone/asyncore-generators.htm +.. [3] https://web.archive.org/web/20050204062901/http://effbot.org/zone/asyncore-generators.htm Copyright ========= diff --git a/pep-0341.txt b/peps/pep-0341.rst similarity index 99% rename from pep-0341.txt rename to peps/pep-0341.rst index daf3d2a58..f2e09dcaa 100644 --- a/pep-0341.txt +++ b/peps/pep-0341.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 04-May-2005 +Python-Version: 2.5 Post-History: diff --git a/pep-0342.txt b/peps/pep-0342.rst similarity index 97% rename from pep-0342.txt rename to peps/pep-0342.rst index 0c9763d6c..4570a1bd1 100644 --- a/pep-0342.txt +++ b/peps/pep-0342.rst @@ -19,13 +19,13 @@ make them usable as simple coroutines. It is basically a combination of ideas from these two PEPs, which may be considered redundant if this PEP is accepted: -- PEP 288, Generators Attributes and Exceptions. The current PEP covers its +- :pep:`288`, Generators Attributes and Exceptions. The current PEP covers its second half, generator exceptions (in fact the ``throw()`` method name was - taken from PEP 288). PEP 342 replaces generator attributes, however, with a - concept from an earlier revision of PEP 288, the *yield expression*. + taken from :pep:`288`). :pep:`342` replaces generator attributes, however, with a + concept from an earlier revision of :pep:`288`, the *yield expression*. -- PEP 325, Resource-Release Support for Generators. PEP 342 ties up a few - loose ends in the PEP 325 spec, to make it suitable for actual +- :pep:`325`, Resource-Release Support for Generators. :pep:`342` ties up a few + loose ends in the :pep:`325` spec, to make it suitable for actual implementation. @@ -314,7 +314,7 @@ the generator should be released whenever its execution is terminated due to an error or normal exit. This will ensure that generators that cannot be resumed do not remain part of an uncollectable reference cycle. This allows other code to potentially use ``close()`` in a ``try/finally`` or ``with`` -block (per PEP 343) to ensure that a given generator is properly finalized. +block (per :pep:`343`) to ensure that a given generator is properly finalized. Optional Extensions @@ -324,7 +324,7 @@ The Extended ``continue`` Statement ----------------------------------- An earlier draft of this PEP proposed a new ``continue EXPR`` syntax for use -in for-loops (carried over from PEP 340), that would pass the value of +in for-loops (carried over from :pep:`340`), that would pass the value of *EXPR* into the iterator being looped over. This feature has been withdrawn for the time being, because the scope of this PEP has been narrowed to focus only on passing values into generator-iterators, and not other kinds of @@ -344,7 +344,7 @@ reflects this preferred resolution. I originally chose ``TypeError`` because it represents gross misbehavior of the generator function, which should be fixed by changing the code. But the - ``with_template`` decorator class in PEP 343 uses ``RuntimeError`` for + ``with_template`` decorator class in :pep:`343` uses ``RuntimeError`` for similar offenses. Arguably they should all use the same exception. I'd rather not introduce a new exception class just for this purpose, since it's not an exception that I want people to catch: I want it to turn into a @@ -579,11 +579,11 @@ This patch was committed to CVS 01-02 August 2005. Acknowledgements ================ -Raymond Hettinger (PEP 288) and Samuele Pedroni (PEP 325) first formally +Raymond Hettinger (:pep:`288`) and Samuele Pedroni (:pep:`325`) first formally proposed the ideas of communicating values or exceptions into generators, and the ability to *close* generators. Timothy Delaney suggested the title of this PEP, and Steven Bethard helped edit a previous version. See also the -Acknowledgements section of PEP 340. +Acknowledgements section of :pep:`340`. References diff --git a/pep-0343.txt b/peps/pep-0343.rst similarity index 93% rename from pep-0343.txt rename to peps/pep-0343.rst index 787173cd2..8e18d8f79 100644 --- a/pep-0343.txt +++ b/peps/pep-0343.rst @@ -8,7 +8,8 @@ Type: Standards Track Content-Type: text/x-rst Created: 13-May-2005 Python-Version: 2.5 -Post-History: 2-Jun-2005, 16-Oct-2005, 29-Oct-2005, 23-Apr-2006, 1-May-2006, 30-Jul-2006 +Post-History: 02-Jun-2005, 16-Oct-2005, 29-Oct-2005, 23-Apr-2006, 01-May-2006, + 30-Jul-2006 Abstract ======== @@ -29,7 +30,7 @@ on python-dev. Any first person references are from Guido's original. Python's alpha release cycle revealed terminology problems in this -PEP and in the associated documentation and implementation [14]_. +PEP and in the associated documentation and implementation [13]_. The PEP stabilised around the time of the first Python 2.5 beta release. @@ -40,8 +41,8 @@ originally in the future are now in the past :) Introduction ============ -After a lot of discussion about PEP 340 and alternatives, I -decided to withdraw PEP 340 and proposed a slight variant on PEP +After a lot of discussion about :pep:`340` and alternatives, I +decided to withdraw :pep:`340` and proposed a slight variant on PEP 310. After more discussion, I have added back a mechanism for raising an exception in a suspended generator using a ``throw()`` method, and a ``close()`` method which throws a new GeneratorExit @@ -52,10 +53,10 @@ exception; these additions were first proposed on python-dev in After acceptance of this PEP, the following PEPs were rejected due to overlap: -- PEP 310, Reliable Acquisition/Release Pairs. This is the +- :pep:`310`, Reliable Acquisition/Release Pairs. This is the original with-statement proposal. -- PEP 319, Python Synchronize/Asynchronize Block. Its use cases +- :pep:`319`, Python Synchronize/Asynchronize Block. Its use cases can be covered by the current PEP by providing suitable with-statement controllers: for 'synchronize' we can use the "locking" template from example 1; for 'asynchronize' we can use @@ -64,7 +65,7 @@ to overlap: important; in fact it may be better to always be explicit about the mutex being used. -PEP 340 and PEP 346 also overlapped with this PEP, but were +:pep:`340` and :pep:`346` also overlapped with this PEP, but were voluntarily withdrawn when this PEP was submitted. Some discussion of earlier incarnations of this PEP took place on @@ -73,7 +74,7 @@ the Python Wiki [3]_. Motivation and Summary ====================== -PEP 340, Anonymous Block Statements, combined many powerful ideas: +:pep:`340`, Anonymous Block Statements, combined many powerful ideas: using generators as block templates, adding exception handling and finalization to generators, and more. Besides praise it received a lot of opposition from people who didn't like the fact that it @@ -86,16 +87,16 @@ But the final blow came when I read Raymond Chen's rant about flow-control macros [1]_. Raymond argues convincingly that hiding flow control in macros makes your code inscrutable, and I find that his argument applies to Python as well as to C. I realized -that PEP 340 templates can hide all sorts of control flow; for +that :pep:`340` templates can hide all sorts of control flow; for example, its example 4 (``auto_retry()``) catches exceptions and repeats the block up to three times. -However, the with-statement of PEP 310 does **not** hide control +However, the with-statement of :pep:`310` does **not** hide control flow, in my view: while a finally-suite temporarily suspends the control flow, in the end, the control flow resumes as if the finally-suite wasn't there at all. -Remember, PEP 310 proposes roughly this syntax (the "VAR =" part is +Remember, :pep:`310` proposes roughly this syntax (the "VAR =" part is optional):: with VAR = EXPR: @@ -129,13 +130,13 @@ other exceptions; the nature of exceptions is that they can happen write bug-free code, a KeyboardInterrupt exception can still cause it to exit between any two virtual machine opcodes.) -This argument almost led me to endorse PEP 310, but I had one idea -left from the PEP 340 euphoria that I wasn't ready to drop: using +This argument almost led me to endorse :pep:`310`, but I had one idea +left from the :pep:`340` euphoria that I wasn't ready to drop: using generators as "templates" for abstractions like acquiring and releasing a lock or opening and closing a file is a powerful idea, as can be seen by looking at the examples in that PEP. -Inspired by a counter-proposal to PEP 340 by Phillip Eby I tried +Inspired by a counter-proposal to :pep:`340` by Phillip Eby I tried to create a decorator that would turn a suitable generator into an object with the necessary ``__enter__()`` and ``__exit__()`` methods. Here I ran into a snag: while it wasn't too hard for the locking @@ -155,7 +156,7 @@ and used it like this:: with f = opening(filename): ...read data from f... -The problem is that in PEP 310, the result of calling ``EXPR`` is +The problem is that in :pep:`310`, the result of calling ``EXPR`` is assigned directly to ``VAR``, and then ``VAR``'s ``__exit__()`` method is called upon exit from ``BLOCK1``. But here, ``VAR`` clearly needs to receive the opened file, and that would mean that ``__exit__()`` would @@ -172,13 +173,13 @@ returns; the wrapper instance's ``__exit__()`` method calls ``next()`` again but expects it to raise StopIteration. (Details below in the section Optional Generator Decorator.) -So now the final hurdle was that the PEP 310 syntax:: +So now the final hurdle was that the :pep:`310` syntax:: with VAR = EXPR: BLOCK1 would be deceptive, since ``VAR`` does **not** receive the value of -``EXPR``. Borrowing from PEP 340, it was an easy step to:: +``EXPR``. Borrowing from :pep:`340`, it was an easy step to:: with EXPR as VAR: BLOCK1 @@ -217,7 +218,7 @@ not make the same guarantee. This applies to Jython, IronPython, and probably to Python running on Parrot. (The details of the changes made to generators can now be found in -PEP 342 rather than in the current PEP) +:pep:`342` rather than in the current PEP) Use Cases ========= @@ -306,7 +307,7 @@ as if the body of the generator were expanded in-line at the place of the with-statement. The motivation for passing the exception details to ``__exit__()``, as -opposed to the argument-less ``__exit__()`` from PEP 310, was given by +opposed to the argument-less ``__exit__()`` from :pep:`310`, was given by the ``transactional()`` use case, example 3 below. The template in that example must commit or roll back the transaction depending on whether an exception occurred or not. Rather than just having a @@ -361,7 +362,7 @@ and 'as' are always keywords. Generator Decorator =================== -With PEP 342 accepted, it is possible to write a decorator +With :pep:`342` accepted, it is possible to write a decorator that makes it possible to use a generator that yields exactly once to control a with-statement. Here's a sketch of such a decorator:: @@ -536,7 +537,7 @@ of any major objections on python-dev). 1. What exception should ``GeneratorContextManager`` raise when the underlying generator-iterator misbehaves? The following quote is the reason behind Guido's choice of ``RuntimeError`` for both this - and for the generator ``close()`` method in PEP 342 (from [8]_): + and for the generator ``close()`` method in :pep:`342` (from [8]_): "I'd rather not introduce a new exception class just for this purpose, since it's not an exception that I want people to catch: @@ -557,7 +558,7 @@ of any major objections on python-dev). 3. Objects with ``__enter__/__exit__`` methods are called "context managers" and the decorator to convert a generator function into a context manager factory is ``contextlib.contextmanager``. - There were some other suggestions [16]_ during the 2.5 release + There were some other suggestions [15]_ during the 2.5 release cycle but no compelling arguments for switching away from the terms that had been used in the PEP implementation were made. @@ -568,16 +569,16 @@ Rejected Options For several months, the PEP prohibited suppression of exceptions in order to avoid hidden flow control. Implementation revealed this to be a right royal pain, so Guido restored the -ability [13]_. +ability [12]_. Another aspect of the PEP that caused no end of questions and terminology debates was providing a ``__context__()`` method that was analogous to an iterable's ``__iter__()`` method [5]_ [7]_ [9]_. -The ongoing problems [10]_ [13]_ with explaining what it was and why +The ongoing problems [10]_ [12]_ with explaining what it was and why it was and how it was meant to work eventually lead to Guido -killing the concept outright [15]_ (and there was much rejoicing!). +killing the concept outright [14]_ (and there was much rejoicing!). -The notion of using the PEP 342 generator API directly to define +The notion of using the :pep:`342` generator API directly to define the with statement was also briefly entertained [6]_, but quickly dismissed as making it too difficult to write non-generator based context managers. @@ -586,7 +587,7 @@ based context managers. Examples ======== -The generator based examples rely on PEP 342. Also, some of the +The generator based examples rely on :pep:`342`. Also, some of the examples are unnecessary in practice, as the appropriate objects, such as ``threading.RLock``, are able to be used directly in with statements. @@ -820,7 +821,7 @@ refers to an action which is to be done in the ``__exit__`` method. (Python 2.5's contextlib module contains a version of this context manager) -11. PEP 319 gives a use case for also having a ``released()`` +11. :pep:`319` gives a use case for also having a ``released()`` context to temporarily release a previously acquired lock; this can be written very similarly to the locked context manager above by swapping the ``acquire()`` and ``release()`` calls:: @@ -906,8 +907,8 @@ Acknowledgements ================ Many people contributed to the ideas and concepts in this PEP, -including all those mentioned in the acknowledgements for PEP 340 -and PEP 346. +including all those mentioned in the acknowledgements for :pep:`340` +and :pep:`346`. Additional thanks goes to (in no meaningful order): Paul Moore, Phillip J. Eby, Greg Ewing, Jason Orendorff, Michael Hudson, @@ -952,32 +953,19 @@ References .. [11] Guido says AttributeError is fine for missing special methods https://mail.python.org/pipermail/python-dev/2005-October/057625.html -.. [12] Original PEP 342 implementation patch - http://sourceforge.net/tracker/index.php?func=detail&aid=1223381&group_id=5470&atid=305470 - -.. [13] Guido restores the ability to suppress exceptions +.. [12] Guido restores the ability to suppress exceptions https://mail.python.org/pipermail/python-dev/2006-February/061909.html -.. [14] A simple question kickstarts a thorough review of PEP 343 +.. [13] A simple question kickstarts a thorough review of PEP 343 https://mail.python.org/pipermail/python-dev/2006-April/063859.html -.. [15] Guido kills the __context__() method +.. [14] Guido kills the __context__() method https://mail.python.org/pipermail/python-dev/2006-April/064632.html -.. [16] Proposal to use 'context guard' instead of 'context manager' +.. [15] Proposal to use 'context guard' instead of 'context manager' https://mail.python.org/pipermail/python-dev/2006-May/064676.html Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0344.txt b/peps/pep-0344.rst similarity index 98% rename from pep-0344.txt rename to peps/pep-0344.rst index 5150f60df..de5fc298d 100644 --- a/pep-0344.txt +++ b/peps/pep-0344.rst @@ -14,7 +14,7 @@ Post-History: Numbering Note ============== -This PEP has been renumbered to PEP 3134. The text below is the last version +This PEP has been renumbered to :pep:`3134`. The text below is the last version submitted under the old number. @@ -70,7 +70,7 @@ original exception. Greg Ewing [4]_ and Guido van Rossum [5]_, and probably others, have previously mentioned adding a traceback attribute to ``Exception`` instances. -This is noted in PEP 3000. +This is noted in :pep:`3000`. This PEP was motivated by yet another recent Python-Dev reposting of the same ideas [6]_ [7]_. @@ -85,8 +85,8 @@ exception, the exception must be retained implicitly. To support intentional translation of an exception, there must be a way to chain exceptions explicitly. This PEP addresses both. -Several attribute names for chained exceptions have been suggested on Python- -Dev [2]_, including ``cause``, ``antecedent``, ``reason``, ``original``, +Several attribute names for chained exceptions have been suggested on +Python-Dev [2]_, including ``cause``, ``antecedent``, ``reason``, ``original``, ``chain``, ``chainedexc``, ``xc_chain``, ``excprev``, ``previous`` and ``precursor``. For an explicitly chained exception, this PEP suggests ``__cause__`` because of its specific meaning. For an implicitly chained @@ -463,7 +463,7 @@ Possible Future Compatible Changes These changes are consistent with the appearance of exceptions as a single object rather than a triple at the interpreter level. -- If PEP 340 or PEP 343 is accepted, replace the three (``type``, ``value``, +- If :pep:`340` or :pep:`343` is accepted, replace the three (``type``, ``value``, ``traceback``) arguments to ``__exit__`` with a single exception argument. - Deprecate ``sys.exc_type``, ``sys.exc_value``, ``sys.exc_traceback``, and diff --git a/pep-0345.txt b/peps/pep-0345.rst similarity index 92% rename from pep-0345.txt rename to peps/pep-0345.rst index 5f73afeda..a9a9f359b 100644 --- a/pep-0345.txt +++ b/peps/pep-0345.rst @@ -1,15 +1,17 @@ PEP: 345 Title: Metadata for Python Software Packages 1.2 -Version: $Revision$ -Last-Modified: $Date$ Author: Richard Jones -Discussions-To: Distutils SIG -Status: Final +Discussions-To: distutils-sig@python.org +Status: Superseded Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 28-Apr-2005 -Python-Version: 2.5 -Post-History: +Python-Version: 2.7 +Post-History: `22-Dec-2009 `__ +Replaces: 314 +Superseded-By: 566 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/MKHXVV746H7ZDFN62Z72VNAX6KIRXNRO/ Abstract @@ -20,8 +22,8 @@ It includes specifics of the field names, and their semantics and usage. This document specifies version 1.2 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. Version 1.2 of the metadata format adds a number of optional fields designed to make third-party packaging of Python Software easier. @@ -30,7 +32,7 @@ These fields are "Requires-Python", "Requires-External", "Requires-Dist", "Platform" field. Three new fields were also added: "Maintainer", "Maintainer-email" and "Project-URL". -Last, this new version also adds `environment markers`. +Last, this new version also adds ``environment markers``. Fields ====== @@ -68,7 +70,7 @@ Version ::::::: A string containing the distribution's version number. This -field must be in the format specified in PEP 440. +field must be in the format specified in :pep:`440`. Example:: @@ -127,15 +129,15 @@ field as-is. This means that authors should be conservative in the markup they use. To support empty lines and lines with indentation with respect to -the RFC 822 format, any CRLF character has to be suffixed by 7 spaces +the :rfc:`822` format, any CRLF character has to be suffixed by 7 spaces followed by a pipe ("|") char. As a result, the Description field is -encoded into a folded field that can be interpreted by RFC822 -parser [2]_. +encoded into a folded field that can be interpreted by :rfc:`822#section-3.1.1` +parser. Example:: Description: This project provides powerful math functions - |For example, you can use `sum()` to sum numbers: + |For example, you can use ``sum()`` to sum numbers: | |Example:: | @@ -145,7 +147,7 @@ Example:: This encoding implies that any occurrences of a CRLF followed by 7 spaces and a pipe char have to be replaced by a single CRLF when the field is unfolded -using a RFC822 reader. +using a :rfc:`822` reader. Keywords (optional) @@ -193,7 +195,7 @@ Author-email (optional) ::::::::::::::::::::::: A string containing the author's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` ``From:`` header. Example:: @@ -221,7 +223,7 @@ Maintainer-email (optional) ::::::::::::::::::::::::::: A string containing the maintainer's e-mail address. It can contain -a name and e-mail address in the legal forms for a RFC-822 +a name and e-mail address in the legal forms for a :rfc:`822` ``From:`` header. Note that this field is intended for use when a project is being @@ -255,7 +257,7 @@ Classifier (multiple use) ::::::::::::::::::::::::: Each entry is a string giving a single classification value -for the distribution. Classifiers are described in PEP 301 [3]_. +for the distribution. Classifiers are described in :pep:`301`. Examples:: @@ -371,7 +373,7 @@ parentheses. Because they refer to non-Python software releases, version numbers for this field are **not** required to conform to the format -specified in PEP 440: they should correspond to the +specified in :pep:`440`: they should correspond to the version scheme used by the external dependency. Notice that there's is no particular rule on the strings to be used. @@ -395,8 +397,8 @@ Examples:: Project-URL: Documentation, https://setuptools.readthedocs.io/ Project-URL: Funding, https://donate.pypi.org -The label is free text, with a maximum length of 32 characters. Notice -that distributions uploaded to PyPI will have these extra entries +The label is free text, with a maximum length of 32 characters. Notice +that distributions uploaded to PyPI will have these extra entries displayed under the "Project links" section of their landing page. @@ -411,7 +413,7 @@ Any number of conditional operators can be specified, e.g. the string ">1.0, !=1.3.4, <2.0" is a legal version declaration. The comma (",") is equivalent to the **and** operator. -Each version number must be in the format specified in PEP 440. +Each version number must be in the format specified in :pep:`440`. When a version is provided, it always includes all versions that starts with the same value. For example, the "2.5" version of Python @@ -536,19 +538,14 @@ References ========== This document specifies version 1.2 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. .. [1] reStructuredText markup: http://docutils.sourceforge.net/ .. _`Python Package Index`: http://pypi.python.org/pypi/ -.. [2] RFC 822 Long Header Fields: - http://www.freesoft.org/CIE/RFC/822/7.htm - -.. [3] PEP 301, Package Index and Metadata for Distutils: - http://www.python.org/dev/peps/pep-0301/ Copyright ========= diff --git a/pep-0346.txt b/peps/pep-0346.rst similarity index 93% rename from pep-0346.txt rename to peps/pep-0346.rst index 2e0ce4f3d..4666ff6c9 100644 --- a/pep-0346.txt +++ b/peps/pep-0346.rst @@ -14,10 +14,10 @@ Post-History: Abstract ======== -This PEP is a combination of PEP 310's "Reliable Acquisition/Release -Pairs" with the "Anonymous Block Statements" of Guido's PEP 340. This -PEP aims to take the good parts of PEP 340, blend them with parts of -PEP 310 and rearrange the lot into an elegant whole. It borrows from +This PEP is a combination of :pep:`310`'s "Reliable Acquisition/Release +Pairs" with the "Anonymous Block Statements" of Guido's :pep:`340`. This +PEP aims to take the good parts of :pep:`340`, blend them with parts of +:pep:`310` and rearrange the lot into an elegant whole. It borrows from various other PEPs in order to paint a complete picture, and is intended to stand on its own. @@ -25,24 +25,24 @@ intended to stand on its own. Author's Note ============= -During the discussion of PEP 340, I maintained drafts of this PEP as +During the discussion of :pep:`340`, I maintained drafts of this PEP as PEP 3XX on my own website (since I didn't have CVS access to update a submitted PEP fast enough to track the activity on python-dev). -Since the first draft of this PEP, Guido wrote PEP 343 as a simplified -version of PEP 340. PEP 343 (at the time of writing) uses the exact +Since the first draft of this PEP, Guido wrote :pep:`343` as a simplified +version of :pep:`340`. :pep:`343` (at the time of writing) uses the exact same semantics for the new statements as this PEP, but uses a slightly different mechanism to allow generators to be used to write statement templates. However, Guido has indicated that he intends to accept a -new PEP being written by Raymond Hettinger that will integrate PEP 288 -and PEP 325, and will permit a generator decorator like the one +new PEP being written by Raymond Hettinger that will integrate :pep:`288` +and :pep:`325`, and will permit a generator decorator like the one described in this PEP to be used to write statement templates for PEP 343. The other difference was the choice of keyword ('with' versus 'do') and Guido has stated he will organise a vote on that in the -context of PEP 343. +context of :pep:`343`. Accordingly, the version of this PEP submitted for archiving on -python.org is to be WITHDRAWN immediately after submission. PEP 343 +python.org is to be WITHDRAWN immediately after submission. :pep:`343` and the combined generator enhancement PEP will cover the important ideas. @@ -58,51 +58,51 @@ a 'user defined statement', and the associated class definitions are called 'statement templates'. The above is the main point of the PEP. However, if that was all it -said, then PEP 310 would be sufficient and this PEP would be +said, then :pep:`310` would be sufficient and this PEP would be essentially redundant. Instead, this PEP recommends additional enhancements that make it natural to write these statement templates using appropriately decorated generators. A side effect of those enhancements is that it becomes important to appropriately deal with the management of resources inside generators. -This is quite similar to PEP 343, but the exceptions that occur are +This is quite similar to :pep:`343`, but the exceptions that occur are re-raised inside the generators frame, and the issue of generator finalisation needs to be addressed as a result. The template generator decorator suggested by this PEP also creates reusable -templates, rather than the single use templates of PEP 340. +templates, rather than the single use templates of :pep:`340`. -In comparison to PEP 340, this PEP eliminates the ability to suppress +In comparison to :pep:`340`, this PEP eliminates the ability to suppress exceptions, and makes the user defined statement a non-looping construct. The other main difference is the use of a decorator to turn generators into statement templates, and the incorporation of ideas for addressing iterator finalisation. If all that seems like an ambitious operation. . . well, Guido was the -one to set the bar that high when he wrote PEP 340 :) +one to set the bar that high when he wrote :pep:`340` :) Relationship with other PEPs ============================ -This PEP competes directly with PEP 310 [1]_, PEP 340 [2]_ and PEP 343 -[3]_, as those PEPs all describe alternative mechanisms for handling +This PEP competes directly with :pep:`310`, :pep:`340` and :pep:`343`, +as those PEPs all describe alternative mechanisms for handling deterministic resource management. -It does not compete with PEP 342 [4]_ which splits off PEP 340's +It does not compete with :pep:`342` which splits off :pep:`340`'s enhancements related to passing data into iterators. The associated changes to the ``for`` loop semantics would be combined with the iterator finalisation changes suggested in this PEP. User defined statements would not be affected. Neither does this PEP compete with the generator enhancements -described in PEP 288 [5]_. While this PEP proposes the ability to +described in :pep:`288`. While this PEP proposes the ability to inject exceptions into generator frames, it is an internal implementation detail, and does not require making that ability -publicly available to Python code. PEP 288 is, in part, about +publicly available to Python code. :pep:`288` is, in part, about making that implementation detail easily accessible. This PEP would, however, make the generator resource release support -described in PEP 325 [6]_ redundant - iterators which require +described in :pep:`325` redundant - iterators which require finalisation should provide an appropriate implementation of the statement template protocol. @@ -110,7 +110,7 @@ statement template protocol. User defined statements ======================= -To steal the motivating example from PEP 310, correct handling of a +To steal the motivating example from :pep:`310`, correct handling of a synchronisation lock currently looks like this:: the_lock.acquire() @@ -119,7 +119,7 @@ synchronisation lock currently looks like this:: finally: the_lock.release() -Like PEP 310, this PEP proposes that such code be able to be written +Like :pep:`310`, this PEP proposes that such code be able to be written as:: with the_lock: @@ -372,16 +372,16 @@ Accordingly, if this PEP is accepted, ``yield``, like ``return``, will supply a default value of ``None`` (i.e. ``yield`` and ``yield None`` will become equivalent statements). -This same change is being suggested in PEP 342. Obviously, it would +This same change is being suggested in :pep:`342`. Obviously, it would only need to be implemented once if both PEPs were accepted :) Template generator decorator: ``statement_template`` ---------------------------------------------------- -As with PEP 343, a new decorator is suggested that wraps a generator +As with :pep:`343`, a new decorator is suggested that wraps a generator in an object with the appropriate statement template semantics. -Unlike PEP 343, the templates suggested here are reusable, as the +Unlike :pep:`343`, the templates suggested here are reusable, as the generator is instantiated anew in each call to ``__enter__()``. Additionally, any exceptions that occur in ``BLOCK1`` are re-raised in the generator's internal frame:: @@ -756,7 +756,7 @@ Examples with tag('a', href="http://www.python.org"): print "Not a dead parrot!" -12. From PEP 343, another useful example would be an operation that +12. From :pep:`343`, another useful example would be an operation that blocks signals. The use could be like this:: from signal import blocked_signals @@ -801,7 +801,7 @@ Rejected Options Having the basic construct be a looping construct ------------------------------------------------- -The major issue with this idea, as illustrated by PEP 340's +The major issue with this idea, as illustrated by :pep:`340`'s ``block`` statements, is that it causes problems with factoring ``try`` statements that are inside loops, and contain ``break`` and ``continue`` statements (as these statements would then apply to the @@ -823,7 +823,7 @@ construct. With the current PEP, there is no such problem - ``for`` loops continue to be used for iteration, and the new ``do`` statements are used to factor out exception handling. -Another issue, specifically with PEP 340's anonymous block statements, +Another issue, specifically with :pep:`340`'s anonymous block statements, is that they make it quite difficult to write statement templates directly (i.e. not using a generator). This problem is addressed by the current proposal, as can be seen by the relative simplicity of the @@ -837,7 +837,7 @@ Allowing statement templates to suppress exceptions Earlier versions of this PEP gave statement templates the ability to suppress exceptions. The BDFL expressed concern over the associated complexity, and I agreed after reading an article by Raymond Chen -about the evils of hiding flow control inside macros in C code [7]_. +about the evils of hiding flow control inside macros in C code [1]_. Removing the suppression ability eliminated a whole lot of complexity from both the explanation and implementation of user defined @@ -880,7 +880,7 @@ Differentiating between non-exceptional exits Earlier versions of this PEP allowed statement templates to distinguish between exiting the block normally, and exiting via a ``return``, ``break`` or ``continue`` statement. The BDFL flirted -with a similar idea in PEP 343 and its associated discussion. This +with a similar idea in :pep:`343` and its associated discussion. This added significant complexity to the description of the semantics, and it required each and every statement template to decide whether or not those statements should be treated like exceptions, or like a normal @@ -899,7 +899,7 @@ template is concerned. Not injecting raised exceptions into generators ----------------------------------------------- -PEP 343 suggests simply invoking next() unconditionally on generators +:pep:`343` suggests simply invoking next() unconditionally on generators used to define statement templates. This means the template generators end up looking rather unintuitive, and the retention of the ban against yielding inside ``try``/``finally`` means that Python's @@ -937,17 +937,17 @@ generator objects to be used to implement generator finalisation. Using ``do`` as the keyword --------------------------- -``do`` was an alternative keyword proposed during the PEP 340 +``do`` was an alternative keyword proposed during the :pep:`340` discussion. It reads well with appropriately named functions, but it reads poorly when used with methods, or with objects that provide native statement template support. -When ``do`` was first suggested, the BDFL had rejected PEP 310's +When ``do`` was first suggested, the BDFL had rejected :pep:`310`'s ``with`` keyword, based on a desire to use it for a Pascal/Delphi style ``with`` statement. Since then, the BDFL has retracted this objection, as he no longer intends to provide such a statement. This change of heart was apparently based on the C# developers reasons for -not providing the feature [8]_. +not providing the feature [2]_. Not having a keyword @@ -1251,9 +1251,9 @@ addition to the iterator protocol backwards compatible. Acknowledgements ================ -The acknowledgements section for PEP 340 applies, since this text grew +The acknowledgements section for :pep:`340` applies, since this text grew out of the discussion of that PEP, but additional thanks go to Michael -Hudson, Paul Moore and Guido van Rossum for writing PEP 310 and PEP +Hudson, Paul Moore and Guido van Rossum for writing :pep:`310` and PEP 340 in the first place, and to (in no meaningful order) Fredrik Lundh, Phillip J. Eby, Steven Bethard, Josiah Carlson, Greg Ewing, Tim Delaney and Arnold deVos for prompting particular ideas that made @@ -1263,28 +1263,10 @@ their way into this text. References ========== -.. [1] Reliable Acquisition/Release Pairs - (http://www.python.org/dev/peps/pep-0310/) - -.. [2] Anonymous block statements - (http://www.python.org/dev/peps/pep-0340/) - -.. [3] Anonymous blocks, redux - (http://www.python.org/dev/peps/pep-0343/) - -.. [4] Enhanced Iterators - (http://www.python.org/dev/peps/pep-0342/) - -.. [5] Generator Attributes and Exceptions - (http://www.python.org/dev/peps/pep-0288/) - -.. [6] Resource-Release Support for Generators - (http://www.python.org/dev/peps/pep-0325/) - -.. [7] A rant against flow control macros +.. [1] A rant against flow control macros (http://blogs.msdn.com/oldnewthing/archive/2005/01/06/347666.aspx) -.. [8] Why doesn't C# have a 'with' statement? +.. [2] Why doesn't C# have a 'with' statement? (http://msdn.microsoft.com/vcsharp/programming/language/ask/withstatement/) diff --git a/pep-0347.txt b/peps/pep-0347.rst similarity index 99% rename from pep-0347.txt rename to peps/pep-0347.rst index 59099edc1..8445bd2c5 100644 --- a/pep-0347.txt +++ b/peps/pep-0347.rst @@ -3,7 +3,7 @@ Title: Migrating the Python CVS to Subversion Version: $Revision$ Last-Modified: $Date$ Author: Martin von Löwis -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Process Content-Type: text/x-rst diff --git a/pep-0348.txt b/peps/pep-0348.rst similarity index 96% rename from pep-0348.txt rename to peps/pep-0348.rst index 866710d05..d72229047 100644 --- a/pep-0348.txt +++ b/peps/pep-0348.rst @@ -55,7 +55,7 @@ simplify catching them in ``except`` clauses. To allow people to be able to rely on this hierarchy, a common superclass that all raise objects must inherit from is being proposed. It also allows guarantees about the interface to raised objects to be made (see -PEP 344 [#PEP344]_). A discussion about all of this has occurred +:pep:`344`). A discussion about all of this has occurred before on python-dev [#Summary2004-08-01]_. As bare ``except`` clauses stand now, they catch *all* exceptions. @@ -118,7 +118,7 @@ New Hierarchy +-- BaseException (new; broader inheritance for subclasses) +-- Exception - +-- GeneratorExit (defined in PEP 342 [#PEP342]_) + +-- GeneratorExit (defined in :pep:`342`) +-- StandardError +-- ArithmeticError +-- DivideByZeroError @@ -221,7 +221,7 @@ Required Superclass for ``raise`` By requiring all objects passed to a ``raise`` statement to inherit from a specific superclass, all exceptions are guaranteed to have -certain attributes. If PEP 344 [#PEP344]_ is accepted, the attributes +certain attributes. If :pep:`344` is accepted, the attributes outlined there will be guaranteed to be on all exceptions raised. This should help facilitate debugging by making the querying of information from exceptions much easier. @@ -256,7 +256,7 @@ semantics are what most people want for bare ``except`` clauses. The complete removal of bare ``except`` clauses has been argued for. The case has been made that they violate both Only One Way To Do It (OOWTDI) and Explicit Is Better Than Implicit (EIBTI) as listed in the -Zen of Python [#zen]_. But Practicality Beats Purity (PBP), also in +:pep:`Zen of Python <20>`. But Practicality Beats Purity (PBP), also in the Zen of Python, trumps both of these in this case. The BDFL has stated that bare ``except`` clauses will work this way [#python-dev8]_. @@ -294,7 +294,7 @@ reflect several programming guidelines: the inheritance from RuntimeError The documentation for the 'exceptions' module [#exceptions-stdlib]_, -tutorial [#tutorial]_, and PEP 290 [#PEP290]_ will all require +tutorial [#tutorial]_, and :pep:`290` will all require updating. @@ -446,15 +446,6 @@ discussion. References ========== -.. [#PEP342] PEP 342 (Coroutines via Enhanced Generators) - http://www.python.org/dev/peps/pep-0342/ - -.. [#PEP344] PEP 344 (Exception Chaining and Embedded Tracebacks) - http://www.python.org/dev/peps/pep-0344/ - -.. [#PEP290] PEP 290 (Code Migration and Modernization) - http://www.python.org/dev/peps/pep-0290/ - .. [#Summary2004-08-01] python-dev Summary (An exception is an exception, unless it doesn't inherit from Exception) http://www.python.org/dev/summary/2004-08-01_2004-08-15.html#an-exception-is-an-exception-unless-it-doesn-t-inherit-from-exception @@ -504,9 +495,6 @@ References .. [#python-dev8] python-dev email (PEP 348 (exception reorg) revised again) https://mail.python.org/pipermail/python-dev/2005-August/055423.html -.. [#zen] PEP 20 (The Zen of Python) - http://www.python.org/dev/peps/pep-0020/ - .. [#tutorial] Python Tutorial http://docs.python.org/tutorial/ diff --git a/pep-0349.txt b/peps/pep-0349.rst similarity index 100% rename from pep-0349.txt rename to peps/pep-0349.rst diff --git a/pep-0350.txt b/peps/pep-0350.rst similarity index 100% rename from pep-0350.txt rename to peps/pep-0350.rst diff --git a/pep-0351.txt b/peps/pep-0351.rst similarity index 98% rename from pep-0351.txt rename to peps/pep-0351.rst index 0178a7886..5923d1b05 100644 --- a/pep-0351.txt +++ b/peps/pep-0351.rst @@ -41,7 +41,7 @@ and immutable counterparts, and it would be useful to have a standard protocol for conversion of such objects. sets.Set objects expose a "protocol for automatic conversion to -immutable" so that you can create sets.Sets of sets.Sets. PEP 218 +immutable" so that you can create sets.Sets of sets.Sets. :pep:`218` deliberately dropped this feature from built-in sets. This PEP advances that the feature is still useful and proposes a standard mechanism for its support. diff --git a/pep-0352.txt b/peps/pep-0352.rst similarity index 98% rename from pep-0352.txt rename to peps/pep-0352.rst index 3065a21cd..3c0db918e 100644 --- a/pep-0352.txt +++ b/peps/pep-0352.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 27-Oct-2005 +Python-Version: 2.5 Post-History: @@ -45,7 +46,7 @@ from the common superclass instead of Exception will make it easy for people to write ``except`` clauses that are not overreaching and not catch exceptions that should propagate up. -This PEP is based on previous work done for PEP 348 [#pep348]_. +This PEP is based on previous work done for :pep:`348`. Requiring a Common Superclass @@ -272,9 +273,6 @@ original deprecation of ``args`` has been retracted. References ========== -.. [#pep348] PEP 348 (Exception Reorganization for Python 3.0) - http://www.python.org/dev/peps/pep-0348/ - .. [#hierarchy-good] python-dev Summary for 2004-08-01 through 2004-08-15 http://www.python.org/dev/summary/2004-08-01_2004-08-15.html#an-exception-is-an-exception-unless-it-doesn-t-inherit-from-exception diff --git a/pep-0353.txt b/peps/pep-0353.rst similarity index 99% rename from pep-0353.txt rename to peps/pep-0353.rst index 3d2dcbfad..16e178b16 100644 --- a/pep-0353.txt +++ b/peps/pep-0353.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 18-Dec-2005 +Python-Version: 2.5 Post-History: diff --git a/pep-0354.txt b/peps/pep-0354.rst similarity index 99% rename from pep-0354.txt rename to peps/pep-0354.rst index 5f9aab091..360aabdad 100644 --- a/pep-0354.txt +++ b/peps/pep-0354.rst @@ -22,7 +22,7 @@ modules. Also, the PEP has generated no widespread interest. For those who need enumerations, there are cookbook recipes and PyPI packages that meet these needs. -*Note: this PEP was superseded by PEP 435, which has been accepted in +*Note: this PEP was superseded by* :pep:`435`, *which has been accepted in May 2013.* Abstract diff --git a/pep-0355.txt b/peps/pep-0355.rst similarity index 96% rename from pep-0355.txt rename to peps/pep-0355.rst index ed5cec73d..55cdac94e 100644 --- a/pep-0355.txt +++ b/peps/pep-0355.rst @@ -82,7 +82,7 @@ current common practice: - Subclassability - the ``Path`` object can be extended to support paths other than filesystem paths. The programmer does not need - to learn a new API, but can reuse his or her knowledge of Path + to learn a new API, but can reuse their knowledge of Path to deal with the extended class. - With all related functionality in one place, the right approach @@ -167,7 +167,7 @@ for brevity; see the reference implementation for more detail):: """p.splitpath() -> Return (p.parent, p.name).""" def stripext(self) => Path """p.stripext() -> Remove one file extension from the path.""" - def splitunc(self): ... [1]_ + def splitunc(self): ... # See footnote [1] def splitall(self): ... def relpath(self): ... def relpathto(self, dest): ... @@ -219,7 +219,7 @@ for brevity; see the reference implementation for more detail):: def isfile(self): ... def islink(self): ... def ismount(self): ... - def samefile(self, other): ... [1]_ + def samefile(self, other): ... # See footnote [1] def atime(self): ... """Last access time of the file.""" def mtime(self): ... @@ -231,17 +231,17 @@ for brevity; see the reference implementation for more detail):: Windows), is the creation time for path. """ def size(self): ... - def access(self, mode): ... [1]_ + def access(self, mode): ... # See footnote [1] def stat(self): ... def lstat(self): ... - def statvfs(self): ... [1]_ - def pathconf(self, name): ... [1]_ + def statvfs(self): ... # See footnote [1] + def pathconf(self, name): ... # See footnote [1] # Methods for manipulating information about the filesystem # path. def utime(self, times) => None def chmod(self, mode) => None - def chown(self, uid, gid) => None [1]_ + def chown(self, uid, gid) => None # See footnote [1] def rename(self, new) => None def renames(self, new) => None @@ -273,8 +273,8 @@ for brevity; see the reference implementation for more detail):: def rmtree(self, ignore_errors = False, onerror = None): ... # Special stuff from os - def chroot(self): ... [1]_ - def startfile(self): ... [1]_ + def chroot(self): ... # See footnote [1] + def startfile(self): ... # See footnote [1] Replacing older functions with the Path class @@ -546,7 +546,7 @@ replacement for a. References and Footnotes ======================== -.. [1] Method is not guaranteed to be available on all platforms. +[1] Method is not guaranteed to be available on all platforms. .. [2] "(idea) subclassable string: path object?", van Rossum, 2001 https://mail.python.org/pipermail/python-dev/2001-August/016663.html @@ -570,12 +570,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0356.txt b/peps/pep-0356.rst similarity index 87% rename from pep-0356.txt rename to peps/pep-0356.rst index 1e02cecbd..472222fc1 100644 --- a/pep-0356.txt +++ b/peps/pep-0356.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Neal Norwitz, Guido van Rossum, Anthony Baxter Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 07-Feb-2006 Python-Version: 2.5 @@ -50,18 +51,18 @@ Release Schedule Completed features for 2.5 ========================== -- PEP 308: Conditional Expressions -- PEP 309: Partial Function Application -- PEP 314: Metadata for Python Software Packages v1.1 -- PEP 328: Absolute/Relative Imports -- PEP 338: Executing Modules as Scripts -- PEP 341: Unified try-except/try-finally to try-except-finally -- PEP 342: Coroutines via Enhanced Generators -- PEP 343: The "with" Statement (still need updates in Doc/ref and for the +- :pep:`308`: Conditional Expressions +- :pep:`309`: Partial Function Application +- :pep:`314`: Metadata for Python Software Packages v1.1 +- :pep:`328`: Absolute/Relative Imports +- :pep:`338`: Executing Modules as Scripts +- :pep:`341`: Unified try-except/try-finally to try-except-finally +- :pep:`342`: Coroutines via Enhanced Generators +- :pep:`343`: The "with" Statement (still need updates in Doc/ref and for the ``contextlib`` module) -- PEP 352: Required Superclass for Exceptions -- PEP 353: Using ``ssize_t`` as the index type -- PEP 357: Allowing Any Object to be Used for Slicing +- :pep:`352`: Required Superclass for Exceptions +- :pep:`353`: Using ``ssize_t`` as the index type +- :pep:`357`: Allowing Any Object to be Used for Slicing - ASCII became the default coding @@ -97,7 +98,7 @@ Other notable features: - Added support for the Unicode 4.1 UCD -- Added PEP 302 ``zipfile``/``__loader__`` support to the following modules: +- Added :pep:`302` ``zipfile``/``__loader__`` support to the following modules: ``warnings``, ``linecache``, ``inspect``, ``traceback``, ``site``, and ``doctest`` @@ -171,7 +172,7 @@ Open issues * https://bugs.python.org/issue1467929 - %-formatting and dicts * https://bugs.python.org/issue1446043 - ``unicode()`` does not raise ``LookupError`` -- The PEP 302 changes to (at least) ``pkgutil``, ``runpy`` and ``pydoc`` must +- The :pep:`302` changes to (at least) ``pkgutil``, ``runpy`` and ``pydoc`` must be documented. - ``test_zipfile64`` takes too long and too much disk space for diff --git a/pep-0357.txt b/peps/pep-0357.rst similarity index 97% rename from pep-0357.txt rename to peps/pep-0357.rst index 0ec14fff3..c7467115f 100644 --- a/pep-0357.txt +++ b/peps/pep-0357.rst @@ -1,7 +1,5 @@ PEP: 357 Title: Allowing Any Object to be Used for Slicing -Version: $Revision$ -Last-Modified: $Date$ Author: Travis Oliphant Status: Final Type: Standards Track @@ -237,13 +235,3 @@ Copyright ========= This document is placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: \ No newline at end of file diff --git a/pep-0358.txt b/peps/pep-0358.rst similarity index 99% rename from pep-0358.txt rename to peps/pep-0358.rst index 01ba06acd..feffd9815 100644 --- a/pep-0358.txt +++ b/peps/pep-0358.rst @@ -14,7 +14,7 @@ Post-History: Update ====== -This PEP has partially been superseded by PEP 3137. +This PEP has partially been superseded by :pep:`3137`. Abstract @@ -190,7 +190,7 @@ Out of Scope Issues byte arrays. This decision is out of scope. * A bytes literal of the form ``b"..."`` is also proposed. This is - the subject of PEP 3112. + the subject of :pep:`3112`. Open Issues diff --git a/pep-0359.txt b/peps/pep-0359.rst similarity index 100% rename from pep-0359.txt rename to peps/pep-0359.rst diff --git a/pep-0360.txt b/peps/pep-0360.rst similarity index 100% rename from pep-0360.txt rename to peps/pep-0360.rst diff --git a/pep-0361.txt b/peps/pep-0361.rst similarity index 81% rename from pep-0361.txt rename to peps/pep-0361.rst index f87782e35..03bb5bc36 100644 --- a/pep-0361.txt +++ b/peps/pep-0361.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Neal Norwitz, Barry Warsaw Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 29-Jun-2006 Python-Version: 2.6, 3.0 @@ -89,7 +90,7 @@ Release Schedule Completed features for 3.0 ========================== -See PEP 3000 [pep3000]_ and PEP 3100 [pep3100]_ for details on the +See :pep:`3000` and :pep:`3100` for details on the Python 3.0 project. @@ -98,15 +99,15 @@ Completed features for 2.6 PEPs: -- 352: Raising a string exception now triggers a ``TypeError``. +- :pep:`352`: Raising a string exception now triggers a ``TypeError``. Attempting to catch a string exception raises ``DeprecationWarning``. - ``BaseException.message`` has been deprecated. [pep352]_ -- 358: The "bytes" Object [pep358]_ -- 366: Main module explicit relative imports [pep366]_ -- 370: Per user site-packages directory [pep370]_ -- 3112: Bytes literals in Python 3000 [pep3112]_ -- 3127: Integer Literal Support and Syntax [pep3127]_ -- 371: Addition of the multiprocessing package [pep371]_ + ``BaseException.message`` has been deprecated. +- :pep:`358`: The "bytes" Object +- :pep:`366`: Main module explicit relative imports +- :pep:`370`: Per user site-packages directory +- :pep:`3112`: Bytes literals in Python 3000 +- :pep:`3127`: Integer Literal Support and Syntax +- :pep:`371`: Addition of the multiprocessing package New modules in the standard library: @@ -145,12 +146,12 @@ Warnings for features removed in Py3k: - ``{}.has_key()`` - ``file.xreadlines`` - softspace removal for ``print()`` function -- removal of modules because of PEP 4/3100/3108 +- removal of modules because of :pep:`4`/:pep:`3100`/:pep:`3108` Other major features: - ``with``/``as`` will be keywords -- a ``__dir__()`` special method to control ``dir()`` was added [1] +- a ``__dir__()`` special method to control ``dir()`` was added [1]_ - AtheOS support stopped. - ``warnings`` module implemented in C - ``compile()`` takes an AST and can convert to byte code @@ -273,36 +274,6 @@ References .. [1] Adding a __dir__() magic method https://mail.python.org/pipermail/python-dev/2006-July/067139.html -.. [pep352] PEP 352 (Required Superclass for Exceptions) - http://www.python.org/dev/peps/pep-0352 - -.. [pep358] PEP 358 (The "bytes" Object) - http://www.python.org/dev/peps/pep-0358 - -.. [pep366] PEP 366 (Main module explicit relative imports) - http://www.python.org/dev/peps/pep-0366 - -.. [pep367] PEP 367 (New Super) - http://www.python.org/dev/peps/pep-0367 - -.. [pep370] PEP 370 (Per user site-packages directory) - http://www.python.org/dev/peps/pep-0370 - -.. [pep371] PEP 371 (Addition of the multiprocessing package) - http://www.python.org/dev/peps/pep-0371 - -.. [pep3000] PEP 3000 (Python 3000) - http://www.python.org/dev/peps/pep-3000 - -.. [pep3100] PEP 3100 (Miscellaneous Python 3.0 Plans) - http://www.python.org/dev/peps/pep-3100 - -.. [pep3112] PEP 3112 (Bytes literals in Python 3000) - http://www.python.org/dev/peps/pep-3112 - -.. [pep3127] PEP 3127 (Integer Literal Support and Syntax) - http://www.python.org/dev/peps/pep-3127 - .. _Google calendar: http://www.google.com/calendar/ical/b6v58qvojllt0i6ql654r1vh00%40group.calendar.google.com/public/basic.ics @@ -310,14 +281,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0362.txt b/peps/pep-0362.rst similarity index 98% rename from pep-0362.txt rename to peps/pep-0362.rst index 9fdf9d01a..0d001365d 100644 --- a/pep-0362.txt +++ b/peps/pep-0362.rst @@ -42,12 +42,12 @@ it stores a `Parameter object`_ in its ``parameters`` collection. A Signature object has the following public attributes and methods: -* return_annotation : object +* return_annotation \: object The "return" annotation for the function. If the function has no "return" annotation, this attribute is set to ``Signature.empty``. -* parameters : OrderedDict +* parameters \: OrderedDict An ordered mapping of parameters' names to the corresponding Parameter objects. @@ -134,16 +134,16 @@ function parameter. A Parameter object has the following public attributes and methods: -* name : str +* name \: str The name of the parameter as a string. Must be a valid python identifier name (with the exception of ``POSITIONAL_ONLY`` parameters, which can have it set to ``None``.) -* default : object +* default \: object The default value for the parameter. If the parameter has no default value, this attribute is set to ``Parameter.empty``. -* annotation : object +* annotation \: object The annotation for the parameter. If the parameter has no annotation, this attribute is set to ``Parameter.empty``. @@ -220,14 +220,14 @@ to the function's parameters. Has the following public attributes: -* arguments : OrderedDict +* arguments \: OrderedDict An ordered, mutable mapping of parameters' names to arguments' values. Contains only explicitly bound arguments. Arguments for which ``bind()`` relied on a default value are skipped. -* args : tuple +* args \: tuple Tuple of positional arguments values. Dynamically computed from the 'arguments' attribute. -* kwargs : dict +* kwargs \: dict Dict of keyword arguments values. Dynamically computed from the 'arguments' attribute. @@ -545,7 +545,7 @@ Annotation Checker Acceptance ========== -PEP 362 was accepted by Guido, Friday, June 22, 2012 [#accepted]_ . +:pep:`362` was accepted by Guido, Friday, June 22, 2012 [#accepted]_ . The reference implementation was committed to trunk later that day. diff --git a/pep-0363.txt b/peps/pep-0363.rst similarity index 100% rename from pep-0363.txt rename to peps/pep-0363.rst diff --git a/pep-0364.txt b/peps/pep-0364.rst similarity index 88% rename from pep-0364.txt rename to peps/pep-0364.rst index 80acf77a9..0210ad5ca 100644 --- a/pep-0364.txt +++ b/peps/pep-0364.rst @@ -14,8 +14,8 @@ Post-History: Abstract ======== -PEP 3108 describes the reorganization of the Python standard library -for the Python 3.0 release [1]_. This PEP describes a +:pep:`3108` describes the reorganization of the Python standard library +for the Python 3.0 release. This PEP describes a mechanism for transitioning from the Python 2.x standard library to the Python 3.0 standard library. This transition will allow and encourage Python programmers to use the new Python 3.0 library names @@ -28,22 +28,22 @@ existing Python programs. Rationale ========= -PEP 3108 presents a rationale for Python standard library (stdlib) +:pep:`3108` presents a rationale for Python standard library (stdlib) reorganization. The reader is encouraged to consult that PEP for details about why and how the library will be reorganized. Should -PEP 3108 be accepted in part or in whole, then it is advantageous to +:pep:`3108` be accepted in part or in whole, then it is advantageous to allow Python programmers to begin the transition to the new stdlib module names in Python 2.x, so that they can write forward compatible code starting with Python 2.6. -Note that PEP 3108 proposes to remove some "silly old stuff", +Note that :pep:`3108` proposes to remove some "silly old stuff", i.e. modules that are no longer useful or necessary. The PEP you are reading does not address this because there are no forward compatibility issues for modules that are to be removed, except to stop using such modules. This PEP concerns only the mechanism by which mappings from old stdlib -names to new stdlib names are maintained. Please consult PEP 3108 for +names to new stdlib names are maintained. Please consult :pep:`3108` for all specific module renaming proposals. Specifically see the section titled ``Modules to Rename`` for guidelines on the old name to new name mappings. The few examples in this PEP are given for @@ -70,8 +70,8 @@ Two use cases supported by this PEP include renaming simple top-level modules, such as ``StringIO``, as well as modules within packages, such as ``email.MIMEText``. -In the former case, PEP 3108 currently recommends ``StringIO`` be -renamed to ``stringio``, following PEP 8 recommendations [2]_. +In the former case, :pep:`3108` currently recommends ``StringIO`` be +renamed to ``stringio``, following :pep:`8` recommendations. In the latter case, the email 4.0 package distributed with Python 2.5 already renamed ``email.MIMEText`` to ``email.mime.text``, although it @@ -120,13 +120,13 @@ Implementation Specification This section provides the full specification for how module renamings in Python 2.x are implemented. The central mechanism relies on -various import hooks as described in PEP 302 [3]_. Specifically +various import hooks as described in :pep:`302`. Specifically ``sys.path_importer_cache``, ``sys.path``, and ``sys.meta_path`` are all employed to provide the necessary functionality. When Python's import machinery is initialized, the oldlib package is imported. Inside oldlib there is a class called ``OldStdlibLoader``. -This class implements the PEP 302 interface and is automatically +This class implements the :pep:`302` interface and is automatically instantiated, with zero arguments. The constructor reads all the ``.mv`` files from the oldlib package directory, automatically registering all the remappings found in those ``.mv`` files. This is @@ -137,7 +137,7 @@ modules. Instead, you can access the global OldStdlibLoader instance via the ``sys.stdlib_remapper`` instance. Use this instance if you want programmatic access to the remapping machinery. -One important implementation detail: as needed by the PEP 302 API, a +One important implementation detail: as needed by the :pep:`302` API, a magic string is added to sys.path, and module __path__ attributes in order to hook in our remapping loader. This magic string is currently ```` and some changes were necessary to Python's site.py file @@ -230,25 +230,16 @@ Reference Implementation A reference implementation, in the form of a patch against the current (as of this writing) state of the Python 2.6 svn trunk, is available -as SourceForge patch #1675334 [4]_. Note that this patch includes a +as SourceForge patch #1675334 [1]_. Note that this patch includes a rename of ``cStringIO`` to ``cstringio``, but this is primarily for illustrative and unit testing purposes. Should the patch be accepted, -we might want to split this change off into other PEP 3108 changes. +we might want to split this change off into other :pep:`3108` changes. References ========== -.. [1] PEP 3108, Standard Library Reorganization, Cannon - (http://www.python.org/dev/peps/pep-3108) - -.. [2] PEP 8, Style Guide for Python Code, GvR, Warsaw - (http://www.python.org/dev/peps/pep-0008) - -.. [3] PEP 302, New Import Hooks, JvR, Moore - (http://www.python.org/dev/peps/pep-0302) - -.. [4] Reference implementation +.. [1] Reference implementation (http://bugs.python.org/issue1675334) Copyright diff --git a/pep-0365.txt b/peps/pep-0365.rst similarity index 99% rename from pep-0365.txt rename to peps/pep-0365.rst index f52a617a1..5fa999167 100644 --- a/pep-0365.txt +++ b/peps/pep-0365.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Phillip J. Eby Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 30-Apr-2007 Post-History: 30-Apr-2007 diff --git a/pep-0366.txt b/peps/pep-0366.rst similarity index 90% rename from pep-0366.txt rename to peps/pep-0366.rst index 302835919..73eeb61f4 100644 --- a/pep-0366.txt +++ b/peps/pep-0366.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 01-May-2007 Python-Version: 2.6, 3.0 -Post-History: 1-May-2007, 4-Jul-2007, 7-Jul-2007, 23-Nov-2007 +Post-History: 01-May-2007, 04-Jul-2007, 07-Jul-2007, 23-Nov-2007 Abstract @@ -17,7 +17,7 @@ Abstract This PEP proposes a backwards compatible mechanism that permits the use of explicit relative imports from executable modules within packages. Such imports currently fail due to an awkward interaction -between PEP 328 and PEP 338. +between :pep:`328` and :pep:`338`. By adding a new module level attribute, this PEP allows relative imports to work automatically if the module is executed using the ``-m`` switch. @@ -35,7 +35,7 @@ be based on this attribute rather than the module ``__name__`` attribute. As with the current ``__name__`` attribute, setting ``__package__`` will -be the responsibility of the PEP 302 loader used to import a module. +be the responsibility of the :pep:`302` loader used to import a module. Loaders which use ``imp.new_module()`` to create the module object will have the new attribute set automatically to ``None``. When the import system encounters an explicit relative import in a module without @@ -91,7 +91,7 @@ This PEP is intended to provide a solution which permits explicit relative imports from main modules, without incurring any significant costs during interpreter startup or normal module import. -The section in PEP 338 on relative imports and the main module provides +The section in :pep:`338` on relative imports and the main module provides further details and background on this problem. @@ -103,12 +103,12 @@ which stored the main module's real module name in the ``__module_name__`` attribute. It was reverted due to the fact that 2.5 was already in beta by that time. -Patch 1487 [4] is the proposed implementation for this PEP. +Patch 1487 [4]_ is the proposed implementation for this PEP. Alternative Proposals ===================== -PEP 3122 proposed addressing this problem by changing the way +:pep:`3122` proposed addressing this problem by changing the way the main module is identified. That's a significant compatibility cost to incur to fix something that is a pretty minor bug in the overall scheme of things, and the PEP was rejected [3]_. @@ -125,7 +125,7 @@ References ========== .. [1] Absolute/relative import not working? - (https://bugs.python.org/issue1510172) + (https://github.com/python/cpython/issues/43535) .. [2] c.l.p. question about modules and relative imports (http://groups.google.com/group/comp.lang.python/browse_thread/thread/c44c769a72ca69fa/) @@ -134,7 +134,7 @@ References (https://mail.python.org/pipermail/python-3000/2007-April/006793.html) .. [4] PEP 366 implementation patch - (http://bugs.python.org/issue1487) + (https://github.com/python/cpython/issues/45828) .. [5] Acceptance of the PEP (https://mail.python.org/pipermail/python-dev/2007-November/075475.html) @@ -143,11 +143,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0367.txt b/peps/pep-0367.rst similarity index 95% rename from pep-0367.txt rename to peps/pep-0367.rst index 718698c53..168db7a1e 100644 --- a/pep-0367.txt +++ b/peps/pep-0367.rst @@ -9,12 +9,15 @@ Type: Standards Track Content-Type: text/x-rst Created: 28-Apr-2007 Python-Version: 2.6 -Post-History: 28-Apr-2007, 29-Apr-2007 (1), 29-Apr-2007 (2), 14-May-2007 +Post-History: `28-Apr-2007 `__, + `29-Apr-2007 `__, + `29-Apr-2007 `__, + `14-May-2007 `__ Numbering Note ============== -This PEP has been renumbered to PEP 3135. The text below is the last +This PEP has been renumbered to :pep:`3135`. The text below is the last version submitted under the old number. Abstract @@ -119,10 +122,10 @@ Should ``super`` actually become a keyword? With this proposal, ``super`` would become a keyword to the same extent that ``None`` is a keyword. It is possible that further restricting the ``super`` -name may simplify implementation, however some are against the actual keyword- -ization of super. The simplest solution is often the correct solution and the -simplest solution may well not be adding additional keywords to the language -when they are not needed. Still, it may solve other open issues. +name may simplify implementation, however some are against the actual +keyword-ization of super. The simplest solution is often the correct solution +and the simplest solution may well not be adding additional keywords to the +language when they are not needed. Still, it may solve other open issues. Closed Issues @@ -510,10 +513,10 @@ super(self, \*args) or __super__(self, \*args) This solution only solves the problem of the type indication, does not handle differently named super methods, and is explicit about the name of the instance. It is less flexible without being able to enacted on other method -names, in cases where that is needed. One use case this fails is where a base- -class has a factory classmethod and a subclass has two factory classmethods, -both of which needing to properly make super calls to the one in the base- -class. +names, in cases where that is needed. One use case this fails is where a +base-class has a factory classmethod and a subclass has two factory +classmethods, both of which needing to properly make super calls to the one +in the base-class. super.foo(self, \*args) ----------------------- diff --git a/pep-0368.txt b/peps/pep-0368.rst similarity index 99% rename from pep-0368.txt rename to peps/pep-0368.rst index 82b030a81..319e7bf6a 100644 --- a/pep-0368.txt +++ b/peps/pep-0368.rst @@ -492,7 +492,7 @@ Non-planar images offer the following additional methods: The ``mode``, ``size`` and ``buffer`` (including the address in memory of the ``buffer``) never change after an image is created. -It is expected that, if PEP 3118 is accepted, all the image objects +It is expected that, if :pep:`3118` is accepted, all the image objects will support the new buffer protocol, however this is beyond the scope of this PEP. @@ -797,7 +797,7 @@ compatibility should be considered: functionality, so no major compatibility issues are expected. * **Python 3.0**: the legacy contents of the ``imageop`` module will - be deleted, according to PEP 3108; everything defined in this + be deleted, according to :pep:`3108`; everything defined in this proposal will work like in Python 2.x with the exception of the usual 2.x/3.0 differences (e.g. support for ``long`` integers and for interpreting ``str`` instances as sequences of bytes will be diff --git a/pep-0369.txt b/peps/pep-0369.rst similarity index 99% rename from pep-0369.txt rename to peps/pep-0369.rst index 6c40112a9..40caa8c66 100644 --- a/pep-0369.txt +++ b/peps/pep-0369.rst @@ -34,7 +34,7 @@ Rationale ========= Python has no API to hook into the import machinery and execute code -*after* a module is successfully loaded. The import hooks of PEP 302 are +*after* a module is successfully loaded. The import hooks of :pep:`302` are about finding modules and loading modules but they were not designed to as post import hooks. diff --git a/pep-0370.txt b/peps/pep-0370.rst similarity index 87% rename from pep-0370.txt rename to peps/pep-0370.rst index 84b422597..fda0200cb 100644 --- a/pep-0370.txt +++ b/peps/pep-0370.rst @@ -1,7 +1,5 @@ PEP: 370 Title: Per user site-packages directory -Version: $Revision$ -Last-Modified: $Date$ Author: Christian Heimes Status: Final Type: Standards Track @@ -203,48 +201,39 @@ References http://peak.telecommunity.com/DevCenter/EasyInstall#creating-a-virtual-python .. [2] Working Env - http://pypi.python.org/pypi/workingenv.py - http://blog.ianbicking.org/workingenv-revisited.html + https://pypi.org/project/workingenv.py/ + https://ianbicking.org/archive/workingenv-revisited.html .. [3] Virtual Env - http://pypi.python.org/pypi/virtualenv + https://pypi.org/project/virtualenv/ .. [4] reference implementation - http://bugs.python.org/issue1799 + https://github.com/python/cpython/issues/46132 http://svn.python.org/view/sandbox/trunk/pep370 .. [5] MSDN: CSIDL - http://msdn2.microsoft.com/en-us/library/bb762494.aspx + https://learn.microsoft.com/en/windows/win32/shell/csidl -.. [6] Initial suggestion for a per user site-packages directory - http://permalink.gmane.org/gmane.comp.python.devel/90902 +[6] Initial suggestion for a per user site-packages directory +\ https://mail.python.org/archives/list/python-dev@python.org/message/V23CUKRH3VCHFLV33ADMHJSM53STPA7I/ .. [7] Suggestion of ~/.local/ - http://permalink.gmane.org/gmane.comp.python.devel/90925 + https://mail.python.org/pipermail/python-dev/2008-January/075985.html .. [8] APPDATA discussion - http://permalink.gmane.org/gmane.comp.python.devel/90932 + https://mail.python.org/pipermail/python-dev/2008-January/075993.html .. [9] Security concerns and -s option - http://permalink.gmane.org/gmane.comp.python.devel/91063 + https://mail.python.org/pipermail/python-dev/2008-January/076130.html .. [10] Discussion about the bin directory - http://permalink.gmane.org/gmane.comp.python.devel/91095 + https://mail.python.org/pipermail/python-dev/2008-January/076162.html .. [11] freedesktop.org XGD basedir specs mentions ~/.local - http://www.freedesktop.org/wiki/Specifications/basedir-spec + https://www.freedesktop.org/wiki/Specifications/basedir-spec/ .. [12] ~/.local for Mac and usercustomize file - http://permalink.gmane.org/gmane.comp.python.devel/91167 + https://mail.python.org/pipermail/python-dev/2008-January/076236.html .. [13] Roaming profile on Windows - http://permalink.gmane.org/gmane.comp.python.devel/91187 - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: + https://mail.python.org/pipermail/python-dev/2008-January/076256.html diff --git a/pep-0371.txt b/peps/pep-0371.rst similarity index 93% rename from pep-0371.txt rename to peps/pep-0371.rst index 6b197cf4c..9e47a7c98 100644 --- a/pep-0371.txt +++ b/peps/pep-0371.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 06-May-2008 Python-Version: 2.6, 3.0 -Post-History: +Post-History: `03-Jun-2008 `__ Abstract @@ -56,7 +56,7 @@ their programming paradigm (i.e.: dropping threaded programming for another "concurrent" approach - Twisted, Actors, etc). The Processing package offers CPython a "known API" which mirrors -albeit in a PEP 8 compliant manner, that of the threading API, +albeit in a :pep:`8` compliant manner, that of the threading API, with known semantics and easy scalability. In the future, the package might not be as relevant should the @@ -82,7 +82,7 @@ simple change of the import to:: from processing import process as worker The code would now execute through the processing.process class. -Obviously, with the renaming of the API to PEP 8 compliance there +Obviously, with the renaming of the API to :pep:`8` compliance there would be additional renaming which would need to occur within user applications, however minor. @@ -144,7 +144,7 @@ All of the code for this can be downloaded from http://jessenoller.com/code/bench-src.tgz The basic method of execution for these benchmarks is in the -run_benchmarks.py script, which is simply a wrapper to execute a +run_benchmarks.py [6]_ script, which is simply a wrapper to execute a target function through a single threaded (linear), multi-threaded (via threading), and multi-process (via pyprocessing) function for a static number of iterations with increasing numbers of execution @@ -331,9 +331,9 @@ API Naming While the aim of the package's API is designed to closely mimic that of the threading and ``Queue`` modules as of python 2.x, those modules are not -PEP 8 compliant. It has been decided that instead of adding the package -"as is" and therefore perpetuating the non-PEP 8 compliant naming, we -will rename all APIs, classes, etc to be fully PEP 8 compliant. +:pep:`8` compliant. It has been decided that instead of adding the package +"as is" and therefore perpetuating the non-:pep:`8` compliant naming, we +will rename all APIs, classes, etc to be fully :pep:`8` compliant. This change does affect the ease-of-drop in replacement for those using the threading module, but that is an acceptable side-effect in the view @@ -341,11 +341,11 @@ of the authors, especially given that the threading module's own API will change. Issue 3042 in the tracker proposes that for Python 2.6 there will be -two APIs for the threading module - the current one, and the PEP 8 +two APIs for the threading module - the current one, and the :pep:`8` compliant one. Warnings about the upcoming removal of the original java-style API will be issued when -3 is invoked. -In Python 3000, the threading API will become PEP 8 compliant, which +In Python 3000, the threading API will become :pep:`8` compliant, which means that the multiprocessing module and the threading module will again have matching APIs. @@ -403,19 +403,19 @@ Closed Issues References ========== -.. [1] PyProcessing home page - http://pyprocessing.berlios.de/ +.. [1] The 2008 era PyProcessing project (the pyprocessing name was since repurposed) + https://web.archive.org/web/20080914113946/https://pyprocessing.berlios.de/ .. [2] See Adam Olsen's "safe threading" project - http://code.google.com/p/python-safethread/ + https://code.google.com/archive/p/python-safethread/ .. [3] See: Addition of "pyprocessing" module to standard lib. https://mail.python.org/pipermail/python-dev/2008-May/079417.html -.. [4] http://mpi4py.scipy.org/ +.. [4] https://mpi4py.readthedocs.io/ .. [5] See "Cluster Computing" - http://wiki.python.org/moin/ParallelProcessing + https://wiki.python.org/moin/ParallelProcessing#Cluster_Computing .. [6] The original run_benchmark.py code was published in Python Magazine in December 2007: "Python Threads and the Global @@ -424,20 +424,7 @@ References .. [7] http://groups.google.com/group/python-dev2/msg/54cf06d15cbcbc34 -.. [8] Addition Python-Dev discussion - https://mail.python.org/pipermail/python-dev/2008-June/080011.html - Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0372.txt b/peps/pep-0372.rst similarity index 94% rename from pep-0372.txt rename to peps/pep-0372.rst index ffd102fc0..84d044330 100644 --- a/pep-0372.txt +++ b/peps/pep-0372.rst @@ -1,8 +1,6 @@ PEP: 372 Title: Adding an ordered dictionary to collections -Version: $Revision$ -Last-Modified: $Date$ -Author: Armin Ronacher +Author: Armin Ronacher , Raymond Hettinger Status: Final Type: Standards Track @@ -27,7 +25,7 @@ Patch A working Py3.1 patch including tests and documentation is at: - `OrderedDict patch `_ + `OrderedDict patch `_ The check-in was in revisions: 70101 and 70102 @@ -69,7 +67,7 @@ situations: Additionally, many ordered dicts are implemented in an inefficient way, making many operations more complex then they have to be. -- PEP 3115 allows metaclasses to change the mapping object used for +- :pep:`3115` allows metaclasses to change the mapping object used for the class body. An ordered dict could be used to create ordered member declarations similar to C structs. This could be useful, for example, for future ``ctypes`` releases as well as ORMs that define @@ -148,7 +146,7 @@ constructor? Is the ordered dict a dict subclass? Why? - Yes. Like ``defaultdict``, an ordered dictionary `` subclasses ``dict``. + Yes. Like ``defaultdict``, an ordered dictionary subclasses ``dict``. Being a dict subclass make some of the methods faster (like ``__getitem__`` and ``__len__``). More importantly, being a dict subclass lets ordered dictionaries be usable with tools like json that @@ -197,7 +195,7 @@ How well does OrderedDict work with the json module, PyYAML, and ConfigParser? In Py2.6, the object_hook for json decoders passes-in an already built dictionary so order is lost before the object hook sees it. This problem is being fixed for Python 2.7/3.1 by adding a new hook that - preserves order (see http://bugs.python.org/issue5381 ). + preserves order (see https://github.com/python/cpython/issues/49631 ). With the new hook, order can be preserved:: >>> jtext = '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}' @@ -264,7 +262,7 @@ Reference Implementation An implementation with tests and documentation is at: - `OrderedDict patch `_ + `OrderedDict patch `_ The proposed version has several merits: @@ -308,21 +306,10 @@ of the source file. References ========== -.. [1] http://bugs.python.org/issue1371075 +.. [1] https://github.com/python/cpython/issues/42649 Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0373.txt b/peps/pep-0373.rst similarity index 96% rename from pep-0373.txt rename to peps/pep-0373.rst index 2adcb8149..3a2188b49 100644 --- a/pep-0373.txt +++ b/peps/pep-0373.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Benjamin Peterson Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 03-Nov-2008 Python-Version: 2.7 @@ -28,14 +29,14 @@ Update (April 2014) The End Of Life date (EOL, sunset date) for Python 2.7 has been moved five years into the future, to 2020. This decision was made to clarify the status of Python 2.7 and relieve worries for those users -who cannot yet migrate to Python 3. See also PEP 466. +who cannot yet migrate to Python 3. See also :pep:`466`. This declaration does not guarantee that bugfix releases will be made on a regular basis, but it should enable volunteers who want to contribute bugfixes for Python 2.7 and it should satisfy vendors who still have to support Python 2 for years to come. -There will be no Python 2.8 (see PEP 404). +There will be no Python 2.8 (see :pep:`404`). Release Manager and Crew diff --git a/pep-0374.txt b/peps/pep-0374.rst similarity index 99% rename from pep-0374.txt rename to peps/pep-0374.rst index e41ffbfdb..49c944e8f 100644 --- a/pep-0374.txt +++ b/peps/pep-0374.rst @@ -11,7 +11,7 @@ Status: Final Type: Process Content-Type: text/x-rst Created: 07-Nov-2008 -Post-History: 07-Nov-2008 +Post-History: 07-Nov-2008, 22-Jan-2009 @@ -274,8 +274,8 @@ This identity may be associated with a full name. All of the DVCSs will query the system to get some approximation to this information, but that may not be what you want. They also -support setting this information on a per-user basis, and on a per- -project basis. Convenience commands to set these attributes vary, +support setting this information on a per-user basis, and on a +per-project basis. Convenience commands to set these attributes vary, but all allow direct editing of configuration files. Some VCSs support end-of-line (EOL) conversions on checkout/checkin. @@ -663,7 +663,7 @@ hg git ''' We assume a patch created by git-format-patch. This is a Unix mbox -file containing one or more patches, each formatted as an RFC 2822 +file containing one or more patches, each formatted as an :rfc:`2822` message. git-am interprets each message as a commit as follows. The author of the patch is taken from the From: header, the date from the Date header. The commit log is created by concatenating the content @@ -1101,7 +1101,7 @@ looms. Doing a Python Release ---------------------- -How does PEP 101 change when using a DVCS? +How does :pep:`101` change when using a DVCS? bzr @@ -1122,7 +1122,7 @@ release off the bzr/hg mirrors. hg '' -Clearly, details specific to Subversion in PEP 101 and in the +Clearly, details specific to Subversion in :pep:`101` and in the release script will need to be updated. In particular, release tagging and maintenance branches creation process will have to be modified to use Mercurial's features; this will simplify and @@ -1508,7 +1508,7 @@ was to be the next version control system for Python. Transition Plan =============== -PEP 385 outlines the transition from svn to hg. +:pep:`385` outlines the transition from svn to hg. Copyright diff --git a/pep-0375.txt b/peps/pep-0375.rst similarity index 94% rename from pep-0375.txt rename to peps/pep-0375.rst index b30951251..c3929cb90 100644 --- a/pep-0375.txt +++ b/peps/pep-0375.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Benjamin Peterson Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 08-Feb-2009 Python-Version: 3.1 @@ -67,7 +68,7 @@ Features for 3.1 - importlib - io in C - Update simplejson to the latest external version [#simplejson]_. -- Ordered dictionary for collections [#ordered]_. +- Ordered dictionary for collections (:pep:`372`). - auto-numbered replacement fields in str.format() strings [#strformat]_ - Nested with-statements in one with statement @@ -78,9 +79,6 @@ Footnotes .. [#simplejson] http://bugs.python.org/issue4136 -.. [#ordered] PEP 372 - http://www.python.org/dev/peps/pep-0372/ - .. [#strformat] http://bugs.python.org/issue5237 diff --git a/pep-0376.txt b/peps/pep-0376.rst similarity index 96% rename from pep-0376.txt rename to peps/pep-0376.rst index 82481aada..60119c69e 100644 --- a/pep-0376.txt +++ b/peps/pep-0376.rst @@ -1,14 +1,17 @@ PEP: 376 Title: Database of Installed Python Distributions -Version: $Revision$ -Last-Modified: $Date$ Author: Tarek ZiadĂ© Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 22-Feb-2009 Python-Version: 2.7, 3.2 -Post-History: +Post-History: `22-Jun-2009 `__ + + +.. canonical-pypa-spec:: :ref:`packaging:core-metadata` + Abstract ======== @@ -21,10 +24,10 @@ To achieve this goal, the PEP proposes a new format to describe installed distributions on a system. It also describes a reference implementation for the standard library. -In the past an attempt was made to create an installation database (see PEP 262 -[#pep262]_). +In the past an attempt was made to create an installation database +(see :pep:`262`). -Combined with PEP 345, the current proposal supersedes PEP 262. +Combined with :pep:`345`, the current proposal supersedes :pep:`262`. Note: the implementation plan didn't go as expected, so it should be considered informative only for this PEP. @@ -61,7 +64,7 @@ extra module and executable scripts, three elements are installed in - ``docutils``: The ``docutils`` package. - ``roman.py``: An extra module used by ``docutils``. - ``docutils-0.5-py2.6.egg-info``: A file containing the distribution metadata - as described in PEP 314 [#pep314]_. This file corresponds to the file + as described in :pep:`314`. This file corresponds to the file called ``PKG-INFO``, built by the ``sdist`` command. Some executable scripts, such as ``rst2html.py``, are also added in the @@ -151,7 +154,7 @@ This distinct directory is named as follows:: This ``.dist-info`` directory can contain these files: -- ``METADATA``: contains metadata, as described in PEP 345, PEP 314 and PEP 241. +- ``METADATA``: contains metadata, as described in :pep:`345`, :pep:`314` and :pep:`241`. - ``RECORD``: records the list of installed files - ``INSTALLER``: records the name of the tool used to install the project - ``REQUESTED``: the presence of this file indicates that the project @@ -187,7 +190,7 @@ should be used when creating system packages. Third-party installation tools also should not overwrite or delete files that are not in a RECORD file without prompting or warning. -This RECORD file is inspired from PEP 262 FILES [#pep262]_. +This RECORD file is inspired from :pep:`262` FILES. The ``RECORD`` file is a CSV file, composed of records, one line per installed file. The ``csv`` module is used to read the file, with @@ -239,7 +242,7 @@ The ``csv`` module is used to generate this file, so the field separator is ``csv``. When the file is read, the ``U`` option is used so the universal newline -support (see PEP 278 [#pep278]_) is activated, avoiding any trouble +support (see :pep:`278`) is activated, avoiding any trouble reading a file produced on a platform that uses a different new line terminator. @@ -443,8 +446,8 @@ And following methods: Notice that the API is organized in five classes that work with directories -and Zip files (so it works with files included in Zip files, see PEP 273 for -more details [#pep273]_). These classes are described in the documentation +and Zip files (so it works with files included in Zip files, see :pep:`273` for +more details). These classes are described in the documentation of the prototype implementation for interested readers [#prototype]_. Examples @@ -498,7 +501,7 @@ Distutils already provides a very basic way to install a distribution, which is running the ``install`` command over the ``setup.py`` script of the distribution. -Distutils2 [#pep262]_ will provide a very basic ``uninstall`` function, that +:pep:`Distutils2 <262>` will provide a very basic ``uninstall`` function, that is added in ``distutils2.util`` and takes the name of the distribution to uninstall as its argument. ``uninstall`` uses the APIs described earlier and remove all unique files, as long as their hash didn't change. Then it removes @@ -610,12 +613,6 @@ References .. [#distutils2] http://hg.python.org/distutils2 -.. [#pep262] - http://www.python.org/dev/peps/pep-0262 - -.. [#pep314] - http://www.python.org/dev/peps/pep-0314 - .. [#setuptools] http://peak.telecommunity.com/DevCenter/setuptools @@ -628,12 +625,6 @@ References .. [#eggformats] http://peak.telecommunity.com/DevCenter/EggFormats -.. [#pep273] - http://www.python.org/dev/peps/pep-0273 - -.. [#pep278] - http://www.python.org/dev/peps/pep-0278 - .. [#fedora] http://fedoraproject.org/wiki/Packaging/Python/Eggs#Providing_Eggs_using_Setuptools diff --git a/pep-0377.txt b/peps/pep-0377.rst similarity index 98% rename from pep-0377.txt rename to peps/pep-0377.rst index 7f313256d..ef3184bc3 100644 --- a/pep-0377.txt +++ b/peps/pep-0377.rst @@ -51,7 +51,7 @@ clause unbound in this case, a new ``StatementSkipped`` singleton (similar to the existing ``NotImplemented`` singleton) will be assigned to all names that appear in the ``as`` clause. -The components of the ``with`` statement remain as described in PEP 343 [2]_:: +The components of the ``with`` statement remain as described in :pep:`343`:: with EXPR as VAR: BLOCK @@ -266,9 +266,6 @@ References .. [1] Issue 5251: contextlib.nested inconsistent with nested with statements (http://bugs.python.org/issue5251) -.. [2] PEP 343: The "with" Statement - (http://www.python.org/dev/peps/pep-0343/) - .. [3] Import-style syntax to reduce indentation of nested with statements (https://mail.python.org/pipermail/python-ideas/2009-March/003188.html) diff --git a/pep-0378.txt b/peps/pep-0378.rst similarity index 99% rename from pep-0378.txt rename to peps/pep-0378.rst index 2eca31335..da2293915 100644 --- a/pep-0378.txt +++ b/peps/pep-0378.rst @@ -92,7 +92,7 @@ Current Version of the Mini-Language .. _Python 2.6 docs: https://docs.python.org/2.6/library/string.html#formatstrings -* PEP 3101 Advanced String Formatting +* :pep:`3101` Advanced String Formatting Research into what Other Languages Do diff --git a/pep-0379.txt b/peps/pep-0379.rst similarity index 100% rename from pep-0379.txt rename to peps/pep-0379.rst diff --git a/pep-0380.txt b/peps/pep-0380.rst similarity index 100% rename from pep-0380.txt rename to peps/pep-0380.rst diff --git a/pep-0381.txt b/peps/pep-0381.rst similarity index 86% rename from pep-0381.txt rename to peps/pep-0381.rst index 6af6e5c94..6aac3a78c 100644 --- a/pep-0381.txt +++ b/peps/pep-0381.rst @@ -1,10 +1,9 @@ PEP: 381 Title: Mirroring infrastructure for PyPI -Version: $Revision$ -Last-Modified: $Date$ -Author: Tarek ZiadĂ© , Martin v. Löwis +Author: Tarek ZiadĂ© , Martin von Löwis Status: Withdrawn Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 21-Mar-2009 Post-History: @@ -38,8 +37,8 @@ Rationale ========= PyPI is hosting over 6000 projects and is used on a daily basis -by people to build applications. Especially systems like `easy_install` -and `zc.buildout` make intensive usage of PyPI. +by people to build applications. Especially systems like ``easy_install`` +and ``zc.buildout`` make intensive usage of PyPI. For people making intensive use of PyPI, it can act as a single point of failure. People have started to set up some mirrors, both private @@ -75,19 +74,19 @@ last host name. Mirror operators should use a static address, and report planned changes to that address in advance to distutils-sig. -The new mirror also appears at `http://pypi.python.org/mirrors` +The new mirror also appears at ``http://pypi.python.org/mirrors`` which is a human-readable page that gives the list of mirrors. This page also explains how to register a new mirror. Statistics page ::::::::::::::: -PyPI provides statistics on downloads at `/stats`. This page is +PyPI provides statistics on downloads at ``/stats``. This page is calculated daily by PyPI, by reading all mirrors' local stats and summing them. -The stats are presented in daily or monthly files, under `/stats/days` -and `/stats/months`. Each file is a `bzip2` file with these formats: +The stats are presented in daily or monthly files, under ``/stats/days`` +and ``/stats/months``. Each file is a ``bzip2`` file with these formats: - YYYY-MM-DD.bz2 for daily files - YYYY-MM.bz2 for monthly files @@ -119,7 +118,7 @@ attack, package authors need to sign their packages using PGP keys, so that users verify that the package comes from the author they trust. The central index provides a DSA key at the URL /serverkey, in the PEM -format as generated by "openssl dsa -pubout" (i.e. RFC 3280 +format as generated by "openssl dsa -pubout" (i.e. :rfc:`3280` SubjectPublicKeyInfo, with the algorithm 1.3.14.3.2.12). This URL must *not* be mirrored, and clients must fetch the official serverkey from PyPI directly, or use the copy that came with the PyPI client software. @@ -127,8 +126,8 @@ Mirrors should still download the key, to detect a key rollover. For each package, a mirrored signature is provided at /serversig/. This is the DSA signature of the parallel URL -/simple/, in DER form, using SHA-1 with DSA (i.e. as a RFC -3279 Dsa-Sig-Value, created by algorithm 1.2.840.10040.4.3) +/simple/, in DER form, using SHA-1 with DSA (i.e. as a +:rfc:`3279` Dsa-Sig-Value, created by algorithm 1.2.840.10040.4.3) Clients using a mirror need to perform the following steps to verify a package: @@ -181,7 +180,7 @@ that represents the last synchronisation date the mirror maintains. The date is provided in GMT time, using the ISO 8601 format [#iso8601]_. Each mirror will be responsible to maintain its last modified date. -This page must be located at : `/last-modified` and must be a +This page must be located at : ``/last-modified`` and must be a text/plain page. Local statistics @@ -192,8 +191,8 @@ via it. This is used by PyPI to sum up all downloads, to be able to display the grand total. These statistics are in CSV-like form, with a header in the first -line. It needs to obey PEP 305 [#pep305]_. Basically, it should be -readable by Python's `csv` module. +line. It needs to obey :pep:`305`. Basically, it should be +readable by Python's ``csv`` module. The fields in this file are: @@ -210,26 +209,26 @@ The content will look like this:: ... The counting starts the day the mirror is launched, and there is one -file per day, compressed using the `bzip2` format. Each file is named -like the day. For example, `2008-11-06.bz2` is the file for the 6th of +file per day, compressed using the ``bzip2`` format. Each file is named +like the day. For example, ``2008-11-06.bz2`` is the file for the 6th of November 2008. -They are then provided in a folder called `days`. For example: +They are then provided in a folder called ``days``. For example: - /local-stats/days/2008-11-06.bz2 - /local-stats/days/2008-11-07.bz2 - /local-stats/days/2008-11-08.bz2 -This page must be located at `/local-stats`. +This page must be located at ``/local-stats``. How a mirror should synchronize with PyPI ========================================= -A mirroring protocol called `Simple Index` was described and +A mirroring protocol called ``Simple Index`` was described and implemented by Martin v. Loewis and Jim Fulton, based on how -`easy_install` works. This section synthesizes it and gives a few -relevant links, plus a small part about `User-Agent`. +``easy_install`` works. This section synthesizes it and gives a few +relevant links, plus a small part about ``User-Agent``. The mirroring protocol :::::::::::::::::::::: @@ -268,7 +267,7 @@ How a client can use PyPI and its mirrors ::::::::::::::::::::::::::::::::::::::::: Clients that are browsing PyPI should be able to use alternative -mirrors, by getting the list of the mirrors using `last.pypi.python.org`. +mirrors, by getting the list of the mirrors using ``last.pypi.python.org``. Code example:: @@ -306,11 +305,11 @@ Extra package indexes It is obvious that some packages will not be uploaded to PyPI, whether because they are private or whether because the project maintainer -runs his own server where people might get the project package. +runs their own server where people might get the project package. However, it is strongly encouraged that a public package index follows PyPI and Distutils protocols. -In other words, the `register` and `upload` command should be +In other words, the ``register`` and ``upload`` command should be compatible with any package index server out there. Software that are compatible with PyPI and Distutils so far: @@ -343,9 +342,6 @@ It is up the client to deal with the merging. References ========== -.. [#pep305] - http://www.python.org/dev/peps/pep-0305/#id19 - .. [#pep381client] http://pypi.python.org/pypi/pep381client @@ -378,13 +374,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0382.txt b/peps/pep-0382.rst similarity index 94% rename from pep-0382.txt rename to peps/pep-0382.rst index bc18474e6..ecbbaeacc 100644 --- a/pep-0382.txt +++ b/peps/pep-0382.rst @@ -1,8 +1,6 @@ PEP: 382 Title: Namespace Packages -Version: $Revision$ -Last-Modified: $Date$ -Author: Martin v. Löwis +Author: Martin von Löwis Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -14,7 +12,7 @@ Rejection Notice ================ On the first day of sprints at US PyCon 2012 we had a long and -fruitful discussion about PEP 382 and PEP 402. We ended up rejecting +fruitful discussion about :pep:`382` and :pep:`402`. We ended up rejecting both but a new PEP will be written to carry on in the spirit of PEP 402. Martin von Löwis wrote up a summary: [2]_. @@ -131,7 +129,7 @@ this: Impact on Import Hooks ---------------------- -Both loaders and finders as defined in PEP 302 will need to be changed +Both loaders and finders as defined in :pep:`302` will need to be changed to support namespace packages. Failure to conform to the protocol below might cause a package not being recognized as a namespace package; loaders and finders not supporting this protocol must raise @@ -178,9 +176,9 @@ Dinu Gherman then observed that using a marker file is not necessary, and that a directory extension could well serve as a such as a marker. This is what this PEP currently proposes. -Phillip Eby designed PEP 402 as an alternative approach to this PEP, +Phillip Eby designed :pep:`402` as an alternative approach to this PEP, after comparing Python's package syntax with that found in other -languages. PEP 402 proposes not to use a marker file at all. At the +languages. :pep:`402` proposes not to use a marker file at all. At the discussion at PyCon DE 2011, people remarked that having an explicit declaration of a directory as contributing to a package is a desirable property, rather than an obstacle. In particular, Jython developers @@ -215,13 +213,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0383.txt b/peps/pep-0383.rst similarity index 92% rename from pep-0383.txt rename to peps/pep-0383.rst index f332ae78a..cce198367 100644 --- a/pep-0383.txt +++ b/peps/pep-0383.rst @@ -1,8 +1,6 @@ PEP: 383 Title: Non-decodable Bytes in System Character Interfaces -Version: $Revision$ -Last-Modified: $Date$ -Author: Martin v. Löwis +Author: Martin von Löwis Status: Final Type: Standards Track Content-Type: text/x-rst @@ -64,7 +62,7 @@ Specification On Windows, Python uses the wide character APIs to access character-oriented APIs, allowing direct conversion of the -environmental data to Python str objects ([1]_). +environmental data to Python str objects (:pep:`277`). On POSIX systems, Python currently applies the locale's encoding to convert the byte data to Unicode, failing for characters that cannot @@ -72,7 +70,7 @@ be decoded. With this PEP, non-decodable bytes >= 128 will be represented as lone surrogate codes U+DC80..U+DCFF. Bytes below 128 will produce exceptions; see the discussion below. -To convert non-decodable bytes, a new error handler ([2]_) +To convert non-decodable bytes, a new error handler (:pep:`293`) "surrogateescape" is introduced, which produces these surrogates. On encoding, the error handler converts the surrogate back to the corresponding byte. This error handler will be used in any API that @@ -102,7 +100,7 @@ has the limitation that chosen representation only "works" if the data get converted back to bytes with the surrogateescape error handler also. Encoding the data with the locale's encoding and the (default) strict error handler will raise an exception, encoding them with UTF-8 -will produce non-sensical data. +will produce nonsensical data. Data obtained from other sources may conflict with data produced by this PEP. Dealing with such conflicts is out of scope of the PEP. @@ -171,28 +169,10 @@ case; other libraries may show similar problems. References ========== -.. [1] PEP 277 - "Unicode file name support for Windows NT" - http://www.python.org/dev/peps/pep-0277/ - -.. [2] PEP 293 - "Codec Error Handling Callbacks" - http://www.python.org/dev/peps/pep-0293/ - .. [3] UTF-8b - http://permalink.gmane.org/gmane.comp.internationalization.linux/920 + https://web.archive.org/web/20090830064219/http://mail.nl.linux.org/linux-utf8/2000-07/msg00040.html Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0384.txt b/peps/pep-0384.rst similarity index 97% rename from pep-0384.txt rename to peps/pep-0384.rst index 9100ddfdb..867173da9 100644 --- a/pep-0384.txt +++ b/peps/pep-0384.rst @@ -1,8 +1,6 @@ PEP: 384 Title: Defining a Stable ABI -Version: $Revision$ -Last-Modified: $Date$ -Author: Martin v. Löwis +Author: Martin von Löwis Status: Final Type: Standards Track Content-Type: text/x-rst @@ -10,6 +8,9 @@ Created: 17-May-2009 Python-Version: 3.2 Post-History: +.. canonical-doc:: :ref:`python:stable` (user docs) and + :ref:`devguide:c-api` (development docs) + Abstract ======== @@ -344,7 +345,7 @@ applications conforming to this PEP should then link to the former but rather rely on runtime linking). The ABI version is symbolically available as ``PYTHON_ABI_VERSION``. -Also on Unix, the PEP 3149 tag abi is accepted +Also on Unix, the :pep:`3149` tag abi is accepted in file names of extension modules. No checking is performed that files named in this way are actually restricted to the limited API, and no support for building such files will be added to distutils @@ -372,13 +373,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0385.txt b/peps/pep-0385.rst similarity index 99% rename from pep-0385.txt rename to peps/pep-0385.rst index 8495717a6..6a4d20e7a 100644 --- a/pep-0385.txt +++ b/peps/pep-0385.rst @@ -19,7 +19,7 @@ migration still has to be performed. In the case of an important piece of infrastructure like the version control system for a large, distributed project like Python, this is a significant effort. This PEP is an attempt to describe the steps that must be taken for further -discussion. It's somewhat similar to `PEP 347`_, which discussed the +discussion. It's somewhat similar to :pep:`347`, which discussed the migration to SVN. To make the most of hg, we would like to make a high-fidelity @@ -37,7 +37,6 @@ contents of the repository and determine if some things are still valuable. In this spirit, the following sections also propose discarding some of the older metadata. -.. _PEP 347: http://www.python.org/dev/peps/pep-0347/ .. _hgsubversion: http://bitbucket.org/durin42/hgsubversion/ diff --git a/pep-0386.txt b/peps/pep-0386.rst similarity index 96% rename from pep-0386.txt rename to peps/pep-0386.rst index ba009ff87..6197dbc8a 100644 --- a/pep-0386.txt +++ b/peps/pep-0386.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Tarek ZiadĂ© Status: Superseded Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 04-Jun-2009 Superseded-By: 440 @@ -14,7 +15,7 @@ Abstract ======== Note: This PEP has been superseded by the version identification and -dependency specification scheme defined in PEP 440. +dependency specification scheme defined in :pep:`440`. This PEP proposed a new version comparison schema system in Distutils. @@ -24,13 +25,13 @@ Motivation In Python there are no real restrictions yet on how a project should manage its versions, and how they should be incremented. -Distutils provides a `version` distribution meta-data field but it is freeform and +Distutils provides a ``version`` distribution meta-data field but it is freeform and current users, such as PyPI usually consider the latest version pushed as the -`latest` one, regardless of the expected semantics. +``latest`` one, regardless of the expected semantics. Distutils will soon extend its capabilities to allow distributions to express a dependency on other distributions through the ``Requires-Dist`` metadata field -(see PEP 345) and it will optionally allow use of that field to +(see :pep:`345`) and it will optionally allow use of that field to restrict the dependency to a set of compatible versions. Notice that this field is replacing ``Requires`` that was expressing dependencies on modules and packages. @@ -270,8 +271,8 @@ a particular package was using and to provide tools on top of PyPI. Distutils classes are not really used in Python projects, but the Setuptools function is quite widespread because it's used by tools like -`easy_install` [#ezinstall]_, `pip` [#pip]_ or `zc.buildout` [#zc.buildout]_ -to install dependencies of a given project. +``easy_install`` [#ezinstall]_, ``pip`` [#pip]_ or ``zc.buildout`` +[#zc.buildout]_ to install dependencies of a given project. While Setuptools *does* provide a mechanism for comparing/sorting versions, it is much preferable if the versioning spec is such that a human can make a @@ -291,7 +292,7 @@ The new versioning algorithm During Pycon, members of the Python, Ubuntu and Fedora community worked on a version standard that would be acceptable for everyone. -It's currently called `verlib` and a prototype lives at [#prototype]_. +It's currently called ``verlib`` and a prototype lives at [#prototype]_. The pseudo-format supported is:: @@ -361,9 +362,9 @@ Note that ``c`` is the preferred marker for third party projects. NormalizedVersion ----------------- -The `NormalizedVersion` class is used to hold a version and to compare it with -others. It takes a string as an argument, that contains the representation of -the version:: +The ``NormalizedVersion`` class is used to hold a version and to compare it +with others. It takes a string as an argument, that contains the representation +of the version:: >>> from verlib import NormalizedVersion >>> version = NormalizedVersion('1.0') diff --git a/pep-0387.txt b/peps/pep-0387.rst similarity index 66% rename from pep-0387.txt rename to peps/pep-0387.rst index 031059c1d..1915dc648 100644 --- a/pep-0387.txt +++ b/peps/pep-0387.rst @@ -3,13 +3,14 @@ Title: Backwards Compatibility Policy Version: $Revision$ Last-Modified: $Date$ Author: Benjamin Peterson -BDFL-Delegate: Brett Cannon (on behalf of the steering council) -Discussions-To: https://discuss.python.org/t/pep-387-backwards-compatibilty-policy/ +PEP-Delegate: Brett Cannon Status: Active Type: Process Content-Type: text/x-rst Created: 18-Jun-2009 -Post-History: 19-Jun-2009, 12-Jun-2020 +Post-History: `19-Jun-2009 `__, + `12-Jun-2020 `__, + `16-Jun-2023 `__ Abstract @@ -67,6 +68,8 @@ be removed at any time in any way. These include: types that are prefixed by "_" (except special names). - Anything documented publicly as being private. + Note that if something is not documented at all, it is *not* + automatically considered private. - Imported modules (unless explicitly documented as part of the public API; e.g. importing the ``bacon`` module in the ``spam`` does not @@ -79,7 +82,7 @@ be removed at any time in any way. These include: subdirectories of packages.) - Backward compatibility rules do not apply to any module or API that is - explicitly documented as **Provisional** per PEP 411. + explicitly documented as **Provisional** per :pep:`411`. Basic policy for backwards compatibility @@ -101,6 +104,9 @@ Basic policy for backwards compatibility * Similarly a feature cannot be removed without notice between any two consecutive releases. +* For changes that are unable to raise a deprecation warning, consult + with the steering council. + * The steering council may grant exceptions to this policy. In particular, they may shorten the required deprecation period for a feature. Exceptions are only granted for extreme situations such as @@ -109,6 +115,30 @@ Basic policy for backwards compatibility platforms). +Soft Deprecation +================ + +A soft deprecation can be used when using an API which should no longer +be used to write new code, but it remains safe to continue using it in +existing code. The API remains documented and tested, but will not be +developed further (no enhancement). + +The main difference between a "soft" and a (regular) "hard" deprecation +is that the soft deprecation does not imply scheduling the removal of +the deprecated API. + +Another difference is that a soft deprecation does not issue a warning: +it's only mentioned in the documentation, whereas usually a "hard" +deprecation issues a ``DeprecationWarning`` warning at runtime. The +documentation of a soft deprecation should explain why the API should be +avoided, and if possible propose a replacement. + +If the decision is made to deprecate (in the regular sense) a feature +that is currently soft deprecated, the deprecation must follow the +`Backwards Compatibility Rules`_ (i.e., there is no exception because +the feature is already soft deprecated). + + Making Incompatible Changes =========================== @@ -120,22 +150,35 @@ several releases: appropriate SIG. A PEP or similar document may be written. Hopefully users of the affected API will pipe up to comment. -2. Add a warning. If behavior is changing, the API may gain a new +2. Add a warning to the current ``main`` branch. + If behavior is changing, the API may gain a new function or method to perform the new behavior; old usage should raise the warning. If an API is being removed, simply warn whenever it is entered. ``DeprecationWarning`` is the usual warning category to use, but ``PendingDeprecationWarning`` may be used in special cases where the old and new versions of the API will - coexist for many releases [#warnings]_. Compiler warnings are also - acceptable. The warning message should include the release the - incompatibility is expected to become the default and a link to an - issue that users can post feedback to. + coexist for many releases [#warnings]_. The warning message should + include the release the incompatibility is expected to become the + default and a link to an issue that users can post feedback to. + + For C API, a compiler warning generated by the ``Py_DEPRECATED`` macro + is also acceptable. 3. Wait for the warning to appear in at least two minor Python versions of the same major version, or one minor version in an older - major version (e.g. for a warning in Python 3.10, you either wait + major version (e.g. for a warning in Python 3.10.0, you either wait until at least Python 3.12 or Python 4.0 to make the change). - It's fine to wait more than two releases. + + It's fine to wait more than two releases, for example: + + - If the expected maintenance overhead and security risk of the + deprecated behavior is small (e.g. an old function is reimplemented + in terms of a new, more general one), it can stay indefinitely + (or until the situation changes). + + - If the deprecated feature is replaced by a new one, it should + generally be removed only after the last Python version + *without* the new feature reaches end of support. 4. See if there's any feedback. Users not involved in the original discussions may comment now after seeing the warning. Perhaps @@ -145,6 +188,9 @@ several releases: permanent having reached the declared version. Remove the old version and warning. +6. If a warning cannot be provided to users, consult with the steering + council. + References ========== diff --git a/pep-0389.txt b/peps/pep-0389.rst similarity index 100% rename from pep-0389.txt rename to peps/pep-0389.rst diff --git a/pep-0390.txt b/peps/pep-0390.rst similarity index 94% rename from pep-0390.txt rename to peps/pep-0390.rst index 92605fde1..502e43a4f 100644 --- a/pep-0390.txt +++ b/peps/pep-0390.rst @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Tarek ZiadĂ© BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 10-Oct-2009 Python-Version: 2.7, 3.2 @@ -26,7 +27,7 @@ Rejection Notice As distutils2 is no longer going to be incorporated into the standard library, this PEP was rejected by Nick Coghlan in late April, 2013. -A replacement PEP based on PEP 426 (metadata 2.0) will be created that +A replacement PEP based on :pep:`426` (metadata 2.0) will be created that defines the minimum amount of information needed to generate an sdist archive given a source tarball or VCS checkout. @@ -34,7 +35,7 @@ archive given a source tarball or VCS checkout. Rationale ========= -Today, if you want to list all the Metadata of a distribution (see PEP 314) +Today, if you want to list all the Metadata of a distribution (see :pep:`314`) that is not installed, you need to use the ``setup.py`` command line interface. So, basically, you download it, and run:: @@ -86,7 +87,7 @@ Multi-lines values ================== Some Metadata fields can have multiple values. To keep ``setup.cfg`` compatible -with ``ConfigParser`` and the RFC 822 ``LONG HEADER FIELDS`` (see section 3.1.1), +with ``ConfigParser`` and the :rfc:`822` ``LONG HEADER FIELDS`` (see section 3.1.1), these are expressed with ``,``-separated values:: requires = pywin32, bar > 1.0, foo @@ -124,7 +125,7 @@ Every ``[metadata:condition]`` section will be used only if the condition is met when the file is read. The background motivation for these context-dependant sections is to be able to define requirements that varies depending on the platform the distribution might be installed on. -(see PEP 314). +(see :pep:`314`). The micro-language behind this is the simplest possible: it compares only strings, with the ``==`` and ``in`` operators (and their opposites), and @@ -165,8 +166,8 @@ Impact on PKG-INFO generation and PEP 314 ========================================= When ``PKG-INFO`` is generated by Distutils, every field that relies on a -condition will have that condition written at the end of the line, after a `;` -separator:: +condition will have that condition written at the end of the line, after a +``;`` separator:: Metadata-Version: 1.2 Name: distribute @@ -206,7 +207,7 @@ for another environment:: >>> metadata.get_requires() ['bar > 1.0', 'foo', 'bar'] -PEP 314 is changed accordingly, meaning that each field will be able to +:pep:`314` is changed accordingly, meaning that each field will be able to have that extra condition marker. Compatibility @@ -224,7 +225,7 @@ Limitations We are not providing ``<`` and ``>`` operators at this time, and ``python_version`` is a regular string. This implies using ``or`` operators when a section needs to be restricted to a couple of Python versions. -Although, if PEP 386 is accepted, ``python_version`` could be changed +Although, if :pep:`386` is accepted, ``python_version`` could be changed internally into something comparable with strings, and ``<`` and ``>`` operators introduced. diff --git a/pep-0391.txt b/peps/pep-0391.rst similarity index 96% rename from pep-0391.txt rename to peps/pep-0391.rst index d16dfd3e9..407ecb4b4 100644 --- a/pep-0391.txt +++ b/peps/pep-0391.rst @@ -79,9 +79,9 @@ to which it must conform). Naming ------ -Historically, the logging package has not been PEP 8 conformant [1]_. +Historically, the logging package has not been :pep:`8` conformant. At some future time, this will be corrected by changing method and -function names in the package in order to conform with PEP 8. +function names in the package in order to conform with :pep:`8`. However, in the interests of uniformity, the proposed additions to the API use a naming scheme which is consistent with the present scheme used by logging. @@ -383,7 +383,7 @@ Dictionary Schema - Detail The dictionary passed to ``dictConfig()`` must contain the following keys: -* `version` - to be set to an integer value representing the schema +* ``version`` - to be set to an integer value representing the schema version. The only valid value at present is 1, but having this key allows the schema to evolve while still preserving backwards compatibility. @@ -395,7 +395,7 @@ custom instantiation is required. If so, the mechanism described above is used to instantiate; otherwise, the context is used to determine how to instantiate. -* `formatters` - the corresponding value will be a dict in which each +* ``formatters`` - the corresponding value will be a dict in which each key is a formatter id and each value is a dict describing how to configure the corresponding Formatter instance. @@ -403,7 +403,7 @@ determine how to instantiate. (with defaults of ``None``) and these are used to construct a ``logging.Formatter`` instance. -* `filters` - the corresponding value will be a dict in which each key +* ``filters`` - the corresponding value will be a dict in which each key is a filter id and each value is a dict describing how to configure the corresponding Filter instance. @@ -411,7 +411,7 @@ determine how to instantiate. empty string) and this is used to construct a ``logging.Filter`` instance. -* `handlers` - the corresponding value will be a dict in which each +* ``handlers`` - the corresponding value will be a dict in which each key is a handler id and each value is a dict describing how to configure the corresponding Handler instance. @@ -451,7 +451,7 @@ determine how to instantiate. ``logging.handlers.RotatingFileHandler`` with the keyword arguments ``filename='logconfig.log', maxBytes=1024, backupCount=3``. -* `loggers` - the corresponding value will be a dict in which each key +* ``loggers`` - the corresponding value will be a dict in which each key is a logger name and each value is a dict describing how to configure the corresponding Logger instance. @@ -470,11 +470,11 @@ determine how to instantiate. The specified loggers will be configured according to the level, propagation, filters and handlers specified. -* `root` - this will be the configuration for the root logger. +* ``root`` - this will be the configuration for the root logger. Processing of the configuration will be as for any logger, except that the ``propagate`` setting will not be applicable. -* `incremental` - whether the configuration is to be interpreted as +* ``incremental`` - whether the configuration is to be interpreted as incremental to the existing configuration. This value defaults to ``False``, which means that the specified configuration replaces the existing configuration with the same semantics as used by the @@ -483,10 +483,10 @@ determine how to instantiate. If the specified value is ``True``, the configuration is processed as described in the section on `Incremental Configuration`_, below. -* `disable_existing_loggers` - whether any existing loggers are to be +* ``disable_existing_loggers`` - whether any existing loggers are to be disabled. This setting mirrors the parameter of the same name in ``fileConfig()``. If absent, this parameter defaults to ``True``. - This value is ignored if `incremental` is ``True``. + This value is ignored if ``incremental`` is ``True``. A Working Example ----------------- @@ -678,13 +678,6 @@ http://bitbucket.org/vinay.sajip/dictconfig This incorporates all features other than the socket listener change. -References -========== - -.. [1] PEP 8, Style Guide for Python Code, van Rossum, Warsaw - (http://www.python.org/dev/peps/pep-0008) - - Copyright ========= diff --git a/pep-0392.txt b/peps/pep-0392.rst similarity index 94% rename from pep-0392.txt rename to peps/pep-0392.rst index 0ce3b5eba..6878dcba2 100644 --- a/pep-0392.txt +++ b/peps/pep-0392.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Georg Brandl Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 30-Dec-2009 Python-Version: 3.2 @@ -110,20 +111,12 @@ Release Schedule Features for 3.2 ================ -Note that PEP 3003 [#pep3003]_ is in effect: no changes to language +Note that :pep:`3003` is in effect: no changes to language syntax and no additions to the builtins may be made. No large-scale changes have been recorded yet. -References -========== - -.. [#pep3003] - http://www.python.org/dev/peps/pep-3003/ - - - Copyright ========= diff --git a/pep-0393.txt b/peps/pep-0393.rst similarity index 98% rename from pep-0393.txt rename to peps/pep-0393.rst index 176dd0170..bee0319fd 100644 --- a/pep-0393.txt +++ b/peps/pep-0393.rst @@ -1,8 +1,6 @@ PEP: 393 Title: Flexible String Representation -Version: $Revision$ -Last-Modified: $Date$ -Author: Martin v. Löwis +Author: Martin von Löwis Status: Final Type: Standards Track Content-Type: text/x-rst @@ -263,7 +261,7 @@ UCS4 utility functions: Stable ABI ---------- -The following functions are added to the stable ABI (PEP 384), as they +The following functions are added to the stable ABI (:pep:`384`), as they are independent of the actual representation of Unicode objects: PyUnicode_New, PyUnicode_Substring, PyUnicode_GetLength, PyUnicode_ReadChar, PyUnicode_WriteChar, PyUnicode_Find, @@ -457,19 +455,9 @@ References .. [1] PEP 393 branch https://bitbucket.org/t0rsten/pep-393 .. [2] Django measurement results - http://www.dcl.hpi.uni-potsdam.de/home/loewis/djmemprof/ + https://web.archive.org/web/20160911215951/http://www.dcl.hpi.uni-potsdam.de/home/loewis/djmemprof/ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0394.txt b/peps/pep-0394.rst similarity index 96% rename from pep-0394.txt rename to peps/pep-0394.rst index c2c83d57e..c3438657f 100644 --- a/pep-0394.txt +++ b/peps/pep-0394.rst @@ -87,7 +87,7 @@ For Python runtime distributors * Changing ``python3`` shebangs to ``python3.8`` if the software is built with Python 3.8. -* When a virtual environment (created by the PEP 405 ``venv`` package or a +* When a virtual environment (created by the :pep:`405` ``venv`` package or a similar tool such as ``virtualenv`` or ``conda``) is active, the ``python`` command should refer to the virtual environment's interpreter and should always be available. @@ -174,7 +174,7 @@ on the part of distribution maintainers. Simplified, the recommendation was: the version explicitly. However, these recommendations implicitly assumed that Python 2 would always be -available. As Python 2 is nearing its end of life in 2020 (PEP 373, PEP 404), +available. As Python 2 is nearing its end of life in 2020 (:pep:`373`, :pep:`404`), distributions are making Python 2 optional or removing it entirely. This means either removing the ``python`` command or switching it to invoke Python 3. Some distributors also decided that their users were better served by @@ -183,8 +183,6 @@ administrators with the freedom to configure their systems based on the needs of their particular environment. -.. _rationale: - Current Rationale ================= @@ -338,7 +336,7 @@ Exclusion of MS Windows This PEP deliberately excludes any proposals relating to Microsoft Windows, as devising an equivalent solution for Windows was deemed too complex to handle -here. PEP 397 and the related discussion on the python-dev mailing list +here. :pep:`397` and the related discussion on the python-dev mailing list address this issue. @@ -348,13 +346,13 @@ References .. [1] Support the /usr/bin/python2 symlink upstream (with bonus grammar class!) (https://mail.python.org/pipermail/python-dev/2011-March/108491.html) -.. [2] Rebooting \PEP 394 (aka Support the /usr/bin/python2 symlink upstream) +.. [2] Rebooting PEP 394 (aka Support the /usr/bin/python2 symlink upstream) (https://mail.python.org/pipermail/python-dev/2011-July/112322.html) -.. [3] Implement \PEP 394 in the CPython Makefile +.. [3] Implement PEP 394 in the CPython Makefile (http://bugs.python.org/issue12627) -.. [4] \PEP 394 request for pronouncement (python2 symlink in \*nix systems) +.. [4] PEP 394 request for pronouncement (python2 symlink in \*nix systems) (https://mail.python.org/pipermail/python-dev/2012-February/116435.html) .. [5] Arch Linux announcement that their "python" link now refers Python 3 @@ -363,7 +361,7 @@ References .. [6] PEP 394 - Clarification of what "python" command should invoke (https://mail.python.org/pipermail/python-dev/2014-September/136374.html) -.. [7] PEP 394: Allow the `python` command to not be installed, and other +.. [7] PEP 394: Allow the ``python`` command to not be installed, and other minor edits (https://github.com/python/peps/pull/630) diff --git a/pep-0395.txt b/peps/pep-0395.rst similarity index 94% rename from pep-0395.txt rename to peps/pep-0395.rst index a67256b2c..04e848206 100644 --- a/pep-0395.txt +++ b/peps/pep-0395.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 04-Mar-2011 Python-Version: 3.4 -Post-History: 5-Mar-2011, 19-Nov-2011 +Post-History: 05-Mar-2011, 19-Nov-2011 PEP Withdrawal @@ -34,7 +34,7 @@ This PEP proposes new mechanisms that eliminate some longstanding traps for the unwary when dealing with Python's import system, as well as serialisation and introspection of functions and classes. -It builds on the "Qualified Name" concept defined in PEP 3155. +It builds on the "Qualified Name" concept defined in :pep:`3155`. Relationship with Other PEPs @@ -42,18 +42,18 @@ Relationship with Other PEPs Most significantly, this PEP is currently deferred as it requires significant changes in order to be made compatible with the removal -of mandatory __init__.py files in PEP 420 (which has been implemented +of mandatory __init__.py files in :pep:`420` (which has been implemented and released in Python 3.3). -This PEP builds on the "qualified name" concept introduced by PEP 3155, and +This PEP builds on the "qualified name" concept introduced by :pep:`3155`, and also shares in that PEP's aim of fixing some ugly corner cases when dealing with serialisation of arbitrary functions and classes. -It also builds on PEP 366, which took initial tentative steps towards making +It also builds on :pep:`366`, which took initial tentative steps towards making explicit relative imports from the main module work correctly in at least *some* circumstances. -Finally, PEP 328 eliminated implicit relative imports from imported modules. +Finally, :pep:`328` eliminated implicit relative imports from imported modules. This PEP proposes that the de facto implicit relative imports from main modules that are provided by the current initialisation behaviour for ``sys.path[0]`` also be eliminated. @@ -173,7 +173,7 @@ executing it directly:: # working directory: project python -c "from package.tests.test_foo import main; main()" -Since the implementation of PEP 366 (which defined a mechanism that allows +Since the implementation of :pep:`366` (which defined a mechanism that allows relative imports to work correctly when a module inside a package is executed via the ``-m`` switch), the following also works properly:: @@ -287,12 +287,12 @@ Qualified Names for Modules To make it feasible to fix these problems once and for all, it is proposed to add a new module level attribute: ``__qualname__``. This abbreviation of -"qualified name" is taken from PEP 3155, where it is used to store the naming +"qualified name" is taken from :pep:`3155`, where it is used to store the naming path to a nested class or function definition relative to the top level module. For modules, ``__qualname__`` will normally be the same as ``__name__``, just -as it is for top-level functions and classes in PEP 3155. However, it will +as it is for top-level functions and classes in :pep:`3155`. However, it will differ in some situations so that the above problems can be addressed. Specifically, whenever ``__name__`` is modified for some other purpose (such @@ -311,22 +311,22 @@ Two alternative names were also considered for the new attribute: "full name" (``__fullname__``) and "implementation name" (``__implname__``). Either of those would actually be valid for the use case in this PEP. -However, as a meta-issue, PEP 3155 is *also* adding a new attribute (for +However, as a meta-issue, :pep:`3155` is *also* adding a new attribute (for functions and classes) that is "like ``__name__``, but different in some cases where ``__name__`` is missing necessary information" and those terms aren't -accurate for the PEP 3155 function and class use case. +accurate for the :pep:`3155` function and class use case. -PEP 3155 deliberately omits the module information, so the term "full name" +:pep:`3155` deliberately omits the module information, so the term "full name" is simply untrue, and "implementation name" implies that it may specify an object other than that specified by ``__name__``, and that is never the -case for PEP 3155 (in that PEP, ``__name__`` and ``__qualname__`` always +case for :pep:`3155` (in that PEP, ``__name__`` and ``__qualname__`` always refer to the same function or class, it's just that ``__name__`` is insufficient to accurately identify nested functions and classes). Since it seems needlessly inconsistent to add *two* new terms for attributes that only exist because backwards compatibility concerns keep us from changing the behaviour of ``__name__`` itself, this PEP instead chose to -adopt the PEP 3155 terminology. +adopt the :pep:`3155` terminology. If the relative inscrutability of "qualified name" and ``__qualname__`` encourages interested developers to look them up at least once rather than @@ -525,7 +525,7 @@ as follows:: Compatibility with PEP 382 ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Making this proposal compatible with the PEP 382 namespace packaging PEP is +Making this proposal compatible with the :pep:`382` namespace packaging PEP is trivial. The semantics of ``_is_package_dir()`` are merely changed to be:: def _is_package_dir(fspath): @@ -537,7 +537,7 @@ trivial. The semantics of ``_is_package_dir()`` are merely changed to be:: Incompatibility with PEP 402 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -PEP 402 proposes the elimination of explicit markers in the file system for +:pep:`402` proposes the elimination of explicit markers in the file system for Python packages. This fundamentally breaks the proposed concept of being able to take a filesystem path and a Python module name and work out an unambiguous mapping to the Python module namespace. Instead, the appropriate mapping @@ -545,7 +545,7 @@ would depend on the current values in ``sys.path``, rendering it impossible to ever fix the problems described above with the calculation of ``sys.path[0]`` when the interpreter is initialised. -While some aspects of this PEP could probably be salvaged if PEP 402 were +While some aspects of this PEP could probably be salvaged if :pep:`402` were adopted, the core concept of making import semantics from main and other modules more consistent would no longer be feasible. @@ -715,27 +715,19 @@ None as yet. References ========== -.. [1] Module aliases and/or "real names" - (https://mail.python.org/pipermail/python-ideas/2011-January/008983.html) +.. [1] `Module aliases and/or "real names" + `__ -.. [2] PEP 395 (Module aliasing) and the namespace PEPs - (https://mail.python.org/pipermail/import-sig/2011-November/000382.html) +.. [2] `PEP 395 (Module aliasing) and the namespace PEPs + `__ -.. [3] Updated PEP 395 (aka "Implicit Relative Imports Must Die!") - (https://mail.python.org/pipermail/import-sig/2011-November/000397.html) +.. [3] `Updated PEP 395 (aka "Implicit Relative Imports Must Die!") + `__ -.. [4] Elaboration of compatibility problems between this PEP and PEP 402 - (https://mail.python.org/pipermail/import-sig/2011-November/000403.html) +* `Elaboration of compatibility problems between this PEP and PEP 402 + `__ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0396.txt b/peps/pep-0396.rst similarity index 92% rename from pep-0396.txt rename to peps/pep-0396.rst index c2e4c0a2d..bc022971f 100644 --- a/pep-0396.txt +++ b/peps/pep-0396.rst @@ -5,9 +5,10 @@ Last-Modified: $Date: 2008-08-10 09:59:20 -0400 (Sun, 10 Aug 2008) $ Author: Barry Warsaw Status: Rejected Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 16-Mar-2011 -Post-History: 2011-04-05 +Post-History: 05-Apr-2011 Abstract @@ -29,7 +30,7 @@ PEP Rejection This PEP was formally rejected on 2021-04-14. The packaging ecosystem has changed significantly in the intervening years since this PEP was -first written, and APIs such as ``importlib.metadata.versions()`` [11]_ +first written, and APIs such as ``importlib.metadata.version()`` [11]_ provide for a much better experience. User Stories @@ -76,7 +77,7 @@ ways have grown organically over the years. Often, version numbers can be retrieved from a module programmatically, by importing the module and inspecting an attribute. Classic Python distutils ``setup()`` functions [3]_ describe a ``version`` argument where the -release's version number can be specified. PEP 8 [4]_ describes the +release's version number can be specified. :pep:`8` describes the use of a module attribute called ``__version__`` for recording "Subversion, CVS, or RCS" version strings using keyword expansion. In the PEP author's own email archives, the earliest example of the use @@ -120,14 +121,14 @@ Specification #. The ``__version__`` attribute's value SHOULD be a string. #. Module version numbers SHOULD conform to the normalized version - format specified in PEP 386 [6]_. + format specified in :pep:`386`. #. Module version numbers SHOULD NOT contain version control system supplied revision numbers, or any other semantically different version numbers (e.g. underlying library version number). #. The ``version`` attribute in a classic distutils ``setup.py`` - file, or the PEP 345 [7]_ ``Version`` metadata field SHOULD be + file, or the :pep:`345` ``Version`` metadata field SHOULD be derived from the ``__version__`` field, or vice versa. @@ -256,10 +257,10 @@ programmatically. E.g. in ``elle.py``:: PEP 376 metadata ================ -PEP 376 [10]_ defines a standard for static metadata, but doesn't +:pep:`376` defines a standard for static metadata, but doesn't describe the process by which this metadata gets created. It is highly desirable for the derived version information to be placed into -the PEP 376 ``.dist-info`` metadata at build-time rather than +the :pep:`376` ``.dist-info`` metadata at build-time rather than install-time. This way, the metadata will be available for introspection even when the code is not installed. @@ -275,26 +276,14 @@ References .. [3] http://docs.python.org/distutils/setupscript.html -.. [4] PEP 8, Style Guide for Python Code - (http://www.python.org/dev/peps/pep-0008) - .. [5] sqlite3 module documentation (http://docs.python.org/library/sqlite3.html) -.. [6] PEP 386, Changing the version comparison module in Distutils - (http://www.python.org/dev/peps/pep-0386/) - -.. [7] PEP 345, Metadata for Python Software Packages 1.2 - (http://www.python.org/dev/peps/pep-0345/#version) - .. [8] pkgutil - Package utilities (http://distutils2.notmyidea.org/library/pkgutil.html) .. [9] https://mail.python.org/pipermail/distutils-sig/2011-June/017862.html -.. [10] PEP 376, Database of Installed Python Distributions - (http://www.python.org/dev/peps/pep-0376/) - .. [11] importlib.metadata (https://docs.python.org/3/library/importlib.metadata.html#distribution-versions) diff --git a/pep-0397.txt b/peps/pep-0397.rst similarity index 97% rename from pep-0397.txt rename to peps/pep-0397.rst index e3d07d62e..bec6ac504 100644 --- a/pep-0397.txt +++ b/peps/pep-0397.rst @@ -1,14 +1,13 @@ PEP: 397 Title: Python launcher for Windows -Version: $Revision: a57419aee37d $ -Last-Modified: $Date: 2012/06/19 15:13:49 $ Author: Mark Hammond , - Martin v. Löwis + Martin von Löwis Status: Final Type: Standards Track Content-Type: text/x-rst Created: 15-Mar-2011 -Post-History: 21-July-2011, 17-May-2011, 15-Mar-2011 +Python-Version: 3.3 +Post-History: 21-Jul-2011, 17-May-2011, 15-Mar-2011 Resolution: https://mail.python.org/pipermail/python-dev/2012-June/120505.html Abstract @@ -51,8 +50,8 @@ particular version of Python installed under the operating-system. These symbolic links allow Python to be executed without regard for where Python it actually installed on the machine (eg., without requiring the path where Python is actually installed to be -referenced in the shebang line or in the ``PATH``.) PEP 394 'The "python" -command on Unix-Like Systems' [2]_ describes additional conventions +referenced in the shebang line or in the ``PATH``.) :pep:`394` 'The "python" +command on Unix-Like Systems' describes additional conventions for more fine-grained specification of a particular Python version. These 2 facilities combined allow for a portable and somewhat @@ -411,8 +410,6 @@ References .. [1] http://linux.die.net/man/2/execve -.. [2] http://www.python.org/dev/peps/pep-0394/ - .. [3] https://bitbucket.org/vinay.sajip/pylauncher .. [4] https://bitbucket.org/vinay.sajip/pylauncher/src/tip/Doc/launcher.rst @@ -421,13 +418,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0398.txt b/peps/pep-0398.rst similarity index 77% rename from pep-0398.txt rename to peps/pep-0398.rst index 2ab06965b..6cf800b67 100644 --- a/pep-0398.txt +++ b/peps/pep-0398.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Georg Brandl Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 23-Mar-2011 Python-Version: 3.3 @@ -129,24 +130,24 @@ Features for 3.3 Implemented / Final PEPs: -* PEP 362: Function Signature Object -* PEP 380: Syntax for Delegating to a Subgenerator -* PEP 393: Flexible String Representation -* PEP 397: Python launcher for Windows -* PEP 399: Pure Python/C Accelerator Module Compatibility Requirements -* PEP 405: Python Virtual Environments -* PEP 409: Suppressing exception context -* PEP 412: Key-Sharing Dictionary -* PEP 414: Explicit Unicode Literal for Python 3.3 -* PEP 415: Implement context suppression with exception attributes -* PEP 417: Including mock in the Standard Library -* PEP 418: Add monotonic time, performance counter, and process time functions -* PEP 420: Implicit Namespace Packages -* PEP 421: Adding sys.implementation -* PEP 3118: Revising the buffer protocol (protocol semantics finalised) -* PEP 3144: IP Address manipulation library -* PEP 3151: Reworking the OS and IO exception hierarchy -* PEP 3155: Qualified name for classes and functions +* :pep:`362`: Function Signature Object +* :pep:`380`: Syntax for Delegating to a Subgenerator +* :pep:`393`: Flexible String Representation +* :pep:`397`: Python launcher for Windows +* :pep:`399`: Pure Python/C Accelerator Module Compatibility Requirements +* :pep:`405`: Python Virtual Environments +* :pep:`409`: Suppressing exception context +* :pep:`412`: Key-Sharing Dictionary +* :pep:`414`: Explicit Unicode Literal for Python 3.3 +* :pep:`415`: Implement context suppression with exception attributes +* :pep:`417`: Including mock in the Standard Library +* :pep:`418`: Add monotonic time, performance counter, and process time functions +* :pep:`420`: Implicit Namespace Packages +* :pep:`421`: Adding sys.implementation +* :pep:`3118`: Revising the buffer protocol (protocol semantics finalised) +* :pep:`3144`: IP Address manipulation library +* :pep:`3151`: Reworking the OS and IO exception hierarchy +* :pep:`3155`: Qualified name for classes and functions Other final large-scale changes: @@ -169,9 +170,9 @@ Other planned large-scale changes: Deferred to post-3.3: -* PEP 395: Qualified Names for Modules -* PEP 3143: Standard daemon process library -* PEP 3154: Pickle protocol version 4 +* :pep:`395`: Qualified Names for Modules +* :pep:`3143`: Standard daemon process library +* :pep:`3154`: Pickle protocol version 4 * Breaking out standard library and docs in separate repos * Addition of the "packaging" module, deprecating "distutils" * Addition of the "regex" module diff --git a/pep-0399.txt b/peps/pep-0399.rst similarity index 97% rename from pep-0399.txt rename to peps/pep-0399.rst index 3fe730c98..a87ee2a10 100644 --- a/pep-0399.txt +++ b/peps/pep-0399.rst @@ -161,9 +161,9 @@ could be added. To also help with compatibility, C code should use abstract APIs on objects to prevent accidental dependence on specific types. For instance, if a function accepts a sequence then the C code should -default to using `PyObject_GetItem()` instead of something like -`PyList_GetItem()`. C code is allowed to have a fast path if the -proper `PyList_CheckExact()` is used, but otherwise APIs should work +default to using ``PyObject_GetItem()`` instead of something like +``PyList_GetItem()``. C code is allowed to have a fast path if the +proper ``PyList_CheckExact()`` is used, but otherwise APIs should work with any object that duck types to the proper interface instead of a specific type. diff --git a/pep-0400.txt b/peps/pep-0400.rst similarity index 94% rename from pep-0400.txt rename to peps/pep-0400.rst index e5a9aeffd..5067e5a4e 100644 --- a/pep-0400.txt +++ b/peps/pep-0400.rst @@ -19,10 +19,10 @@ StreamReaderWriter. Duplicate code means that bugs should be fixed twice and that we may have subtle differences between the two implementations. -The codecs module was introduced in Python 2.0 (see the `PEP 100 -`_). The io module was -introduced in Python 2.6 and 3.0 (see the `PEP 3116 -`_), and reimplemented in C in +The codecs module was introduced in Python 2.0 (see the :pep:`100`). +The io module was +introduced in Python 2.6 and 3.0 (see the :pep:`3116`), +and reimplemented in C in Python 2.7 and 3.1. PEP Deferral @@ -47,9 +47,9 @@ since the release of Python 3.0. This new interface overlaps heavily with the legacy codecs.StreamReader, codecs.StreamWriter and codecs.StreamReaderWriter interfaces that were part of the original codec interface design in -PEP 100. These interfaces are organised around the principle of an +:pep:`100`. These interfaces are organised around the principle of an encoding with an associated stream (i.e. the reverse of arrangement in -the io module), so the original PEP 100 design required that codec +the io module), so the original :pep:`100` design required that codec writers provide appropriate StreamReader and StreamWriter implementations in addition to the core codec encode() and decode() methods. This places a heavy burden on codec authors providing these @@ -84,7 +84,7 @@ StreamReader and StreamWriter issues supports UNIX newlines ('\\n'). * StreamReader and StreamWriter are stateful codecs but don't expose functions to control their state (getstate() or setstate()). Each - codec has to handle corner cases, see `Appendix A`_. + codec has to handle corner cases, see `Appendix A `_. * StreamReader and StreamWriter are very similar to IncrementalReader and IncrementalEncoder, some code is duplicated for stateful codecs (e.g. UTF-16). @@ -207,7 +207,7 @@ Python 3.3. Defining a subclass doesn't emit a DeprecationWarning. codecs.open() will be changed to reuse the builtin open() function (TextIOWrapper) to read-write text files. -.. _Appendix A: +.. _PEP 400 Appendix A: Alternative Approach ==================== @@ -312,9 +312,8 @@ writes a new BOM on the second write (`issue #12512 Links ===== -* `PEP 100: Python Unicode Integration - `_ -* `PEP 3116: New I/O `_ +* :pep:`PEP 100: Python Unicode Integration <100>` +* :pep:`PEP 3116: New I/O <3116>` * `Issue #8796: Deprecate codecs.open() `_ * `[python-dev] Deprecate codecs.open() and StreamWriter/StreamReader diff --git a/pep-0401.txt b/peps/pep-0401.rst similarity index 100% rename from pep-0401.txt rename to peps/pep-0401.rst diff --git a/pep-0402.txt b/peps/pep-0402.rst similarity index 97% rename from pep-0402.txt rename to peps/pep-0402.rst index f73f95498..318726599 100644 --- a/pep-0402.txt +++ b/peps/pep-0402.rst @@ -1,10 +1,9 @@ PEP: 402 Title: Simplified Package Layout and Partitioning -Version: $Revision$ -Last-Modified: $Date$ -Author: P.J. Eby +Author: Phillip J. Eby Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 12-Jul-2011 Python-Version: 3.3 @@ -15,7 +14,7 @@ Rejection Notice ================ On the first day of sprints at US PyCon 2012 we had a long and -fruitful discussion about PEP 382 and PEP 402. We ended up rejecting +fruitful discussion about :pep:`382` and :pep:`402`. We ended up rejecting both but a new PEP will be written to carry on in the spirit of PEP 402. Martin von Löwis wrote up a summary: [3]_. @@ -28,7 +27,7 @@ to: * Surprise users of other languages less, * Make it easier to convert a module into a package, and * Support dividing packages into separately installed components - (ala "namespace packages", as described in PEP 382) + (ala "namespace packages", as described in :pep:`382`) The proposed enhancements do not change the semantics of any currently-importable directory layouts, but make it possible for @@ -95,7 +94,7 @@ vendors who package up these module distributions must somehow handle the conflict caused by several module distributions installing that ``__init__`` module to the same location in the filesystem. -This led to the proposing of PEP 382 ("Namespace Packages") - a way +This led to the proposing of :pep:`382` ("Namespace Packages") - a way to signal to Python's import machinery that a directory was importable, using unique filenames per module distribution. @@ -454,7 +453,7 @@ self-contained subpackage. Virtual Paths ------------- -A virtual path is created by obtaining a PEP 302 "importer" object for +A virtual path is created by obtaining a :pep:`302` "importer" object for each of the path entries found in ``sys.path`` (for a top-level module) or the parent ``__path__`` (for a submodule). @@ -533,7 +532,7 @@ Specifically the proposed changes and additions to ``pkgutil`` are: The implementation of this function does a simple top-down traversal of ``sys.virtual_package_paths``, and performs any necessary ``get_subpath()`` calls to identify what path entries need to be - added to the virtual path for that package, given that `path_entry` + added to the virtual path for that package, given that ``path_entry`` has been added to ``sys.path``. (Or, in the case of sub-packages, adding a derived subpath entry, based on their parent package's virtual path.) @@ -546,7 +545,7 @@ Specifically the proposed changes and additions to ``pkgutil`` are: * A new ``iter_virtual_packages(parent='')`` function to allow top-down traversal of virtual packages from ``sys.virtual_package_paths``, by yielding the child virtual - packages of `parent`. For example, calling + packages of ``parent``. For example, calling ``iter_virtual_packages("zope")`` might yield ``zope.app`` and ``zope.products`` (if they are virtual packages listed in ``sys.virtual_package_paths``), but **not** ``zope.foo.bar``. @@ -611,7 +610,7 @@ For users, developers, and distributors of virtual packages: accidentally work. Is that good or bad? -For those implementing PEP 302 importer objects: +For those implementing :pep:`302` importer objects: * Importers that support the ``iter_modules()`` method (used by ``pkgutil`` to locate importable modules and packages) and want to @@ -663,13 +662,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0403.txt b/peps/pep-0403.rst similarity index 95% rename from pep-0403.txt rename to peps/pep-0403.rst index 5b61712e5..e91f00bb0 100644 --- a/pep-0403.txt +++ b/peps/pep-0403.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 13-Oct-2011 Python-Version: 3.4 -Post-History: 2011-10-13 +Post-History: 13-Oct-2011 Abstract @@ -27,7 +27,7 @@ statement that uses it actually makes the code harder to read. It also avoids any name shadowing concerns by making sure the new name is visible only to the statement in the ``@in`` clause. -This PEP is based heavily on many of the ideas in PEP 3150 (Statement Local +This PEP is based heavily on many of the ideas in :pep:`3150` (Statement Local Namespaces) so some elements of the rationale will be familiar to readers of that PEP. Both PEPs remain deferred for the time being, primarily due to the lack of compelling real world use cases in either PEP. @@ -110,7 +110,7 @@ those that don't make any sense in that context, such as ``pass`` - while such code would be legal, there wouldn't be any point in writing it). This permissive structure is easier to define and easier to explain, but a more restrictive approach that only permits operations that "make sense" would -also be possible (see PEP 3150 for a list of possible candidates). +also be possible (see :pep:`3150` for a list of possible candidates). The ``@in`` clause will not create a new scope - all name binding operations aside from the trailing function or class definition will affect @@ -201,7 +201,7 @@ decorators, but covering a broader set of applications. Relation to PEP 3150 -------------------- -PEP 3150 (Statement Local Namespaces) describes its primary motivation +:pep:`3150` (Statement Local Namespaces) describes its primary motivation as being to elevate ordinary assignment statements to be on par with ``class`` and ``def`` statements where the name of the item to be defined is presented to the reader in advance of the details of how the value of that item is @@ -210,12 +210,12 @@ the simple name binding of a standard function definition to be replaced with something else (like assigning the result of the function to a value). Despite having the same author, the two PEPs are in direct competition with -each other. PEP 403 represents a minimalist approach that attempts to achieve +each other. :pep:`403` represents a minimalist approach that attempts to achieve useful functionality with a minimum of change from the status quo. This PEP instead aims for a more flexible standalone statement design, which requires a larger degree of change to the language. -Note that where PEP 403 is better suited to explaining the behaviour of +Note that where :pep:`403` is better suited to explaining the behaviour of generator expressions correctly, this PEP is better able to explain the behaviour of decorator clauses in general. Both PEPs support adequate explanations for the semantics of container comprehensions. @@ -255,7 +255,7 @@ promote short, cryptic function names (including this one, which requires that the name of the trailing definition be supplied at least twice, encouraging the use of shorthand placeholder names like ``f``). -However, the introduction of qualified names in PEP 3155 means that even +However, the introduction of qualified names in :pep:`3155` means that even anonymous classes and functions will now have different representations if they occur in different scopes. For example:: @@ -273,7 +273,7 @@ still a major improvement over the historical situation where everything Possible Implementation Strategy -------------------------------- -This proposal has at least one titanic advantage over PEP 3150: +This proposal has at least one titanic advantage over :pep:`3150`: implementation should be relatively straightforward. The ``@in`` clause will be included in the AST for the associated function or @@ -457,12 +457,12 @@ Using a nested suite -------------------- The problems with using a full nested suite are best described in -PEP 3150. It's comparatively difficult to implement properly, the scoping +:pep:`3150`. It's comparatively difficult to implement properly, the scoping semantics are harder to explain and it creates quite a few situations where there are two ways to do it without clear guidelines for choosing between them (as almost any construct that can be expressed with ordinary imperative code could instead be expressed using a given statement). While the PEP does -propose some new PEP 8 guidelines to help address that last problem, the +propose some new :pep:`8` guidelines to help address that last problem, the difficulties in implementation are not so easily dealt with. By contrast, the decorator inspired syntax in this PEP explicitly limits the @@ -473,7 +473,7 @@ binding of the function is completely unnecessary) then it probably *should* be used. Another possible variant of this idea is to keep the decorator based -*semantics* of this PEP, while adopting the prettier syntax from PEP 3150:: +*semantics* of this PEP, while adopting the prettier syntax from :pep:`3150`:: x = weakref.ref(target, report_destruction) given: def report_destruction(obj): @@ -487,7 +487,7 @@ that could potentially be addressed through a suitable definition of the suite-that-is-not-a-suite in the language grammar). However, a nested suite has not yet been ruled out completely. The latest -version of PEP 3150 uses explicit forward reference and name binding +version of :pep:`3150` uses explicit forward reference and name binding schemes that greatly simplify the semantics of the statement, and it does offer the advantage of allowing the definition of arbitrary subexpressions rather than being restricted to a single function or diff --git a/pep-0404.txt b/peps/pep-0404.rst similarity index 97% rename from pep-0404.txt rename to peps/pep-0404.rst index 8e5d1ade8..75c0119d4 100644 --- a/pep-0404.txt +++ b/peps/pep-0404.rst @@ -1,10 +1,9 @@ PEP: 404 Title: Python 2.8 Un-release Schedule -Version: $Revision$ -Last-Modified: $Date$ Author: Barry Warsaw Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 09-Nov-2011 Python-Version: 2.8 @@ -182,13 +181,3 @@ This document has been placed in the public domain. .. _differences: http://docs.python.org/release/3.0.1/whatsnew/3.0.html .. _porting: http://python3porting.com/ - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0405.txt b/peps/pep-0405.rst similarity index 99% rename from pep-0405.txt rename to peps/pep-0405.rst index 44c40eee2..22e0a2624 100644 --- a/pep-0405.txt +++ b/peps/pep-0405.rst @@ -6,6 +6,7 @@ Author: Carl Meyer BDFL-Delegate: Nick Coghlan Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 13-Jun-2011 Python-Version: 3.3 diff --git a/pep-0406.txt b/peps/pep-0406.rst similarity index 94% rename from pep-0406.txt rename to peps/pep-0406.rst index cad761887..c290980b3 100644 --- a/pep-0406.txt +++ b/peps/pep-0406.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 04-Jul-2011 Python-Version: 3.4 -Post-History: 31-Jul-2011, 13-Nov-2011, 4-Dec-2011 +Post-History: 31-Jul-2011, 13-Nov-2011, 04-Dec-2011 Abstract ======== @@ -37,8 +37,8 @@ The import system has seen substantial changes since this PEP was originally written, as part of :pep:`420` in Python 3.3 and :pep:`451` in Python 3.4. While providing an encapsulation of the import state is still highly -desirable, it is better tackled in a new PEP using PEP 451 as a foundation, -and permitting only the use of PEP 451 compatible finders and loaders (as +desirable, it is better tackled in a new PEP using :pep:`451` as a foundation, +and permitting only the use of :pep:`451` compatible finders and loaders (as those avoid many of the issues of direct manipulation of global state associated with the previous loader API). @@ -66,7 +66,7 @@ over which modules can be imported). The engine would also be subclassed to make it possible to use the import engine API to interact with the existing process-global state. -The namespace PEPs (especially PEP 402) raise a potential need for +The namespace PEPs (especially :pep:`402`) raise a potential need for *additional* process global state, in order to correctly update package paths as ``sys.path`` is modified. @@ -159,7 +159,7 @@ Global variables No changes to finder/loader interfaces ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Rather than attempting to update the PEP 302 APIs to accept additional state, +Rather than attempting to update the :pep:`302` APIs to accept additional state, this PEP proposes that ``ImportEngine`` support the content management protocol (similar to the context substitution mechanisms in the ``decimal`` module). @@ -238,7 +238,7 @@ Brett Cannon's importlib has been developed by Greg Slodkowicz as part of the modifying existing code, and hence duplicates a lot of things unnecessarily. An actual implementation would just modify any such affected code in place. -That earlier draft of the PEP proposed change the PEP 302 APIs to support passing +That earlier draft of the PEP proposed change the :pep:`302` APIs to support passing in an optional engine instance. This had the (serious) downside of not correctly affecting further imports from the imported module, hence the change to the context management based proposal for substituting the global state. @@ -247,9 +247,6 @@ context management based proposal for substituting the global state. References ========== -.. [1] PEP 302, New Import Hooks, J van Rossum, Moore - (http://www.python.org/dev/peps/pep-0302) - .. [2] __import__() builtin function, The Python Standard Library documentation (http://docs.python.org/library/functions.html#__import__) diff --git a/pep-0407.txt b/peps/pep-0407.rst similarity index 99% rename from pep-0407.txt rename to peps/pep-0407.rst index d15130868..c313a2cb2 100644 --- a/pep-0407.txt +++ b/peps/pep-0407.rst @@ -144,7 +144,7 @@ These are open issues that should be worked out during discussion: * What is the policy for security fixes? * Restrict new syntax and similar changes (i.e. everything that was - prohibited by PEP 3003) to LTS versions? + prohibited by :pep:`3003`) to LTS versions? * What is the effect on packagers such as Linux distributions? diff --git a/pep-0408.txt b/peps/pep-0408.rst similarity index 97% rename from pep-0408.txt rename to peps/pep-0408.rst index 333b2545b..bc34c0139 100644 --- a/pep-0408.txt +++ b/peps/pep-0408.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 07-Jan-2012 Python-Version: 3.3 -Post-History: 2012-01-27 +Post-History: 27-Jan-2012 Resolution: https://mail.python.org/pipermail/python-dev/2012-January/115962.html @@ -81,7 +81,7 @@ with an API that has wide acceptance in the Python development community. In any case, modules that are proposed to be added to the standard library, whether via ``__preview__`` or directly, must fulfill the acceptance conditions -set by PEP 2. +set by :pep:`2`. It is important to stress that the aim of this proposal is not to make the process of adding new modules to the standard library more difficult. On the @@ -185,25 +185,25 @@ Candidates for inclusion into __preview__ For Python 3.3, there are a number of clear current candidates: * ``regex`` (http://pypi.python.org/pypi/regex) -* ``daemon`` (PEP 3143) -* ``ipaddr`` (PEP 3144) +* ``daemon`` (:pep:`3143`) +* ``ipaddr`` (:pep:`3144`) Other possible future use cases include: * Improved HTTP modules (e.g. ``requests``) * HTML 5 parsing support (e.g. ``html5lib``) * Improved URL/URI/IRI parsing -* A standard image API (PEP 368) -* Encapsulation of the import state (PEP 368) -* Standard event loop API (PEP 3153) -* A binary version of WSGI for Python 3 (e.g. PEP 444) +* A standard image API (:pep:`368`) +* Encapsulation of the import state (:pep:`368`) +* Standard event loop API (:pep:`3153`) +* A binary version of WSGI for Python 3 (e.g. :pep:`444`) * Generic function support (e.g. ``simplegeneric``) Relationship with PEP 407 ========================= -PEP 407 proposes a change to the core Python release cycle to permit interim +:pep:`407` proposes a change to the core Python release cycle to permit interim releases every 6 months (perhaps limited to standard library updates). If such a change to the release cycle is made, the following policy for the ``__preview__`` namespace is suggested: diff --git a/pep-0409.txt b/peps/pep-0409.rst similarity index 97% rename from pep-0409.txt rename to peps/pep-0409.rst index 8bf2047f5..d8c055467 100644 --- a/pep-0409.txt +++ b/peps/pep-0409.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 26-Jan-2012 +Python-Version: 3.3 Post-History: 30-Aug-2002, 01-Feb-2012, 03-Feb-2012 Superseded-By: 415 Resolution: https://mail.python.org/pipermail/python-dev/2012-February/116136.html @@ -15,7 +16,7 @@ Resolution: https://mail.python.org/pipermail/python-dev/2012-February/116136.ht Abstract ======== -One of the open issues from PEP 3134 is suppressing context: currently +One of the open issues from :pep:`3134` is suppressing context: currently there is no way to do it. This PEP proposes one. @@ -89,7 +90,7 @@ Implementation Discussion ========================= Note: after acceptance of this PEP, a cleaner implementation mechanism -was proposed and accepted in PEP 415. Refer to that PEP for more +was proposed and accepted in :pep:`415`. Refer to that PEP for more details on the implementation actually used in Python 3.3. Currently, ``None`` is the default for both ``__context__`` and ``__cause__``. diff --git a/pep-0410.txt b/peps/pep-0410.rst similarity index 100% rename from pep-0410.txt rename to peps/pep-0410.rst diff --git a/pep-0411.txt b/peps/pep-0411.rst similarity index 91% rename from pep-0411.txt rename to peps/pep-0411.rst index cf5b0276d..8bee39e6b 100644 --- a/pep-0411.txt +++ b/peps/pep-0411.rst @@ -4,12 +4,20 @@ Version: $Revision$ Last-Modified: $Date$ Author: Nick Coghlan , Eli Bendersky -Status: Active +Status: Superseded Type: Informational Content-Type: text/x-rst Created: 10-Feb-2012 Python-Version: 3.3 -Post-History: 2012-02-10, 2012-03-24 +Post-History: 10-Feb-2012, 24-Mar-2012 + +.. note:: + + This PEP has been marked as *Superseded*. A decade after this PEP + was written, experience has shown this is a rarely used feature in + managing the standard library. It has also not helped prevent + people from relying too heavily on provisional modules, such that + changes can still cause significant breakage in the community. Abstract @@ -72,7 +80,7 @@ The phrase "provisional basis" will then be a link to the glossary term This process allows the standard library to continue to evolve over time, without locking in problematic design errors for extended periods of time. - See PEP 411 for more details. + See :pep:`411` for more details. The following will be added to the start of the package's docstring: @@ -95,7 +103,7 @@ community. In any case, packages that are proposed to be added to the standard library, whether via the provisional state or directly, must fulfill the acceptance -conditions set by PEP 2. +conditions set by :pep:`2`. Criteria for "graduation" ------------------------- @@ -169,25 +177,25 @@ Candidates for provisional inclusion into the standard library For Python 3.3, there are a number of clear current candidates: * ``regex`` (http://pypi.python.org/pypi/regex) - approved by Guido [#]_. -* ``daemon`` (PEP 3143) -* ``ipaddr`` (PEP 3144) +* ``daemon`` (:pep:`3143`) +* ``ipaddr`` (:pep:`3144`) Other possible future use cases include: * Improved HTTP modules (e.g. ``requests``) * HTML 5 parsing support (e.g. ``html5lib``) * Improved URL/URI/IRI parsing -* A standard image API (PEP 368) -* Improved encapsulation of import state (PEP 406) -* Standard event loop API (PEP 3153) -* A binary version of WSGI for Python 3 (e.g. PEP 444) +* A standard image API (:pep:`368`) +* Improved encapsulation of import state (:pep:`406`) +* Standard event loop API (:pep:`3153`) +* A binary version of WSGI for Python 3 (e.g. :pep:`444`) * Generic function support (e.g. ``simplegeneric``) Rejected alternatives and variations ==================================== -See PEP 408. +See :pep:`408`. References @@ -199,13 +207,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0412.txt b/peps/pep-0412.rst similarity index 100% rename from pep-0412.txt rename to peps/pep-0412.rst diff --git a/pep-0413.txt b/peps/pep-0413.rst similarity index 94% rename from pep-0413.txt rename to peps/pep-0413.rst index b5da05071..933972b21 100644 --- a/pep-0413.txt +++ b/peps/pep-0413.rst @@ -7,7 +7,7 @@ Status: Withdrawn Type: Process Content-Type: text/x-rst Created: 24-Feb-2012 -Post-History: 2012-02-24, 2012-02-25 +Post-History: 24-Feb-2012, 25-Feb-2012 PEP Withdrawal @@ -36,13 +36,13 @@ versioning scheme) that allows accelerated releases of the Python standard library, while maintaining (or even slowing down) the current rate of change in the core language definition. -Like PEP 407, it aims to adjust the current balance between measured +Like :pep:`407`, it aims to adjust the current balance between measured change that allows the broader community time to adapt and being able to keep pace with external influences that evolve more rapidly than the current release cycle can handle (this problem is particularly notable for standard library elements that relate to web technologies). -However, it's more conservative in its aims than PEP 407, seeking to +However, it's more conservative in its aims than :pep:`407`, seeking to restrict the increased pace of development to builtin and standard library interfaces, without affecting the rate of change for other elements such as the language syntax and version numbering as well as the CPython @@ -52,7 +52,7 @@ binary API and bytecode format. Rationale ========= -To quote the PEP 407 abstract: +To quote the :pep:`407` abstract: Finding a release cycle for an open-source project is a delicate exercise in managing mutually contradicting constraints: developer manpower, @@ -67,13 +67,13 @@ To quote the PEP 407 abstract: more fluid release of features, by introducing the notion of long-term support versions. -I agree with the PEP 407 authors that the current release cycle of the +I agree with the :pep:`407` authors that the current release cycle of the *standard library* is too slow to effectively cope with the pace of change in some key programming areas (specifically, web protocols and related technologies, including databases, templating and serialisation formats). However, I have written this competing PEP because I believe that the -approach proposed in PEP 407 of offering full, potentially binary +approach proposed in :pep:`407` of offering full, potentially binary incompatible releases of CPython every 6 months places too great a burden on the wider Python ecosystem. @@ -97,7 +97,7 @@ quite some time to catch up with language level changes. At a cultural level, the Python community is also accustomed to a certain meaning for Python version numbers - they're linked to deprecation periods, -support periods, all sorts of things. PEP 407 proposes that collective +support periods, all sorts of things. :pep:`407` proposes that collective knowledge all be swept aside, without offering a compelling rationale for why such a course of action is actually *necessary* (aside from, perhaps, making the lives of the CPython core developers a little easier at the expense of @@ -116,7 +116,7 @@ Proposal ======== This PEP proposes the introduction of a new kind of CPython release: -"standard library releases". As with PEP 407, this will give CPython 3 kinds +"standard library releases". As with :pep:`407`, this will give CPython 3 kinds of release: * Language release: "x.y.0" @@ -142,7 +142,7 @@ release may contain any and all of the following changes: * changes to the AST * any other significant changes to the compilation toolchain * changes to the core interpreter eval loop -* binary incompatible changes to the C ABI (although the PEP 384 stable ABI +* binary incompatible changes to the C ABI (although the :pep:`384` stable ABI must still be preserved) * bug fixes @@ -156,7 +156,7 @@ documenting the standard library version. Standard library releases may include the following changes: * new features in pure Python modules -* new features in C extension modules (subject to PEP 399 compatibility +* new features in C extension modules (subject to :pep:`399` compatibility requirements) * new features in language builtins (provided the C ABI remains unaffected) * bug fixes from the corresponding maintenance release @@ -246,7 +246,7 @@ The versioning scheme proposed above is based on a number of user scenarios that are likely to be encountered if this scheme is adopted. In each case, the scenario is described for both the status quo (i.e. slow release cycle) the versioning scheme in this PEP and the free wheeling minor version number -scheme proposed in PEP 407. +scheme proposed in :pep:`407`. To give away the ending, the point of using a separate version number is that for almost all scenarios, the important number is the *language* version, not @@ -314,7 +314,7 @@ compatibility is important). Extension module author, deciding whether or not to make a binary release ------------------------------------------------------------------------- -**Status quo:** unless using the PEP 384 stable ABI, a new binary release is +**Status quo:** unless using the :pep:`384` stable ABI, a new binary release is needed every time the minor version number changes. **This PEP:** same as status quo. @@ -402,8 +402,8 @@ more explicit about their rate of adoption of standard library features. More conservative projects will likely pin their dependency to the language version and avoid features added in the standard library releases. Faster moving projects could instead declare their dependency on a particular -standard library version. However, since PEP 407 does have the advantage of -preserving the status quo, I'm calling this one for PEP 407 (albeit with a +standard library version. However, since :pep:`407` does have the advantage of +preserving the status quo, I'm calling this one for :pep:`407` (albeit with a slim margin). @@ -421,7 +421,7 @@ proved to be insufficient to reproduce the fault). **PEP 407:** same as the status quo -**Verdict:** another marginal win for PEP 407. The new standard library version +**Verdict:** another marginal win for :pep:`407`. The new standard library version *is* an extra piece of information that users may need to pass back to developers when reporting issues with Python libraries (or Python itself, on our own tracker). However, by including it in ``sys.version``, many @@ -444,7 +444,7 @@ both the former maintenance branch and the standard library update branch. but handling security fixes for non-LTS releases is currently an open question. -**Verdict:** until PEP 407 is updated to actually address this scenario, a +**Verdict:** until :pep:`407` is updated to actually address this scenario, a clear win for this PEP. @@ -454,7 +454,7 @@ Effects Effect on development cycle --------------------------- -Similar to PEP 407, this PEP will break up the delivery of new features into +Similar to :pep:`407`, this PEP will break up the delivery of new features into more discrete chunks. Instead of a whole raft of changes landing all at once in a language release, each language release will be limited to 6 months worth of standard library changes, as well as any changes associated with @@ -518,7 +518,7 @@ releases rather than using the new standard library releases. Effect on the community ----------------------- -PEP 407 has this to say about the effects on the community: +:pep:`407` has this to say about the effects on the community: People who value stability can just synchronize on the LTS releases which, with the proposed figures, would give a similar support cycle (both in @@ -543,7 +543,7 @@ going to ship standard library updates for the next language release in definition update, even those changes that are backwards compatible with the previously released version of Python. -The community benefits listed in PEP 407 are equally applicable to this PEP, +The community benefits listed in :pep:`407` are equally applicable to this PEP, at least as far as the standard library is concerned: People who value reactivity and access to new features (without taking the @@ -593,7 +593,7 @@ resolving this becomes even more critical. While Mercurial phases may help to some degree, it would be good to eliminate the problem entirely. One suggestion from Barry Warsaw is to adopt a non-conflicting -separate-files-per-change approach, similar to that used by Twisted [2_]. +separate-files-per-change approach, similar to that used by Twisted [2]_. Given that the current manually updated NEWS file will be used for the 3.3.0 release, one possible layout for such an approach might look like:: @@ -719,7 +719,7 @@ to make standard library releases even *faster* than every 6 months? If the practical issues were ever resolved, then the separate standard library versioning scheme in this PEP could handle it. The tagged version -number approach proposed in PEP 407 could not (at least, not without a lot +number approach proposed in :pep:`407` could not (at least, not without a lot of user confusion and uncertainty). @@ -812,7 +812,7 @@ a little tidying up of the What's New document before making the release. Why isn't PEP 384 enough? ------------------------- -PEP 384 introduced the notion of a "Stable ABI" for CPython, a limited +:pep:`384` introduced the notion of a "Stable ABI" for CPython, a limited subset of the full C ABI that is guaranteed to remain stable. Extensions built against the stable ABI should be able to support all subsequent Python versions with the same binary. @@ -853,7 +853,7 @@ library releases: * Standard library updates * new features in pure Python modules - * new features in C extension modules (subject to PEP 399 compatibility + * new features in C extension modules (subject to :pep:`399` compatibility requirements) * new features in language builtins @@ -874,7 +874,7 @@ And the following changes would be acceptable in language releases: * removal of previously deprecated features * changes to the AST * changes to the emitted bytecode that require altering the magic number -* binary incompatible changes to the C ABI (although the PEP 384 stable ABI +* binary incompatible changes to the C ABI (although the :pep:`384` stable ABI must still be preserved) While such an approach could probably be made to work, there does not appear @@ -906,30 +906,17 @@ fraction of the pain. Acknowledgements ================ -Thanks go to the PEP 407 authors for starting this discussion, as well as +Thanks go to the :pep:`407` authors for starting this discussion, as well as to those authors and Larry Hastings for initial discussions of the proposal made in this PEP. References ========== -.. [1] PEP 407: New release cycle and introducing long-term support versions - http://www.python.org/dev/peps/pep-0407/ - .. [2] Twisted's "topfiles" approach to NEWS generation - http://twistedmatrix.com/trac/wiki/ReviewProcess#Newsfiles + https://web.archive.org/web/20120305142914/http://twistedmatrix.com/trac/wiki/ReviewProcess#Newsfiles Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0414.txt b/peps/pep-0414.rst similarity index 99% rename from pep-0414.txt rename to peps/pep-0414.rst index f303b81bc..d8ef45f61 100644 --- a/pep-0414.txt +++ b/peps/pep-0414.rst @@ -8,6 +8,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 15-Feb-2012 +Python-Version: 3.3 Post-History: 28-Feb-2012, 04-Mar-2012 Resolution: https://mail.python.org/pipermail/python-dev/2012-February/116995.html @@ -115,7 +116,7 @@ Rationale ========= With the release of a Python 3 compatible version of the Web Services Gateway -Interface (WSGI) specification (PEP 3333) for Python 3.2, many parts of the +Interface (WSGI) specification (:pep:`3333`) for Python 3.2, many parts of the Python web ecosystem have been making a concerted effort to support Python 3 without adversely affecting their existing developer and user communities. diff --git a/pep-0415.txt b/peps/pep-0415.rst similarity index 90% rename from pep-0415.txt rename to peps/pep-0415.rst index 827134c01..48715a21f 100644 --- a/pep-0415.txt +++ b/peps/pep-0415.rst @@ -17,9 +17,9 @@ Resolution: https://mail.python.org/pipermail/python-dev/2012-May/119467.html Abstract ======== -PEP 409 introduced support for the ``raise exc from None`` construct to +:pep:`409` introduced support for the ``raise exc from None`` construct to allow the display of the exception context to be explicitly suppressed. -This PEP retains the language level changes already implemented in PEP 409, +This PEP retains the language level changes already implemented in :pep:`409`, but replaces the underlying implementation mechanism with a simpler approach based on a new ``__suppress_context__`` attribute on all ``BaseException`` instances. @@ -34,7 +34,7 @@ This PEP was accepted by Nick Coghlan on the 14th of May, 2012. Rationale ========= -PEP 409 changes ``__cause__`` to be ``Ellipsis`` by default. Then if +:pep:`409` changes ``__cause__`` to be ``Ellipsis`` by default. Then if ``__cause__`` is set to ``None`` by ``raise exc from None``, no context or cause will be printed should the exception be uncaught. @@ -43,9 +43,9 @@ The main problem with this scheme is it complicates the role of whether ``__context__`` should be printed or not. This use of ``__cause__`` is also not easily extended in the future. For example, we may someday want to allow the programmer to select which of ``__context__`` and ``__cause__`` will -be printed. The PEP 409 implementation is not amenable to this. +be printed. The :pep:`409` implementation is not amenable to this. -The use of ``Ellipsis`` is a hack. Before PEP 409, ``Ellipsis`` was used +The use of ``Ellipsis`` is a hack. Before :pep:`409`, ``Ellipsis`` was used exclusively in extended slicing. Extended slicing has nothing to do with exceptions, so it's not clear to someone inspecting an exception object why ``__cause__`` should be set to ``Ellipsis``. Using ``Ellipsis`` by default for diff --git a/pep-0416.txt b/peps/pep-0416.rst similarity index 98% rename from pep-0416.txt rename to peps/pep-0416.rst index 692855684..59e2e4d08 100644 --- a/pep-0416.txt +++ b/peps/pep-0416.rst @@ -131,9 +131,9 @@ just need to be practically constant.* If frozendict is used to harden Python (security purpose), it must be implemented in C. A type implemented in C is also faster. -*The PEP 351 was rejected.* +*The* :pep:`351` *was rejected.* -The PEP 351 tries to freeze an object and so may convert a mutable object to an +The :pep:`351` tries to freeze an object and so may convert a mutable object to an immutable object (using a different type). frozendict doesn't convert anything: hash(frozendict) raises a TypeError if a value is not hashable. Freezing an object is not the purpose of this PEP. @@ -233,7 +233,7 @@ Links `_ * PEP 412: Key-Sharing Dictionary (`issue #13903 `_) -* PEP 351: The freeze protocol +* :pep:`351`: The freeze protocol * `The case for immutable dictionaries; and the central misunderstanding of PEP 351 `_ * `make dictproxy object via ctypes.pythonapi and type() (Python recipe diff --git a/pep-0417.txt b/peps/pep-0417.rst similarity index 100% rename from pep-0417.txt rename to peps/pep-0417.rst diff --git a/pep-0418.txt b/peps/pep-0418.rst similarity index 99% rename from pep-0418.txt rename to peps/pep-0418.rst index bcf698ec8..b728a3fb3 100644 --- a/pep-0418.txt +++ b/peps/pep-0418.rst @@ -1,8 +1,9 @@ PEP: 418 Title: Add monotonic time, performance counter, and process time functions -Version: $Revision$ -Last-Modified: $Date$ -Author: Cameron Simpson , Jim Jewett , Stephen J. Turnbull , Victor Stinner +Author: Cameron Simpson , + Jim J. Jewett , + Stephen J. Turnbull , + Victor Stinner Status: Final Type: Standards Track Content-Type: text/x-rst @@ -1640,14 +1641,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0418/bench_time.c b/peps/pep-0418/bench_time.c similarity index 100% rename from pep-0418/bench_time.c rename to peps/pep-0418/bench_time.c diff --git a/pep-0418/clock_resolution.py b/peps/pep-0418/clock_resolution.py similarity index 100% rename from pep-0418/clock_resolution.py rename to peps/pep-0418/clock_resolution.py diff --git a/pep-0418/clockutils.py b/peps/pep-0418/clockutils.py old mode 100644 new mode 100755 similarity index 100% rename from pep-0418/clockutils.py rename to peps/pep-0418/clockutils.py diff --git a/pep-0419.txt b/peps/pep-0419.rst similarity index 97% rename from pep-0419.txt rename to peps/pep-0419.rst index 1387d8d06..40fd99caf 100644 --- a/pep-0419.txt +++ b/peps/pep-0419.rst @@ -506,27 +506,15 @@ References https://github.com/sampsyo/bluelet .. [3] Twisted: inlineCallbacks - http://twistedmatrix.com/documents/8.1.0/api/twisted.internet.defer.html + https://twisted.org/documents/8.1.0/api/twisted.internet.defer.html -.. [4] Original discussion - https://mail.python.org/pipermail/python-ideas/2012-April/014705.html - -.. [5] Issue #14730: Implementation of the PEP 419 - http://bugs.python.org/issue14730 +[4] Original discussion +\ https://mail.python.org/pipermail/python-ideas/2012-April/014705.html +[5] Implementation of PEP 419 +\ https://github.com/python/cpython/issues/58935 Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0420.txt b/peps/pep-0420.rst similarity index 96% rename from pep-0420.txt rename to peps/pep-0420.rst index c8335ca10..0f0baa733 100644 --- a/pep-0420.txt +++ b/peps/pep-0420.rst @@ -19,7 +19,7 @@ across multiple directories on disk. In current Python versions, an algorithm to compute the packages ``__path__`` must be formulated. With the enhancement proposed here, the import machinery itself will construct the list of directories that make up the package. This PEP builds upon previous work, -documented in PEP 382 and PEP 402. Those PEPs have since been rejected in +documented in :pep:`382` and :pep:`402`. Those PEPs have since been rejected in favor of this one. An implementation of this PEP is at [1]_. @@ -86,8 +86,9 @@ setuptools allows declaring namespace packages in a distribution's ``setup.py``, so that distribution developers don't need to put the magic ``__path__`` modification into ``__init__.py`` themselves. -See PEP 402's "The Problem" section [2]_ for additional motivations -for namespace packages. Note that PEP 402 has been rejected, but the +See :pep:`402`'s :pep:`"The Problem" <402#the-problem>` +section for additional motivations +for namespace packages. Note that :pep:`402` has been rejected, but the motivating use cases are still valid. @@ -213,7 +214,7 @@ path with a new path entry list object. Impact on import finders and loaders ------------------------------------ -PEP 302 defines "finders" that are called to search path elements. +:pep:`302` defines "finders" that are called to search path elements. These finders' ``find_module`` methods return either a "loader" object or ``None``. @@ -242,7 +243,7 @@ back to ``find_module``. Legacy finders which implement ``find_module`` but not ``find_loader`` will be unable to contribute portions to a namespace package. -The specification expands PEP 302 loaders to include an optional method called +The specification expands :pep:`302` loaders to include an optional method called ``module_repr()`` which if present, is used to generate module object reprs. See the section below for further details. @@ -284,9 +285,9 @@ As described above, prior to this PEP ``pkgutil.extend_path()`` was used by legacy portions to create namespace packages. Because it is likely not practical for all existing portions of a namespace package to be migrated to this PEP at once, ``extend_path()`` will be modified -to also recognize PEP 420 namespace packages. This will allow some +to also recognize :pep:`420` namespace packages. This will allow some portions of a namespace to be legacy portions while others are -migrated to PEP 420. These hybrid namespace packages will not have +migrated to :pep:`420`. These hybrid namespace packages will not have the dynamic path computation that normal namespace packages have, since ``extend_path()`` never provided this functionality in the past. @@ -428,7 +429,7 @@ Discussion ========== At PyCon 2012, we had a discussion about namespace packages at which -PEP 382 and PEP 402 were rejected, to be replaced by this PEP [3]_. +:pep:`382` and :pep:`402` were rejected, to be replaced by this PEP [3]_. There is no intention to remove support of regular packages. If a developer knows that her package will never be a portion of a @@ -466,9 +467,9 @@ is summarized here: 2. Minor backward compatibility issues are okay, as long as they are properly documented. -3. This will be addressed in PEP 395. +3. This will be addressed in :pep:`395`. -4. This will also be addressed in PEP 395. +4. This will also be addressed in :pep:`395`. The inclusion of namespace packages in the standard library was motivated by Martin v. Löwis, who wanted the ``encodings`` package to @@ -530,7 +531,7 @@ Module reprs Previously, module reprs were hard coded based on assumptions about a module's ``__file__`` attribute. If this attribute existed and was a string, it was assumed to be a file system path, and the module object's repr would include -this in its value. The only exception was that PEP 302 reserved missing +this in its value. The only exception was that :pep:`302` reserved missing ``__file__`` attributes to built-in modules, and in CPython, this assumption was baked into the module object's implementation. Because of this restriction, some modules contained contrived ``__file__`` values that did not @@ -547,7 +548,7 @@ the definitive way to determine the origin of a module is to check its For example, namespace packages as described in this PEP will have no ``__file__`` attribute because no corresponding file exists. In order to provide flexibility and descriptiveness in the reprs of such modules, a new -optional protocol is added to PEP 302 loaders. Loaders can implement a +optional protocol is added to :pep:`302` loaders. Loaders can implement a ``module_repr()`` method which takes a single argument, the module object. This method should return the string to be used verbatim as the repr of the module. The rules for producing a module repr are now standardized as: @@ -618,9 +619,6 @@ References .. [1] PEP 420 branch (http://hg.python.org/features/pep-420) -.. [2] PEP 402's description of use cases for namespace packages - (http://www.python.org/dev/peps/pep-0402/#the-problem) - .. [3] PyCon 2012 Namespace Package discussion outcome (https://mail.python.org/pipermail/import-sig/2012-March/000421.html) diff --git a/pep-0421.txt b/peps/pep-0421.rst similarity index 94% rename from pep-0421.txt rename to peps/pep-0421.rst index 28c04cea4..e1287ea30 100644 --- a/pep-0421.txt +++ b/peps/pep-0421.rst @@ -8,7 +8,8 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 26-Apr-2012 -Post-History: 26-April-2012 +Python-Version: 3.3 +Post-History: 26-Apr-2012 Resolution: https://mail.python.org/pipermail/python-dev/2012-May/119683.html @@ -98,7 +99,7 @@ define them: ``sys.hexversion``. **cache_tag** - A string used for the PEP 3147 cache tag [#cachetag]_. It would + A string used for the :pep:`3147` cache tag. It would normally be a composite of the name and version (e.g. 'cpython-33' for CPython 3.3). However, an implementation may explicitly use a different cache tag. If ``cache_tag`` is set to None, it indicates @@ -192,7 +193,7 @@ As already noted in `Version Format`_, values in ``sys.implementation`` are intended for use by the standard library. Constraining those values, essentially specifying an API for them, allows them to be used consistently, regardless of how they are -otherwise implemented. However, care should be take to not +otherwise implemented. However, care should be taken to not over-specify the constraints. @@ -240,14 +241,15 @@ in ``platform`` wrapping it). Cache Tag Generation in Frozen Importlib ---------------------------------------- -PEP 3147 defined the use of a module cache and cache tags for file +:pep:`3147` defined the use of a module cache and cache tags for file names. The importlib bootstrap code, frozen into the Python binary as of 3.3, uses the cache tags during the import process. Part of the project to bootstrap importlib has been to clean code out of `Python/import.c`_ that did not need to be there any longer. -The cache tag defined in ``Python/import.c`` was hard-coded to -``"cpython" MAJOR MINOR`` [#cachetag]_. For importlib the options are +The cache tag defined in ``Python/import.c`` was +:pep:`hard-coded <3147#proposal>` +to ``"cpython" MAJOR MINOR``. For importlib the options are either hard-coding it in the same way, or guessing the implementation in the same way as does ``platform.python_implementation()``. @@ -351,21 +353,21 @@ this PEP is a small start, will be considered separately. Past Efforts ============ -``PEP 3139`` ------------- +PEP 3139 +-------- -PEP 3139, from 2008, recommended a clean-up of the ``sys`` module in +:pep:`3139`, from 2008, recommended a clean-up of the ``sys`` module in part by extracting implementation-specific variables and functions -into a separate module. PEP 421 is less ambitious version of that -idea. While PEP 3139 was rejected, its goals are reflected in PEP 421 +into a separate module. :pep:`421` is less ambitious version of that +idea. While :pep:`3139` was rejected, its goals are reflected in :pep:`421` to a large extent, though with a much lighter approach. -``PEP 399`` ------------ +PEP 399 +------- -PEP 399 dictates policy regarding the standard library, helping to make -it friendlier to alternate implementations. PEP 421 is proposed in +:pep:`399` dictates policy regarding the standard library, helping to make +it friendlier to alternate implementations. :pep:`421` is proposed in that same spirit. @@ -373,7 +375,7 @@ The Bigger Picture ================== It's worth noting again that this PEP is a small part of a larger -on-going effort to identify the implementation-specific parts of Python +ongoing effort to identify the implementation-specific parts of Python and mitigate their impact on alternate implementations. ``sys.implementation`` is a focal point for implementation-specific @@ -381,7 +383,7 @@ data, acting as a nexus for cooperation between the language, the standard library, and the different implementations. As time goes by it is feasible that ``sys.implementation`` will assume current attributes of ``sys`` and other builtin/stdlib modules, where -appropriate. In this way, it is a PEP 3137-lite, but starting as +appropriate. In this way, it is a :pep:`3137`-lite, but starting as small as possible. However, as already noted, many other efforts predate @@ -485,9 +487,6 @@ References .. [#guess] The ``platform`` code which divines the implementation name: http://hg.python.org/cpython/file/2f563908ebc5/Lib/platform.py#l1247 -.. [#cachetag] The definition for cache tags in PEP 3147: - http://www.python.org/dev/peps/pep-3147/#id53 - .. [#tag_impl] The original implementation of the cache tag in CPython: http://hg.python.org/cpython/file/2f563908ebc5/Python/import.c#l121 diff --git a/pep-0422.txt b/peps/pep-0422.rst similarity index 97% rename from pep-0422.txt rename to peps/pep-0422.rst index c02fa8a5b..655dc78e8 100644 --- a/pep-0422.txt +++ b/peps/pep-0422.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 05-Jun-2012 Python-Version: 3.5 -Post-History: 5-Jun-2012, 10-Feb-2013 +Post-History: 05-Jun-2012, 10-Feb-2013 Abstract @@ -32,7 +32,7 @@ PEP Withdrawal ============== This proposal has been withdrawn in favour of Martin Teichmann's proposal -in PEP 487, which achieves the same goals through a simpler, easier to use +in :pep:`487`, which achieves the same goals through a simpler, easier to use ``__init_subclass__`` hook that simply isn't invoked for the base class that defines the hook. @@ -71,13 +71,13 @@ execution of the class body (for example, specifying the use of ``collections.OrderedDict`` instead of a regular ``dict``). In Python 2, there was no ``__prepare__`` method (that API was added for -Python 3 by PEP 3115). Instead, a class body could set the ``__metaclass__`` +Python 3 by :pep:`3115`). Instead, a class body could set the ``__metaclass__`` attribute, and the class creation process would extract that value from the class namespace to use as the metaclass hint. There is `published code`_ that makes use of this feature. Another new feature in Python 3 is the zero-argument form of the ``super()`` -builtin, introduced by PEP 3135. This feature uses an implicit ``__class__`` +builtin, introduced by :pep:`3135`. This feature uses an implicit ``__class__`` reference to the class being defined to replace the "by name" references required in Python 2. Just as code invoked during execution of a Python 2 metaclass could not call methods that referenced the class by name (as the @@ -103,7 +103,7 @@ added to Python 3.4 that meets the following criteria: 1. Integrates nicely with class inheritance structures (including mixins and multiple inheritance) 2. Integrates nicely with the implicit ``__class__`` reference and - zero-argument ``super()`` syntax introduced by PEP 3135 + zero-argument ``super()`` syntax introduced by :pep:`3135` 3. Can be added to an existing base class without a significant risk of introducing backwards compatibility problems 4. Restores the ability for class namespaces to have some influence on the @@ -240,12 +240,12 @@ signature of ``__autodecorate__``, the risk in this case is actually even lower than in the case of ``__init__``. -Integrates cleanly with \PEP 3135 ---------------------------------- +Integrates cleanly with PEP 3135 +-------------------------------- Unlike code that runs as part of the metaclass, code that runs as part of the new hook will be able to freely invoke class methods that rely on the -implicit ``__class__`` reference introduced by PEP 3135, including methods +implicit ``__class__`` reference introduced by :pep:`3135`, including methods that use the zero argument form of ``super()``. @@ -484,7 +484,7 @@ detected anyway in order to give a useful error message. This decision was reinforced after noticing that the user experience of defining ``__prepare__`` and forgetting the ``@classmethod`` method -decorator is singularly incomprehensible (particularly since PEP 3115 +decorator is singularly incomprehensible (particularly since :pep:`3115` documents it as an ordinary method, and the current documentation doesn't explicitly say anything one way or the other). diff --git a/pep-0423.txt b/peps/pep-0423.rst similarity index 95% rename from pep-0423.txt rename to peps/pep-0423.rst index 43dd9c6b6..0eba9486c 100644 --- a/pep-0423.txt +++ b/peps/pep-0423.rst @@ -3,9 +3,10 @@ Title: Naming conventions and recipes related to packaging Version: $Revision$ Last-Modified: $Date$ Author: Benoit Bryon -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Deferred Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 24-May-2012 Post-History: @@ -32,7 +33,7 @@ PEP Deferral ============ Further consideration of this PEP has been deferred at least until after -PEP 426 (package metadata 2.0) and related updates have been resolved. +:pep:`426` (package metadata 2.0) and related updates have been resolved. Terminology =========== @@ -43,17 +44,18 @@ Reference is `packaging terminology in Python documentation`_. Relationship with other PEPs ============================ -* `PEP 8`_ deals with code style guide, including names of Python +* :pep:`8#package-and-module-names` + deals with code style guide, including names of Python packages and modules. It covers syntax of package/modules names. -* `PEP 345`_ deals with packaging metadata, and defines name argument +* :pep:`345` deals with packaging metadata, and defines name argument of the ``packaging.core.setup()`` function. -* `PEP 420`_ deals with namespace packages. It brings support of +* :pep:`420` deals with namespace packages. It brings support of namespace packages to Python core. Before, namespaces packages were implemented by external libraries. -* `PEP 3108`_ deals with transition between Python 2.x and Python 3.x +* :pep:`3108` deals with transition between Python 2.x and Python 3.x applied to standard library: some modules to be deleted, some to be renamed. It points out that naming conventions matter and is an example of transition plan. @@ -316,9 +318,10 @@ name. Follow PEP 8 for syntax of package and module names =================================================== -`PEP 8`_ applies to names of Python packages and modules. +:pep:`PEP 8 <8#package-and-module-names>` applies to names of Python packages and modules. -If you `Use a single name`_, `PEP 8`_ also applies to project names. +If you `Use a single name`_, :pep:`PEP 8 <8#package-and-module-names>` +also applies to project names. The exceptions are namespace packages, where dots are required in project name. @@ -388,7 +391,7 @@ But it would be possible if all these distributions used Avoid deep nesting ================== -`The Zen of Python`_ says "Flat is better than nested". +:pep:`The Zen of Python <20>` says "Flat is better than nested". Two levels is almost always enough ---------------------------------- @@ -628,7 +631,7 @@ already been registered: * in the `Python Standard Library`_. -* inside projects at `PyPI`. There is currently no helper for that. +* inside projects at ``PyPI``. There is currently no helper for that. Notice that the more projects follow the `use a single name`_ rule, the easier is the verification. @@ -657,7 +660,8 @@ documentation, so that users understand what happened. * import statements in code. #. Assign ``Obsoletes-Dist`` metadata to new distribution in setup.cfg - file. See `PEP 345 about Obsolete-Dist`_ and `setup.cfg + file. See :pep:`PEP 345 about Obsolete-Dist <345#obsoletes-dist-multiple-use>` + and `setup.cfg specification`_. #. Release a new version of the renamed project, then publish it. @@ -760,7 +764,7 @@ As of Python 3.3 being developed: * packaging (aka distutils2) is on the starting blocks. When it is released, projects will be invited to migrate and use new packaging. -* `PEP 420`_ brings official support of namespace packages to Python. +* :pep:`420` brings official support of namespace packages to Python. It means that most active projects should be about to migrate in the next year(s) to support Python 3.x, new packaging or new namespace @@ -788,11 +792,6 @@ References and footnotes: .. _`packaging terminology in Python documentation`: https://packaging.python.org/glossary/ -.. _`PEP 8`: - http://www.python.org/dev/peps/pep-0008/#package-and-module-names -.. _`PEP 345`: http://www.python.org/dev/peps/pep-0345/ -.. _`PEP 420`: http://www.python.org/dev/peps/pep-0420/ -.. _`PEP 3108`: http://www.python.org/dev/peps/pep-3108/ .. _`Python community`: http://www.python.org/community/ .. _`gp.fileupload`: http://pypi.python.org/pypi/gp.fileupload/ .. _`zest.releaser`: http://pypi.python.org/pypi/zest.releaser/ @@ -815,14 +814,11 @@ References and footnotes: .. _`DateUtils`: http://pypi.python.org/pypi/DateUtils/ .. _`Framework :: Twisted`: http://pypi.python.org/pypi?:action=browse&show=all&c=525 -.. _`The Zen of Python`: http://www.python.org/dev/peps/pep-0020/ .. _`Plone community`: http://plone.org/community/develop .. _`Registering with the Package Index`: https://docs.python.org/3/distutils/packageindex.html .. _`Python Standard Library`: http://docs.python.org/library/index.html -.. _`PEP 345 about Obsolete-Dist`: - http://www.python.org/dev/peps/pep-0345/#obsoletes-dist-multiple-use .. _`setup.cfg specification`: http://docs.python.org/dev/packaging/setupcfg.html .. _`Martin Aspeli's article about names`: diff --git a/pep-0424.txt b/peps/pep-0424.rst similarity index 96% rename from pep-0424.txt rename to peps/pep-0424.rst index 2e3244613..d814323c2 100644 --- a/pep-0424.txt +++ b/peps/pep-0424.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 14-Jul-2012 Python-Version: 3.4 -Post-History: https://mail.python.org/pipermail/python-dev/2012-July/120920.html +Post-History: `15-Jul-2012 `__ Abstract ======== diff --git a/pep-0425.txt b/peps/pep-0425.rst similarity index 83% rename from pep-0425.txt rename to peps/pep-0425.rst index 3ae109e5a..5037b576a 100644 --- a/pep-0425.txt +++ b/peps/pep-0425.rst @@ -6,13 +6,17 @@ Author: Daniel Holth BDFL-Delegate: Nick Coghlan Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 27-Jul-2012 Python-Version: 3.4 -Post-History: 8-Aug-2012, 18-Oct-2012, 15-Feb-2013 +Post-History: 08-Aug-2012, 18-Oct-2012, 15-Feb-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-February/124116.html +.. canonical-pypa-spec:: :ref:`packaging:platform-compatibility-tags` + + Abstract ======== @@ -67,7 +71,7 @@ For example, the tag py27-none-any indicates compatible with Python 2.7 Use === -The `wheel` built package format includes these tags in its filenames, +The ``wheel`` built package format includes these tags in its filenames, of the form ``{distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl``. Other package formats may have their own conventions. @@ -87,31 +91,31 @@ a distribution. Major implementations have abbreviated codes, initially: * pp: PyPy * jy: Jython -Other Python implementations should use `sys.implementation.name`. +Other Python implementations should use ``sys.implementation.name``. -The version is `py_version_nodot`. CPython gets away with no dot, -but if one is needed the underscore `_` is used instead. PyPy should -probably use its own versions here `pp18`, `pp19`. +The version is ``py_version_nodot``. CPython gets away with no dot, +but if one is needed the underscore ``_`` is used instead. PyPy should +probably use its own versions here ``pp18``, ``pp19``. -The version can be just the major version `2` or `3` `py2`, `py3` for +The version can be just the major version ``2`` or ``3`` ``py2``, ``py3`` for many pure-Python distributions. -Importantly, major-version-only tags like `py2` and `py3` are not -shorthand for `py20` and `py30`. Instead, these tags mean the packager +Importantly, major-version-only tags like ``py2`` and ``py3`` are not +shorthand for ``py20`` and ``py30``. Instead, these tags mean the packager intentionally released a cross-version-compatible distribution. A single-source Python 2/3 compatible distribution can use the compound -tag `py2.py3`. See `Compressed Tag Sets`, below. +tag ``py2.py3``. See ``Compressed Tag Sets``, below. ABI Tag ------- The ABI tag indicates which Python ABI is required by any included extension modules. For implementation-specific ABIs, the implementation -is abbreviated in the same way as the Python Tag, e.g. `cp33d` would be +is abbreviated in the same way as the Python Tag, e.g. ``cp33d`` would be the CPython 3.3 ABI with debugging. -The CPython stable ABI is `abi3` as in the shared library suffix. +The CPython stable ABI is ``abi3`` as in the shared library suffix. Implementations with a very unstable ABI may use the first 6 bytes (as 8 base64-encoded characters) of the SHA-256 hash of their source code @@ -122,8 +126,8 @@ decide how to best use the ABI tag. Platform Tag ------------ -The platform tag is simply `distutils.util.get_platform()` with all -hyphens `-` and periods `.` replaced with underscore `_`. +The platform tag is simply ``distutils.util.get_platform()`` with all +hyphens ``-`` and periods ``.`` replaced with underscore ``_``. * win32 * linux_i386 @@ -135,7 +139,7 @@ Use The tags are used by installers to decide which built distribution (if any) to download from a list of potential built distributions. The installer maintains a list of (pyver, abi, arch) tuples that it -will support. If the built distribution's tag is `in` the list, then +will support. If the built distribution's tag is ``in`` the list, then it can be installed. It is recommended that installers try to choose the most feature complete @@ -143,7 +147,7 @@ built distribution available (the one most specific to the installation environment) by default before falling back to pure Python versions published for older Python releases. Installers are also recommended to provide a way to configure and re-order the list of allowed compatibility -tags; for example, a user might accept only the `*-none-any` tags to only +tags; for example, a user might accept only the ``*-none-any`` tags to only download built packages that advertise themselves as being pure Python. Another desirable installer feature might be to include "re-compile from @@ -177,8 +181,8 @@ older version of Python): Sometimes there will be more than one supported built distribution for a particular version of a package. For example, a packager could release -a package tagged `cp33-abi3-linux_x86_64` that contains an optional C -extension and the same distribution tagged `py3-none-any` that does not. +a package tagged ``cp33-abi3-linux_x86_64`` that contains an optional C +extension and the same distribution tagged ``py3-none-any`` that does not. The index of the tag in the supported tags list breaks the tie, and the package with the C extension is installed in preference to the package without because that tag appears first in the list. @@ -190,7 +194,7 @@ To allow for compact filenames of bdists that work with more than one compatibility tag triple, each tag in a filename can instead be a '.'-separated, sorted, set of tags. For example, pip, a pure-Python package that is written to run under Python 2 and 3 with the same source -code, could distribute a bdist with the tag `py2.py3-none-any`. +code, could distribute a bdist with the tag ``py2.py3-none-any``. The full list of simple tags is:: for x in pytag.split('.'): @@ -208,8 +212,8 @@ FAQ What tags are used by default? Tools should use the most-preferred architecture dependent tag - e.g. `cp33-cp33m-win32` or the most-preferred pure python tag - e.g. `py33-none-any` by default. If the packager overrides the + e.g. ``cp33-cp33m-win32`` or the most-preferred pure python tag + e.g. ``py33-none-any`` by default. If the packager overrides the default it indicates that they intended to provide cross-Python compatibility. @@ -223,7 +227,7 @@ What tag do I use if my distribution uses a feature exclusive to the newest vers older release ``beaglevote-1.1.0`` that does not use the new feature, to get a compatible build. -Why isn't there a `.` in the Python version number? +Why isn't there a ``.`` in the Python version number? CPython has lasted 20+ years without a 3-digit major release. This should continue for some time. Other implementations may use _ as a delimiter, since both - and . delimit the surrounding filename. @@ -270,14 +274,11 @@ Why is the ABI tag (the second tag) sometimes "none" in the reference implementa References ========== -.. [1] Egg Filename-Embedded Metadata - (http://peak.telecommunity.com/DevCenter/EggFormats#filename-embedded-metadata) +[1] Egg Filename-Embedded Metadata +\ (http://peak.telecommunity.com/DevCenter/EggFormats#filename-embedded-metadata) -.. [2] Creating Built Distributions - (http://docs.python.org/distutils/builtdist.html) - -.. [3] PEP 3147 -- PYC Repository Directories - (http://www.python.org/dev/peps/pep-3147/) +[2] Creating Built Distributions +\ (https://docs.python.org/3.4/distutils/builtdist.html) Acknowledgements ================ @@ -289,14 +290,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0426.txt b/peps/pep-0426.rst similarity index 95% rename from pep-0426.txt rename to peps/pep-0426.rst index f3195c499..5163310de 100644 --- a/pep-0426.txt +++ b/peps/pep-0426.rst @@ -6,15 +6,16 @@ Author: Nick Coghlan , Daniel Holth , Donald Stufft BDFL-Delegate: Donald Stufft -Discussions-To: Distutils SIG +Discussions-To: distutils-sig@python.org Status: Withdrawn Type: Informational +Topic: Packaging Content-Type: text/x-rst Requires: 440, 508, 518 Created: 30-Aug-2012 -Post-History: 14 Nov 2012, 5 Feb 2013, 7 Feb 2013, 9 Feb 2013, - 27 May 2013, 20 Jun 2013, 23 Jun 2013, 14 Jul 2013, - 21 Dec 2013 +Post-History: 14-Nov-2012, 05-Feb-2013, 07-Feb-2013, 09-Feb-2013, + 27-May-2013, 20-Jun-2013, 23-Jun-2013, 14-Jul-2013, + 21-Dec-2013 Replaces: 345 @@ -22,11 +23,11 @@ PEP Withdrawal ============== The ground-up metadata redesign proposed in this PEP has been withdrawn in -favour of the more modest proposal in PEP 566, which retains the basic +favour of the more modest proposal in :pep:`566`, which retains the basic Key:Value format of previous metadata versions, but also defines a standardised mechanism for translating that format to nested JSON-compatible data structures. -Some of the ideas in this PEP (or the related PEP 459) may still be considered +Some of the ideas in this PEP (or the related :pep:`459`) may still be considered as part of later proposals, but they will be handled in a more incremental fashion, rather than as a single large proposed change with no feasible migration plan. @@ -41,9 +42,9 @@ and their semantics and usage. This document specifies the never released version 2.0 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. -Version 1.2 is specified in PEP 345. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. +Version 1.2 is specified in :pep:`345`. Version 2.0 of the metadata format proposed migrating from directly defining a custom key-value file format to instead defining a JSON-compatible in-memory @@ -62,15 +63,15 @@ This PEP was initially deferred for an extended period, from December 2013 through to March 2017, as distutils-sig worked through a number of other changes. These changes included: -* defining a binary compatibility tagging format in PEP 425 -* defining a binary archive format (``wheel``) in PEP 427 -* explicitly defining versioning and version comparison in PEP 440 -* explicitly defining the PyPI "simple" API in PEP 503 -* explicitly defining dependency specifiers and the extras system in PEP 508 -* declaring static build system dependencies (``pyproject.toml``) in PEP 518 +* defining a binary compatibility tagging format in :pep:`425` +* defining a binary archive format (``wheel``) in :pep:`427` +* explicitly defining versioning and version comparison in :pep:`440` +* explicitly defining the PyPI "simple" API in :pep:`503` +* explicitly defining dependency specifiers and the extras system in :pep:`508` +* declaring static build system dependencies (``pyproject.toml``) in :pep:`518` * migrating PyPI hosting to Rackspace, and placing it behind the Fastly CDN -* shipping ``pip`` with CPython by default in PEP 453, and backporting that - addition to Python 2.7 in PEP 477 +* shipping ``pip`` with CPython by default in :pep:`453`, and backporting that + addition to Python 2.7 in :pep:`477` * establishing `packaging.python.org`_ as the common access point for Python packaging ecosystem documentation * migrating to using the `specifications`_ section of packaging.python.org @@ -81,7 +82,7 @@ metadata format changes were genuinely desirable, and which could be omitted from the revised specification as merely being "change for change's sake". It also allowed a number of features that aren't critical to the core activity -of publishing and distributing software to be moved out to PEP 459, a separate +of publishing and distributing software to be moved out to :pep:`459`, a separate proposal for a number of standard metadata extensions that provide additional optional information about a release. @@ -98,7 +99,7 @@ it doesn't actually help solve any particularly pressing problems: .. _specifications: https://packaging.python.org/specifications/ .. _minor spec version update: https://mail.python.org/pipermail/distutils-sig/2017-September/031465.html -Finally, the PEP was withdrawn in February 2018 in favour of PEP 566 (which +Finally, the PEP was withdrawn in February 2018 in favour of :pep:`566` (which pursues that more incremental strategy). @@ -178,7 +179,7 @@ Supporting definitions The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this -document are to be interpreted as described in RFC 2119. +document are to be interpreted as described in :rfc:`2119`. "Projects" are software components that are made available for integration. Projects include Python libraries, frameworks, scripts, plugins, @@ -442,15 +443,15 @@ in Appendix A. Metadata validation ------------------- -A `jsonschema `__ description of +A `jsonschema `__ description of the distribution metadata is `available -`__. +`__. This schema does NOT currently handle validation of some of the more complex string fields (instead treating them as opaque strings). Except where otherwise noted, all URL fields in the metadata MUST comply -with RFC 3986. +with :rfc:`3986`. .. note:: @@ -486,7 +487,7 @@ Version of the file format; ``"2.0"`` is the only legal value. Automated tools consuming metadata SHOULD warn if ``metadata_version`` is greater than the highest version they support, and MUST fail if ``metadata_version`` has a greater major version than the highest -version they support (as described in PEP 440, the major version is the +version they support (as described in :pep:`440`, the major version is the value before the first dot). For broader compatibility, build tools MAY choose to produce @@ -513,7 +514,7 @@ Examples:: Name ---- -The name of the distribution, as defined in PEP 508. +The name of the distribution, as defined in :pep:`508`. As distribution names are used as part of URLs, filenames, command line parameters and must also interoperate with other packaging systems, the @@ -562,17 +563,17 @@ Example:: Version ------- -The distribution's public or local version identifier, as defined in PEP 440. +The distribution's public or local version identifier, as defined in :pep:`440`. Version identifiers are designed for consumption by automated tools and -support a variety of flexible version specification mechanisms (see PEP 440 +support a variety of flexible version specification mechanisms (see :pep:`440` for details). -Version identifiers MUST comply with the format defined in PEP 440. +Version identifiers MUST comply with the format defined in :pep:`440`. Version identifiers MUST be unique within each project. Index servers MAY place restrictions on the use of local version identifiers -as described in PEP 440. +as described in :pep:`440`. Example:: @@ -658,7 +659,7 @@ A string containing a full URL where the source for this specific version of the distribution can be downloaded. Source URLs MUST be unique within each project. This means that the URL -can't be something like ``"https://github.com/pypa/pip/archive/master.zip"``, +can't be something like ``"https://github.com/pypa/pip/archive/main.zip"``, but instead must be ``"https://github.com/pypa/pip/archive/1.3.1.zip"``. The source URL MUST reference either a source archive or a tag or specific @@ -758,8 +759,8 @@ other extras: * ``all``: if not otherwise defined, implies all declared extras Dependency management is heavily dependent on the version identification -and specification scheme defined in PEP 440 and the dependency specification, -extra, and environment marker schemes defined in PEP 508. +and specification scheme defined in :pep:`440` and the dependency specification, +extra, and environment marker schemes defined in :pep:`508`. All of these fields are optional. Automated tools MUST operate correctly if a distribution does not provide them, by assuming that a missing field @@ -864,10 +865,10 @@ subfields: and installed together. See `Extras (optional dependencies)`_ for details * ``environment``: an environment marker defining the environment that needs these dependencies. The syntax and capabilities of environment - markers are defined in PEP 508 + markers are defined in :pep:`508` Individual entries in the ``requires`` lists are strings using the dependency -declaration format defined in PEP 508, with the exception that environment +declaration format defined in :pep:`508`, with the exception that environment markers MUST NOT be included in the individual dependency declarations, and are instead supplied in the separate ``environment`` field. @@ -960,7 +961,7 @@ However, if this key is omitted, then the implied version is ``1.0``. Automated tools consuming extension metadata SHOULD warn if ``extension_version`` is greater than the highest version they support, and MUST fail if ``extension_version`` has a greater major version than -the highest version they support (as described in PEP 440, the major +the highest version they support (as described in :pep:`440`, the major version is the value before the first dot). For broader compatibility, build tools MAY choose to produce @@ -990,7 +991,7 @@ back on attempting to install from source rather than failing entirely. Extras (optional dependencies) ============================== -As defined in PEP 508, extras are additional dependencies that enable an +As defined in :pep:`508`, extras are additional dependencies that enable an optional aspect of a project release, often corresponding to a ``try: import optional_dependency ...`` block in the code. They are also used to indicate semantic dependencies for activities other than normal runtime using (such as @@ -1125,8 +1126,8 @@ to manually duplicate any of the upstream metadata in a distribution specific format. -Appendix C: Summary of differences from \PEP 345 -================================================= +Appendix C: Summary of differences from PEP 345 +=============================================== * Metadata-Version is now 2.0, with semantics specified for handling version changes @@ -1145,12 +1146,12 @@ Appendix C: Summary of differences from \PEP 345 exist and how they are intended to be used, rather than being a simple description of the permitted contents -* Changed the version scheme to be based on PEP 440 rather than PEP 386 +* Changed the version scheme to be based on :pep:`440` rather than :pep:`386` -* Added the source label mechanism as described in PEP 440 +* Added the source label mechanism as described in :pep:`440` * Formally defined dependency declarations, extras, and environment markers - in PEP 508 + in :pep:`508` * Support for different kinds of dependencies through additional reserved extra names @@ -1171,13 +1172,13 @@ Metadata-Version semantics The semantics of major and minor version increments are now specified, and follow the same model as the format version semantics specified for -the wheel format in PEP 427: minor version increments must behave +the wheel format in :pep:`427`: minor version increments must behave reasonably when processed by a tool that only understand earlier metadata versions with the same major version, while major version increments may include changes that are not compatible with existing tools. The major version number of the specification has been incremented -accordingly, as interpreting PEP 426 metadata obviously cannot be +accordingly, as interpreting :pep:`426` metadata obviously cannot be interpreted in accordance with earlier metadata specifications. Whenever the major version number of the specification is incremented, it @@ -1228,7 +1229,7 @@ target environment. Changing the version scheme --------------------------- -See PEP 440 for a detailed rationale for the various changes made to the +See :pep:`440` for a detailed rationale for the various changes made to the versioning scheme. @@ -1283,7 +1284,7 @@ Support for metadata extensions The new extension effectively allows sections of the metadata namespace to be delegated to other projects, while preserving a -standard overal format metadata format for easy of processing by +standard overall format metadata format for easy of processing by distribution tools that do not support a particular extension. It also works well in combination with the new ``build`` extra @@ -1325,7 +1326,7 @@ Standard extensions ------------------- Some of the information provided by the legacy metadata system has been -moved out to standard extensions defined in PEP 459. +moved out to standard extensions defined in :pep:`459`. This allows publication of the core dependency metadata in a more readily consumable format to proceed even before the full details of those extensions @@ -1472,7 +1473,7 @@ compatibility by the upstream project. Compatible release comparisons in environment markers ----------------------------------------------------- -PEP 440 defines a rich syntax for version comparisons that could +:pep:`440` defines a rich syntax for version comparisons that could potentially be useful with ``python_version`` and ``python_full_version`` in environment markers. However, allowing the full syntax would mean environment markers are no longer a Python subset, while allowing @@ -1499,31 +1500,22 @@ References ========== This document specifies version 2.0 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. -Version 1.2 is specified in PEP 345. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. +Version 1.2 is specified in :pep:`345`. The initial attempt at a standardised version scheme, along with the -justifications for needing such a standard can be found in PEP 386. +justifications for needing such a standard can be found in :pep:`386`. -.. [1] reStructuredText markup: - http://docutils.sourceforge.net/ +* `reStructuredText markup + `__ -.. _Python Package Index: http://pypi.python.org/pypi/ +.. _Python Package Index: https://pypi.org/ -.. _TR39: http://www.unicode.org/reports/tr39/tr39-1.html#Confusable_Detection +.. _TR39: https://www.unicode.org/reports/tr39/tr39-1.html#Confusable_Detection Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0426/pepsort.py b/peps/pep-0426/pepsort.py similarity index 98% rename from pep-0426/pepsort.py rename to peps/pep-0426/pepsort.py index 536248e25..84e28e370 100755 --- a/pep-0426/pepsort.py +++ b/peps/pep-0426/pepsort.py @@ -17,10 +17,10 @@ from distlib.version import suggest_normalized_version, legacy_key, normalized_k logger = logging.getLogger(__name__) -PEP426_VERSION_RE = re.compile('^(\d+(\.\d+)*)((a|b|c|rc)(\d+))?' - '(\.(post)(\d+))?(\.(dev)(\d+))?$') +PEP426_VERSION_RE = re.compile(r'^(\d+(\.\d+)*)((a|b|c|rc)(\d+))?' + r'(\.(post)(\d+))?(\.(dev)(\d+))?$') -PEP426_PRERELEASE_RE = re.compile('(a|b|c|rc|dev)\d+') +PEP426_PRERELEASE_RE = re.compile(r'(a|b|c|rc|dev)\d+') def pep426_key(s): s = s.strip() diff --git a/pep-0426/pydist-schema.json b/peps/pep-0426/pydist-schema.json similarity index 100% rename from pep-0426/pydist-schema.json rename to peps/pep-0426/pydist-schema.json diff --git a/pep-0427.txt b/peps/pep-0427.rst similarity index 95% rename from pep-0427.txt rename to peps/pep-0427.rst index 2ef81b14f..d75ac752a 100644 --- a/pep-0427.txt +++ b/peps/pep-0427.rst @@ -4,20 +4,18 @@ Version: $Revision$ Last-Modified: $Date$ Author: Daniel Holth BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 20-Sep-2012 Post-History: 18-Oct-2012, 15-Feb-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-February/124103.html -Canonical specification -======================= -The canonical version of the wheel format specification is now maintained at -https://packaging.python.org/specifications/binary-distribution-format/ . -This may contain amendments relative to this PEP. +.. canonical-pypa-spec:: :ref:`packaging:binary-distribution-format` + Abstract ======== @@ -26,7 +24,7 @@ This PEP describes a built-package format for Python called "wheel". A wheel is a ZIP-format archive with a specially formatted file name and the ``.whl`` extension. It contains a single distribution nearly as it -would be installed according to PEP 376 with a particular installation +would be installed according to :pep:`376` with a particular installation scheme. Although a specialized installer is recommended, a wheel file may be installed by simply unpacking into site-packages with the standard 'unzip' tool while preserving enough information to spread its contents @@ -160,7 +158,7 @@ on any CPU architecture. The last three components of the filename before the extension are called "compatibility tags." The compatibility tags express the -package's basic interpreter requirements and are detailed in PEP 425. +package's basic interpreter requirements and are detailed in :pep:`425`. Escaping and Unicode '''''''''''''''''''' @@ -193,7 +191,7 @@ its version, e.g. ``1.0.0``, consist of: #. ``{distribution}-{version}.data/`` contains one subdirectory for each non-empty install scheme key not already covered, where the subdirectory name is an index into a dictionary of install paths - (e.g. ``data``, ``scripts``, ``include``, ``purelib``, ``platlib``). + (e.g. ``data``, ``scripts``, ``headers``, ``purelib``, ``platlib``). #. Python scripts must appear in ``scripts`` and begin with exactly ``b'#!python'`` in order to enjoy script wrapper generation and ``#!python`` rewriting at install time. They may have any or no @@ -241,12 +239,12 @@ The .dist-info directory found at the root of sdists. #. WHEEL is the wheel metadata specific to a build of the package. #. RECORD is a list of (almost) all the files in the wheel and their - secure hashes. Unlike PEP 376, every file except RECORD, which + secure hashes. Unlike :pep:`376`, every file except RECORD, which cannot contain a hash of itself, must include its hash. The hash algorithm must be sha256 or better; specifically, md5 and sha1 are not permitted, as signed wheel files rely on the strong hashes in RECORD to validate the integrity of the archive. -#. PEP 376's INSTALLER and REQUESTED are not included in the archive. +#. :pep:`376`'s INSTALLER and REQUESTED are not included in the archive. #. RECORD.jws is used for digital signatures. It is not mentioned in RECORD. #. RECORD.p7s is allowed as a courtesy to anyone who would prefer to @@ -278,7 +276,7 @@ Signed wheel files ------------------ Wheel files include an extended RECORD that enables digital -signatures. PEP 376's RECORD is altered to include a secure hash +signatures. :pep:`376`'s RECORD is altered to include a secure hash ``digestname=urlsafe_b64encode_nopad(digest)`` (urlsafe base64 encoding with no trailing = characters) as the second column instead of an md5sum. All possible entries are hashed, including any @@ -312,10 +310,10 @@ checker only needs to establish that RECORD matches the signature. See -- http://self-issued.info/docs/draft-ietf-jose-json-web-signature.html -- http://self-issued.info/docs/draft-jones-jose-jws-json-serialization.html -- http://self-issued.info/docs/draft-ietf-jose-json-web-key.html -- http://self-issued.info/docs/draft-jones-jose-json-private-key.html +- :rfc:`7515` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-jws-json-serialization.html +- :rfc:`7517` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-json-private-key.html Comparison to .egg diff --git a/pep-0428.txt b/peps/pep-0428.rst similarity index 99% rename from pep-0428.txt rename to peps/pep-0428.rst index e8a738fd5..2bfcc5db3 100644 --- a/pep-0428.txt +++ b/peps/pep-0428.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 30-Jul-2012 Python-Version: 3.4 -Post-History: https://mail.python.org/pipermail/python-ideas/2012-October/016338.html +Post-History: `05-Oct-2012 `__ Resolution: https://mail.python.org/pipermail/python-dev/2013-November/130424.html diff --git a/pep-0429.txt b/peps/pep-0429.rst similarity index 64% rename from pep-0429.txt rename to peps/pep-0429.rst index 05f4f077c..71fb51c9f 100644 --- a/pep-0429.txt +++ b/peps/pep-0429.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Larry Hastings Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 17-Oct-2012 Python-Version: 3.4 @@ -78,28 +79,28 @@ Features for 3.4 Implemented / Final PEPs: -* PEP 428, a "pathlib" module providing object-oriented filesystem paths -* PEP 435, a standardized "enum" module -* PEP 436, a build enhancement that will help generate introspection information for builtins -* PEP 442, improved semantics for object finalization -* PEP 443, adding single-dispatch generic functions to the standard library -* PEP 445, a new C API for implementing custom memory allocators -* PEP 446, changing file descriptors to not be inherited by default in subprocesses -* PEP 450, a new "statistics" module -* PEP 451, standardizing module metadata for Python's module import system -* PEP 453, a bundled installer for the *pip* package manager -* PEP 454, a new "tracemalloc" module for tracing Python memory allocations -* PEP 456, a new hash algorithm for Python strings and binary data -* PEP 3154, a new and improved protocol for pickled objects -* PEP 3156, a new "asyncio" module, a new framework for asynchronous I/O +* :pep:`428`, a "pathlib" module providing object-oriented filesystem paths +* :pep:`435`, a standardized "enum" module +* :pep:`436`, a build enhancement that will help generate introspection information for builtins +* :pep:`442`, improved semantics for object finalization +* :pep:`443`, adding single-dispatch generic functions to the standard library +* :pep:`445`, a new C API for implementing custom memory allocators +* :pep:`446`, changing file descriptors to not be inherited by default in subprocesses +* :pep:`450`, a new "statistics" module +* :pep:`451`, standardizing module metadata for Python's module import system +* :pep:`453`, a bundled installer for the *pip* package manager +* :pep:`454`, a new "tracemalloc" module for tracing Python memory allocations +* :pep:`456`, a new hash algorithm for Python strings and binary data +* :pep:`3154`, a new and improved protocol for pickled objects +* :pep:`3156`, a new "asyncio" module, a new framework for asynchronous I/O Deferred to post-3.4: -* PEP 431, improved support for time zone databases -* PEP 441, improved Python zip application support -* PEP 447, support for __locallookup__ metaclass method -* PEP 448, additional unpacking generalizations -* PEP 455, key transforming dictionary +* :pep:`431`, improved support for time zone databases +* :pep:`441`, improved Python zip application support +* :pep:`447`, support for __locallookup__ metaclass method +* :pep:`448`, additional unpacking generalizations +* :pep:`455`, key transforming dictionary Copyright diff --git a/pep-0430.txt b/peps/pep-0430.rst similarity index 98% rename from pep-0430.txt rename to peps/pep-0430.rst index 500cf1a4b..d00e498c2 100644 --- a/pep-0430.txt +++ b/peps/pep-0430.rst @@ -28,7 +28,7 @@ Background ========== With the transition of the overall Python ecosystem from Python 2 to Python 3 -still in progress, one question which arises periodically [1_, 2_] is when +still in progress, one question which arises periodically [1]_, [2]_ is when and how to handle the change from providing the Python 2 documentation as the default version displayed at the docs.python.org root URL to providing the Python 3 documentation. @@ -77,7 +77,7 @@ the path component. Proposal ======== -This PEP (based on an idea originally put forward back in May [3_]) is to +This PEP (based on an idea originally put forward back in May [3]_) is to *not migrate* the Python 2 specific deep links at all, and instead adopt a scheme where all URLs presented to users on docs.python.org are qualified appropriately with the relevant release series. diff --git a/pep-0431.txt b/peps/pep-0431.rst similarity index 98% rename from pep-0431.txt rename to peps/pep-0431.rst index 084664796..1b313bd14 100644 --- a/pep-0431.txt +++ b/peps/pep-0431.rst @@ -32,7 +32,7 @@ ambiguous datetimes and will never do so. I therefore withdraw this PEP. -**UPDATE**: The PEP 615 "Support for the IANA Time Zone Database in the +**UPDATE**: The :pep:`615` "Support for the IANA Time Zone Database in the Standard Library" added the ``zoneinfo`` module to Python 3.9 and superseded this PEP. @@ -188,13 +188,13 @@ This function takes a name string that must be a string specifying a valid zoneinfo time zone, i.e. "US/Eastern", "Europe/Warsaw" or "Etc/GMT". If not given, the local time zone will be looked up. If an invalid zone name is given, or the local time zone can not be retrieved, the function raises -`UnknownTimeZoneError`. +``UnknownTimeZoneError``. The function also takes an optional path to the location of the zoneinfo database which should be used. If not specified, the function will look for databases in the following order: -1. Check if the `tzdata-update` module is installed, and then use that +1. Check if the ``tzdata-update`` module is installed, and then use that database. 2. Use the database in ``/usr/share/zoneinfo``, if it exists. diff --git a/pep-0432.txt b/peps/pep-0432.rst similarity index 95% rename from pep-0432.txt rename to peps/pep-0432.rst index 2c7ac6e69..6feef5977 100644 --- a/pep-0432.txt +++ b/peps/pep-0432.rst @@ -11,8 +11,9 @@ Type: Standards Track Content-Type: text/x-rst Requires: 587 Created: 28-Dec-2012 -Post-History: 28-Dec-2012, 2-Jan-2013, 30-Mar-2019, 28-Jun-2020 +Post-History: 28-Dec-2012, 02-Jan-2013, 30-Mar-2019, 28-Jun-2020 +.. highlight:: c PEP Withdrawal ============== @@ -24,17 +25,17 @@ and the CPython runtime easier to embed as part of a larger application. For most of that time, the changes were maintained either in a separate feature branch, or else as underscore-prefixed private APIs in the main CPython repo. -In 2019, PEP 587 migrated a subset of those API changes to the public CPython +In 2019, :pep:`587` migrated a subset of those API changes to the public CPython API for Python 3.8+ (specifically, the PEP updated the interpreter runtime to offer an explicitly multi-stage struct-based configuration interface). In June 2020, in response to a query from the Steering Council, the PEP authors decided that it made sense to withdraw the original PEP, as enough has changed -since PEP 432 was first written that we think any further changes to the +since :pep:`432` was first written that we think any further changes to the startup sequence and embedding API would be best formulated as a new PEP (or -PEPs) that take into account not only the not-yet-implemented ideas from PEP 432 +PEPs) that take into account not only the not-yet-implemented ideas from :pep:`432` that weren't considered sufficiently well validated to make their way into -PEP 587, but also any feedback on the public PEP 587 API, and any other lessons +:pep:`587`, but also any feedback on the public :pep:`587` API, and any other lessons that have been learned while adjusting the CPython implementation to be more embedding and subinterpreter friendly. @@ -118,7 +119,7 @@ well-defined phases during the initialization sequence: * Initialized - main interpreter fully available, subinterpreter creation available -PEP 587 is a more detailed proposal that covers separating out the +:pep:`587` is a more detailed proposal that covers separating out the Pre-Initialization phase from the last two phases, but doesn't allow embedding applications to run arbitrary code while in the "Runtime Initialized" state (instead, initializing the core runtime will also always fully initialize the @@ -151,10 +152,10 @@ Background Over time, CPython's initialization sequence has become progressively more complicated, offering more options, as well as performing more complex tasks -(such as configuring the Unicode settings for OS interfaces in Python 3 [10_], +(such as configuring the Unicode settings for OS interfaces in Python 3 [10]_, bootstrapping a pure Python implementation of the import system, and implementing an isolated mode more suitable for system applications that run -with elevated privileges [6_]). +with elevated privileges [6]_). Much of this complexity is formally accessible only through the ``Py_Main`` and ``Py_Initialize`` APIs, offering embedding applications little @@ -166,9 +167,9 @@ API cannot be used safely. A number of proposals are on the table for even *more* sophisticated startup behaviour, such as better control over ``sys.path`` initialization (e.g. easily adding additional directories on the command line -in a cross-platform fashion [7_], controlling the configuration of -``sys.path[0]`` [8_]), easier configuration of utilities like coverage -tracing when launching Python subprocesses [9_]). +in a cross-platform fashion [7]_, controlling the configuration of +``sys.path[0]`` [8]_), easier configuration of utilities like coverage +tracing when launching Python subprocesses [9]_). Rather than continuing to bolt such behaviour onto an already complicated system indefinitely, this PEP proposes to start simplifying the status quo by @@ -195,7 +196,7 @@ The CPython startup sequence as of Python 3.6 was difficult to understand, and even more difficult to modify. It was not clear what state the interpreter was in while much of the initialization code executed, leading to behaviour such as lists, dictionaries and Unicode values being created prior to the call -to ``Py_Initialize`` when the ``-X`` or ``-W`` options are used [1_]. +to ``Py_Initialize`` when the ``-X`` or ``-W`` options are used [1]_. By moving to an explicitly multi-phase startup sequence, developers should only need to understand: @@ -209,7 +210,7 @@ only need to understand: fully configured (which will hopefully be a relatively small subset of the full C API) -The first two aspects of that are covered by PEP 587, while the details of the +The first two aspects of that are covered by :pep:`587`, while the details of the latter distinction are still being considered. By basing the new design on a combination of C structures and Python @@ -238,11 +239,15 @@ should minimise their impact on the startup overhead. Experience with the importlib migration suggests that the startup time is dominated by IO operations. However, to monitor the impact of any changes, a simple benchmark can be used to check how long it takes to start and then -tear down the interpreter:: +tear down the interpreter: + +.. code-block:: bash python3 -m timeit -s "from subprocess import call" "call(['./python', '-Sc', 'pass'])" -Current numbers on my system for Python 3.7 (as built by the Fedora project):: +Current numbers on my system for Python 3.7 (as built by the Fedora project): + +.. code-block:: console $ python3 -m timeit -s "from subprocess import call" "call(['python3', '-Sc', 'pass'])" 50 loops, best of 5: 6.48 msec per loop @@ -261,7 +266,7 @@ sophisticated microbenchmark will be developed to assist in investigation. Required Configuration Settings =============================== -See PEP 587 for a detailed listing of CPython interpreter configuration settings +See :pep:`587` for a detailed listing of CPython interpreter configuration settings and the various means available for setting them. @@ -269,7 +274,7 @@ Implementation Strategy ======================= An initial attempt was made at implementing an earlier version of this PEP for -Python 3.4 [2_], with one of the significant problems encountered being merge +Python 3.4 [2]_, with one of the significant problems encountered being merge conflicts after the initial structural changes were put in place to start the refactoring process. Unlike some other previous major changes, such as the switch to an AST-based compiler in Python 2.5, or the switch to the importlib @@ -283,7 +288,7 @@ of exposing the new functions and structures as public API elements in CPython 3.8. After the initial merge, Victor Stinner then proceeded to actually migrate -settings to the new structure in order to successfully implement the PEP 540 +settings to the new structure in order to successfully implement the :pep:`540` UTF-8 mode changes (which required the ability to track all settings that had previously been decoded with the locale encoding, and decode them again using UTF-8 instead). Eric Snow also migrated a number of internal subsystems over as @@ -291,7 +296,7 @@ part of making the subinterpreter feature more robust. That work showed that the detailed design originally proposed in this PEP had a range of practical issues, so Victor designed and implemented an improved -private API (inspired by an earlier iteration of this PEP), which PEP 587 +private API (inspired by an earlier iteration of this PEP), which :pep:`587` proposes to promote to a public API in Python 3.8. @@ -307,7 +312,7 @@ Design Details * https://github.com/python/cpython/blob/master/Include/cpython/pystate.h * https://github.com/python/cpython/blob/master/Include/cpython/pylifecycle.h - PEP 587 covers the aspects of the API that are considered potentially stable + :pep:`587` covers the aspects of the API that are considered potentially stable enough to make public. Where a proposed API is covered by that PEP, "(see PEP 587)" is added to the text below. @@ -319,7 +324,7 @@ simplifying a number of operations that currently need to rely on basic C functionality rather than being able to use the richer data structures provided by the CPython C API. -PEP 587 covers a subset of that task, which is splitting out the components that +:pep:`587` covers a subset of that task, which is splitting out the components that even the existing "May be called before ``Py_Initialize``" interfaces need (like memory allocators and operating system interface encoding details) into a separate pre-configuration step. @@ -343,7 +348,7 @@ The following distinct interpreter initialisation phases are proposed: which encoding to use to access operating system interfaces (or chooses to delegate those decisions to the Python runtime) * Application starts the initialization process by calling one of the - ``Py_PreInitialize`` APIs (see PEP 587) + ``Py_PreInitialize`` APIs (see :pep:`587`) * Runtime Pre-Initialization: @@ -354,7 +359,7 @@ The following distinct interpreter initialisation phases are proposed: * The embedding application determines the settings required to initialize the core CPython runtime and create the main interpreter and moves to the next phase by calling ``Py_InitializeRuntime`` - * Note: as of PEP 587, the embedding application instead calls ``Py_Main()``, + * Note: as of :pep:`587`, the embedding application instead calls ``Py_Main()``, ``Py_UnixMain``, or one of the ``Py_Initialize`` APIs, and hence jumps directly to the Initialized state. @@ -368,7 +373,7 @@ The following distinct interpreter initialisation phases are proposed: * The embedding application determines and applies the settings required to complete the initialization process by calling ``Py_InitializeMainInterpreter`` - * Note: as of PEP 587, this state is not reachable via any public API, it + * Note: as of :pep:`587`, this state is not reachable via any public API, it only exists as an implicit internal state while one of the ``Py_Initialize`` functions is running @@ -396,7 +401,7 @@ over CPython's initial state, it will be able to use the new, finer grained API, which allows the embedding application greater control over the initialization process. -PEP 587 covers an initial iteration of that API, separating out the +:pep:`587` covers an initial iteration of that API, separating out the pre-initialization phase without attempting to separate core runtime initialization from main interpreter initialization. @@ -411,7 +416,7 @@ to the embedded Python runtime. This covers telling Python which memory allocator to use, as well as which text encoding to use when processing provided settings. -PEP 587 defines the settings needed to exit this state in its ``PyPreConfig`` +:pep:`587` defines the settings needed to exit this state in its ``PyPreConfig`` struct. A new query API will allow code to determine if the interpreter hasn't even @@ -426,7 +431,7 @@ The query for a completely uninitialized environment would then be Runtime Pre-Initialization Phase -------------------------------- -.. note:: In PEP 587, the settings for this phase are not yet separated out, +.. note:: In :pep:`587`, the settings for this phase are not yet separated out, and are instead only available through the combined ``PyConfig`` struct The pre-initialization phase is where an embedding application determines @@ -462,9 +467,9 @@ The proposed APIs for this step in the startup sequence are:: If ``Py_IsInitializing()`` is false, the ``Py_InitializeRuntime`` functions will implicitly call the corresponding ``Py_PreInitialize`` function. The ``use_environment`` setting will be passed down, while other settings will be -processed according to their defaults, as described in PEP 587. +processed according to their defaults, as described in :pep:`587`. -The ``PyInitError`` return type is defined in PEP 587, and allows an embedding +The ``PyInitError`` return type is defined in :pep:`587`, and allows an embedding application to gracefully handle Python runtime initialization failures, rather than having the entire process abruptly terminated by ``Py_FatalError``. @@ -653,7 +658,7 @@ application wants to adjust a setting rather than replace it completely, such as removing ``sys.path[0]``). The ``c_config`` argument is an optional pointer to a ``PyConfig`` structure, -as defined in PEP 587. If provided, it is used in preference to reading settings +as defined in :pep:`587`. If provided, it is used in preference to reading settings directly from the environment or process global state. Merely reading the configuration has no effect on the interpreter state: it @@ -675,10 +680,10 @@ or not the interpreter is the main interpreter will be configured on a per interpreter basis. Other fields will be reviewed for whether or not they can feasibly be made interpreter specific over the course of the implementation. -.. note:: The list of config fields below is currently out of sync with PEP 587. - Where they differ, PEP 587 takes precedence. +.. note:: The list of config fields below is currently out of sync with :pep:`587`. + Where they differ, :pep:`587` takes precedence. -The ``PyConfigAsObjects`` struct mirrors the ``PyConfig`` struct from PEP 587, +The ``PyConfigAsObjects`` struct mirrors the ``PyConfig`` struct from :pep:`587`, but uses full Python objects to store values, rather than C level data types. It adds ``raw_argv`` and ``argv`` list fields, so later initialisation steps don't need to accept those separately. @@ -831,7 +836,7 @@ state if ``import site`` is later explicitly executed in the process. Preparing the main module ------------------------- -.. note:: In PEP 587, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not +.. note:: In :pep:`587`, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not exposed separately, and are instead accessed through a ``Py_RunMain`` API that both prepares and executes main, and then finalizes the Python interpreter. @@ -902,7 +907,7 @@ configuration system) Executing the main module ------------------------- -.. note:: In PEP 587, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not +.. note:: In :pep:`587`, ``PyRun_PrepareMain`` and ``PyRun_ExecMain`` are not exposed separately, and are instead accessed through a ``Py_RunMain`` API that both prepares and executes main, and then finalizes the Python interpreter. @@ -971,7 +976,7 @@ settings are part of the CPython implementation, rather than part of the Python language definition. If new settings are needed to support cross-implementation compatibility in the standard library, then those should be agreed with the other implementations and exposed as new required -attributes on ``sys.implementation``, as described in PEP 421. +attributes on ``sys.implementation``, as described in :pep:`421`. These are *snapshots* of the initial configuration settings. They are not modified by the interpreter during runtime (except as noted above). @@ -1069,7 +1074,7 @@ aspects are the fact that user site directories are enabled, environment variables are trusted and that the directory containing the executed file is placed at the beginning of the import path. -Issue 16499 [6_] added a ``-I`` option to change the behaviour of +Issue 16499 [6]_ added a ``-I`` option to change the behaviour of the normal CPython executable, but this is a hard to discover solution (and adds yet another option to an already complex CLI). This PEP proposes to instead add a separate ``system-python`` executable @@ -1112,7 +1117,7 @@ The reference implementation is being developed as a private API refactoring within the CPython reference interpreter (as attempting to maintain it as an independent project proved impractical). -PEP 587 extracts a subset of the proposal that is considered sufficiently stable +:pep:`587` extracts a subset of the proposal that is considered sufficiently stable to be worth proposing as a public API for Python 3.8. @@ -1123,7 +1128,7 @@ The current mechanisms for configuring the interpreter have accumulated in a fairly ad hoc fashion over the past 20+ years, leading to a rather inconsistent interface with varying levels of documentation. -Also see PEP 587 for further discussion of the existing settings and their +Also see :pep:`587` for further discussion of the existing settings and their handling. (Note: some of the info below could probably be cleaned up and added to the @@ -1173,9 +1178,9 @@ Locating Python and the standard library The location of the Python binary and the standard library is influenced by several elements. The algorithm used to perform the calculation is -not documented anywhere other than in the source code [3_,4_]. Even that +not documented anywhere other than in the source code [3]_, [4]_. Even that description is incomplete, as it failed to be updated for the virtual -environment support added in Python 3.3 (detailed in PEP 405). +environment support added in Python 3.3 (detailed in :pep:`405`). These calculations are affected by the following function calls (made prior to calling ``Py_Initialize()``) and environment variables: @@ -1184,7 +1189,7 @@ prior to calling ``Py_Initialize()``) and environment variables: * ``Py_SetPythonHome()`` * ``PYTHONHOME`` -The filesystem is also inspected for ``pyvenv.cfg`` files (see PEP 405) or, +The filesystem is also inspected for ``pyvenv.cfg`` files (see :pep:`405`) or, failing that, a ``lib/os.py`` (Windows) or ``lib/python$VERSION/os.py`` file. @@ -1213,7 +1218,7 @@ the ``PYTHONPATH`` environment variable. The ``site`` module, which is implicitly imported at startup (unless disabled via the ``-S`` option) adds additional paths to this initial -set of paths, as described in its documentation [5_]. +set of paths, as described in its documentation [5]_. The ``-s`` command line option can be used to exclude the user site directory from the list of directories added. Embedding applications @@ -1372,7 +1377,7 @@ connection by the operating system. TBD: Document how the "-x" option is handled (skips processing of the first comment line in the main script) -Also see detailed sequence of operations notes at [1_] +Also see detailed sequence of operations notes at [1]_. References diff --git a/pep-0433.txt b/peps/pep-0433.rst similarity index 100% rename from pep-0433.txt rename to peps/pep-0433.rst diff --git a/pep-0433/bench_cloexec.py b/peps/pep-0433/bench_cloexec.py similarity index 100% rename from pep-0433/bench_cloexec.py rename to peps/pep-0433/bench_cloexec.py diff --git a/pep-0433/openbsd_bug.py b/peps/pep-0433/openbsd_bug.py similarity index 100% rename from pep-0433/openbsd_bug.py rename to peps/pep-0433/openbsd_bug.py diff --git a/pep-0434.txt b/peps/pep-0434.rst similarity index 99% rename from pep-0434.txt rename to peps/pep-0434.rst index 02f95de8d..b754d2a48 100644 --- a/pep-0434.txt +++ b/peps/pep-0434.rst @@ -9,9 +9,9 @@ Status: Active Type: Informational Content-Type: text/x-rst Created: 16-Feb-2013 -Post-History: 16-Feb-2013 - 03-Mar-2013 - 21-Mar-2013 +Post-History: 16-Feb-2013, + 03-Mar-2013, + 21-Mar-2013, 30-Mar-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-March/125003.html diff --git a/pep-0435.txt b/peps/pep-0435.rst similarity index 98% rename from pep-0435.txt rename to peps/pep-0435.rst index 79e330872..df1d2a299 100644 --- a/pep-0435.txt +++ b/peps/pep-0435.rst @@ -10,7 +10,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 23-Feb-2013 Python-Version: 3.4 -Post-History: 2013-02-23, 2013-05-02 +Post-History: 23-Feb-2013, 02-May-2013 Replaces: 354 Resolution: https://mail.python.org/pipermail/python-dev/2013-May/126112.html @@ -28,7 +28,7 @@ enumeration itself can be iterated over. Status of discussions ===================== -The idea of adding an enum type to Python is not new - PEP 354 [2]_ is a +The idea of adding an enum type to Python is not new - :pep:`354` is a previous attempt that was rejected in 2005. Recently a new set of discussions was initiated [3]_ on the ``python-ideas`` mailing list. Many new ideas were proposed in several threads; after a lengthy discussion Guido proposed adding @@ -61,7 +61,7 @@ The PEP was accepted by Guido on May 10th, 2013 [1]_. Motivation ========== -*[Based partly on the Motivation stated in PEP 354]* +*[Based partly on the Motivation stated in* :pep:`354`\ *]* The properties of an enumeration are useful for defining an immutable, related set of constant values that may or may not have a semantic meaning. Classic @@ -581,14 +581,13 @@ Acknowledgments This PEP was initially proposing including the ``flufl.enum`` package [8]_ by Barry Warsaw into the stdlib, and is inspired in large parts by it. -Ben Finney is the author of the earlier enumeration PEP 354. +Ben Finney is the author of the earlier enumeration :pep:`354`. References ========== .. [1] https://mail.python.org/pipermail/python-dev/2013-May/126112.html -.. [2] http://www.python.org/dev/peps/pep-0354/ .. [3] https://mail.python.org/pipermail/python-ideas/2013-January/019003.html .. [4] https://mail.python.org/pipermail/python-ideas/2013-February/019373.html .. [5] To make enums behave similarly to Python classes like bool, and diff --git a/pep-0436.txt b/peps/pep-0436.rst similarity index 99% rename from pep-0436.txt rename to peps/pep-0436.rst index 9b834cdba..c1a7f277e 100644 --- a/pep-0436.txt +++ b/peps/pep-0436.rst @@ -3,11 +3,12 @@ Title: The Argument Clinic DSL Version: $Revision$ Last-Modified: $Date$ Author: Larry Hastings -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst Created: 22-Feb-2013 +Python-Version: 3.4 Abstract diff --git a/pep-0437.txt b/peps/pep-0437.rst similarity index 95% rename from pep-0437.txt rename to peps/pep-0437.rst index a6bcb5c3d..92f0d729a 100644 --- a/pep-0437.txt +++ b/peps/pep-0437.rst @@ -22,7 +22,7 @@ definitions in *.pyx* files to generate the required information. However, CPython's C-API functions often require additional initialization and cleanup snippets that would be hard to specify in a *cdef*. -PEP 436 proposes a domain specific language (DSL) enclosed in C comments +:pep:`436` proposes a domain specific language (DSL) enclosed in C comments that largely resembles a per-parameter configuration file. A preprocessor reads the comment and emits an argument parsing function, docstrings and a header for the function that utilizes the results of the parsing step. @@ -42,9 +42,9 @@ designing the `second iteration of the PEP 436 DSL`_. Rationale ========= -Opinions differ regarding the suitability of the PEP 436 DSL in the context +Opinions differ regarding the suitability of the :pep:`436` DSL in the context of a C file. This PEP proposes an alternative DSL. The specific issues with -PEP 436 that spurred the counter proposal will be explained in the final +:pep:`436` that spurred the counter proposal will be explained in the final section of this PEP. @@ -348,7 +348,7 @@ Comparison with PEP 436 ======================= The author of this PEP has the following concerns about the DSL proposed -in PEP 436: +in :pep:`436`: * The whitespace sensitive configuration file like syntax looks out of place in a C file. @@ -360,20 +360,20 @@ in PEP 436: By contrast, in the alternative DSL the structure of the function definition can be understood at a single glance. -* The PEP 436 DSL has 14 documented flags and at least one undocumented +* The :pep:`436` DSL has 14 documented flags and at least one undocumented (allow_fd) flag. Figuring out which of the 2**15 possible combinations are valid places an unnecessary burden on the user. - Experience with the PEP-3118 buffer flags has shown that sorting out + Experience with the :pep:`3118` buffer flags has shown that sorting out (and exhaustively testing!) valid combinations is an extremely tedious - task. The PEP-3118 flags are still not well understood by many people. + task. The :pep:`3118` flags are still not well understood by many people. By contrast, the alternative DSL has a central file Include/converters.h that can be quickly searched for the desired converter. Many of the converters are already known, perhaps even memorized by people (due to frequent use). -* The PEP 436 DSL allows too much freedom. Types can apparently be omitted, +* The :pep:`436` DSL allows too much freedom. Types can apparently be omitted, the preprocessor accepts (and ignores) unknown keywords, sometimes adding white space after a docstring results in an assertion error. diff --git a/pep-0438.txt b/peps/pep-0438.rst similarity index 99% rename from pep-0438.txt rename to peps/pep-0438.rst index 1f1b1588a..30ee2d530 100644 --- a/pep-0438.txt +++ b/peps/pep-0438.rst @@ -7,6 +7,7 @@ BDFL-Delegate: Richard Jones Discussions-To: distutils-sig@python.org Status: Superseded Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 15-Mar-2013 Post-History: 19-May-2013 diff --git a/pep-0439.txt b/peps/pep-0439.rst similarity index 96% rename from pep-0439.txt rename to peps/pep-0439.rst index beb804e7a..b07a72043 100644 --- a/pep-0439.txt +++ b/peps/pep-0439.rst @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Richard Jones BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 18-Mar-2013 Python-Version: 3.4 @@ -32,7 +33,7 @@ PEP Rejection This PEP has been rejected in favour of a more explicit mechanism that should achieve the same end result in a more reliable fashion. The more -explicit bootstrapping mechanism is described in PEP 453. +explicit bootstrapping mechanism is described in :pep:`453`. Rationale ========= @@ -82,7 +83,7 @@ require the user to install pip. The pip bootstrap ----------------- -The Python installation includes an executable called "pip3" (see PEP 394 for +The Python installation includes an executable called "pip3" (see :pep:`394` for naming rationale etc.) that attempts to import pip machinery. If it can then the pip command proceeds as normal. If it cannot it will bootstrap pip by downloading the pip implementation and setuptools wheel files. Hereafter the @@ -97,10 +98,10 @@ upgrade timeframe and processes. To avoid issues with sudo we will have the bootstrap default to installing the pip implementation to the per-user site-packages -directory defined in PEP 370 and implemented in Python 2.6/3.0. Since +directory defined in :pep:`370` and implemented in Python 2.6/3.0. Since we avoid installing to the system Python we also avoid conflicting with any other packaging system (on Linux systems, for example.) If -the user is inside a virtual environment [1]_ then the pip +the user is inside a :pep:`405` virtual environment then the pip implementation will be installed into that virtual environment. The bootstrap process will proceed as follows: @@ -118,7 +119,7 @@ The bootstrap process will proceed as follows: package. This choice will also be present as a command-line option to pip so non-interactive use is possible. 5. The bootstrap will and contact PyPI to obtain the latest download wheel - file (see PEP 427.) + file (see :pep:`427`.) 6. Upon downloading the file it is installed using "python setup.py install". 7. The pip tool may now import the pip implementation and continues to process the requested user command normally. @@ -188,8 +189,8 @@ Implementation ============== The changes to pip required by this PEP are being tracked in that project's -issue tracker [2]_. Most notably, the addition of --bootstrap and --bootstrap- -to-system to the pip command-line. +issue tracker [2]_. Most notably, the addition of --bootstrap and +--bootstrap-to-system to the pip command-line. It would be preferable that the pip and setuptools projects distribute a wheel format download. @@ -223,9 +224,6 @@ now much reduced. References ========== -.. [1] PEP 405, Python Virtual Environments - http://www.python.org/dev/peps/pep-0405/ - .. [2] pip issue tracking work needed for this PEP https://github.com/pypa/pip/issues/863 diff --git a/pep-0440.txt b/peps/pep-0440.rst similarity index 95% rename from pep-0440.txt rename to peps/pep-0440.rst index 2fa3dacf1..86a82ea1e 100644 --- a/pep-0440.txt +++ b/peps/pep-0440.rst @@ -1,18 +1,17 @@ PEP: 440 Title: Version Identification and Dependency Specification -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Coghlan , Donald Stufft BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG -Status: Active -Type: Informational +Discussions-To: distutils-sig@python.org +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 18-Mar-2013 -Post-History: 30 Mar 2013, 27 May 2013, 20 Jun 2013, - 21 Dec 2013, 28 Jan 2014, 08 Aug 2014 - 22 Aug 2014 +Post-History: 30-Mar-2013, 27-May-2013, 20-Jun-2013, + 21-Dec-2013, 28-Jan-2014, 08-Aug-2014, + 22-Aug-2014 Replaces: 386 Resolution: https://mail.python.org/pipermail/distutils-sig/2014-August/024673.html @@ -24,7 +23,7 @@ This PEP describes a scheme for identifying versions of Python software distributions, and declaring dependencies on particular versions. This document addresses several limitations of the previous attempt at a -standardized approach to versioning, as described in PEP 345 and PEP 386. +standardized approach to versioning, as described in :pep:`345` and :pep:`386`. Definitions @@ -32,13 +31,13 @@ Definitions The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this -document are to be interpreted as described in RFC 2119. +document are to be interpreted as described in :rfc:`2119`. "Projects" are software components that are made available for integration. Projects include Python libraries, frameworks, scripts, plugins, applications, collections of data or other resources, and various combinations thereof. Public Python projects are typically registered on -the `Python Package Index `__. +the `Python Package Index `__. "Releases" are uniquely identified snapshots of a project. @@ -94,10 +93,10 @@ this scheme but MUST also include the normalizations specified below. Installation tools MAY warn the user when non-compliant or ambiguous versions are detected. -See also `Appendix B : Parsing version strings with regular expressions` which -provides a regular expression to check strict conformance with the canonical -format, as well as a more permissive regular expression accepting inputs that -may require subsequent normalization. +See also ``Appendix B : Parsing version strings with regular expressions`` +which provides a regular expression to check strict conformance with the +canonical format, as well as a more permissive regular expression accepting +inputs that may require subsequent normalization. Public version identifiers are separated into up to five segments: @@ -651,10 +650,10 @@ are permitted and MUST be ordered as shown:: .devN, aN, bN, rcN, , .postN -Note that `c` is considered to be semantically equivalent to `rc` and must be -sorted as if it were `rc`. Tools MAY reject the case of having the same ``N`` -for both a ``c`` and a ``rc`` in the same release segment as ambiguous and -remain in compliance with the PEP. +Note that ``c`` is considered to be semantically equivalent to ``rc`` and must +be sorted as if it were ``rc``. Tools MAY reject the case of having the same +``N`` for both a ``c`` and a ``rc`` in the same release segment as ambiguous +and remain in compliance with the PEP. Within an alpha (``1.0a1``), beta (``1.0b1``), or release candidate (``1.0rc1``, ``1.0c1``), the following suffixes are permitted and MUST be @@ -676,6 +675,7 @@ shared prefix, ordering MUST be by the value of the numeric component. The following example covers many of the possible combinations:: + 1.dev0 1.0.dev456 1.0a1 1.0a2.dev456 @@ -693,15 +693,16 @@ The following example covers many of the possible combinations:: 1.0+5 1.0.post456.dev34 1.0.post456 + 1.0.15 1.1.dev1 Version ordering across different metadata versions --------------------------------------------------- -Metadata v1.0 (PEP 241) and metadata v1.1 (PEP 314) do not specify a standard -version identification or ordering scheme. However metadata v1.2 (PEP 345) -does specify a scheme which is defined in PEP 386. +Metadata v1.0 (:pep:`241`) and metadata v1.1 (:pep:`314`) do not specify a standard +version identification or ordering scheme. However metadata v1.2 (:pep:`345`) +does specify a scheme which is defined in :pep:`386`. Due to the nature of the simple installer API it is not possible for an installer to be aware of which metadata version a particular distribution was @@ -712,7 +713,7 @@ necessitate a standardization across one parsing mechanism to be used for all versions of a project. Due to the above, this PEP MUST be used for all versions of metadata and -supersedes PEP 386 even for metadata v1.2. Tools SHOULD ignore any versions +supersedes :pep:`386` even for metadata v1.2. Tools SHOULD ignore any versions which cannot be parsed by the rules in this PEP, but MAY fall back to implementation defined version parsing and ordering schemes if no versions complying with this PEP are available. @@ -759,7 +760,7 @@ specify the appropriate version order. Specific build information may also be included in local version labels. -.. _Semantic versioning: http://semver.org/ +.. _Semantic versioning: https://semver.org/ DVCS based version labels @@ -1202,6 +1203,10 @@ defined in a new PEP. Summary of differences from pkg_resources.parse_version ======================================================= +* Note: this comparison is to ``pkg_resourses.parse_version`` as it existed at + the time the PEP was written. After the PEP was accepted, setuptools 6.0 and + later versions adopted the behaviour described in this PEP. + * Local versions sort differently, this PEP requires that they sort as greater than the same version without a local version, whereas ``pkg_resources.parse_version`` considers it a pre-release marker. @@ -1215,8 +1220,8 @@ Summary of differences from pkg_resources.parse_version single use of each type and they must exist in a certain order. -Summary of differences from \PEP 386 -==================================== +Summary of differences from PEP 386 +=================================== * Moved the description of version specifiers into the versioning PEP @@ -1261,7 +1266,7 @@ Changing the version scheme --------------------------- One key change in the version scheme in this PEP relative to that in -PEP 386 is to sort top level developmental releases like ``X.Y.devN`` ahead +:pep:`386` is to sort top level developmental releases like ``X.Y.devN`` ahead of alpha releases like ``X.Ya1``. This is a far more logical sort order, as projects already using both development releases and alphas/betas/release candidates do not want their developmental releases sorted in @@ -1291,7 +1296,7 @@ trailing ``\n`` character were found on PyPI. Various other normalisation rules were also added as described in the separate section on version normalisation below. -`Appendix A` shows detailed results of an analysis of PyPI distribution +``Appendix A`` shows detailed results of an analysis of PyPI distribution version information, as collected on 8th August, 2014. This analysis compares the behavior of the explicitly ordered version scheme defined in this PEP with the de facto standard defined by the behavior of setuptools. @@ -1304,7 +1309,7 @@ appropriate order as setuptools does). A more opinionated description of the versioning scheme ------------------------------------------------------- -As in PEP 386, the primary focus is on codifying existing practices to make +As in :pep:`386`, the primary focus is on codifying existing practices to make them more amenable to automation, rather than demanding that existing projects make non-trivial changes to their workflow. However, the standard scheme allows significantly more flexibility than is needed @@ -1503,7 +1508,7 @@ effects of each transformation as simple search and replace style transforms increase the likelihood of ambiguous or "junk" versions. For an extended discussion on the various types of normalizations that were -considered, please see the proof of concept for PEP 440 within pip [5]_. +considered, please see the proof of concept for :pep:`440` within pip [5]_. Allowing Underscore in Normalization @@ -1515,8 +1520,8 @@ reason for this is that the Wheel normalization scheme specifies that ``-`` gets normalized to a ``_`` to enable easier parsing of the filename. -Summary of changes to \PEP 440 -============================== +Summary of changes to PEP 440 +============================= The following changes were made to this PEP based on feedback received after the initial reference implementation was released in setuptools 8.0 and pip @@ -1547,37 +1552,37 @@ References ========== The initial attempt at a standardised version scheme, along with the -justifications for needing such a standard can be found in PEP 386. +justifications for needing such a standard can be found in :pep:`386`. -.. [1] Reference Implementation of PEP 440 Versions and Specifiers - https://github.com/pypa/packaging/pull/1 +.. [2] `Version compatibility analysis script + `__ -.. [2] Version compatibility analysis script: - https://github.com/pypa/packaging/blob/master/tasks/check.py +.. [4] `File URIs in Windows + `__ -.. [3] Pessimistic version constraint - http://docs.rubygems.org/read/chapter/16 +.. [5] `Proof of Concept: PEP 440 within pip + `__ -.. [4] File URIs in Windows - http://blogs.msdn.com/b/ie/archive/2006/12/06/file-uris-in-windows.aspx +.. [6] `PEP440: foo-X.Y.Z does not satisfy "foo>X.Y" + `__ -.. [5] Proof of Concept: PEP 440 within pip - https://github.com/pypa/pip/pull/1894 +.. [7] `PEP440: >1.7 vs >=1.7 + `__ -.. [6] PEP440: foo-X.Y.Z does not satisfy "foo>X.Y" - https://mail.python.org/pipermail/distutils-sig/2014-December/025451.html +.. [8] `Amend PEP 440 with Wider Feedback on Release Candidates + `__ -.. [7] PEP440: >1.7 vs >=1.7 - https://mail.python.org/pipermail/distutils-sig/2014-December/025507.html +.. [10] `PEP 440: regex should not permit Unicode [Nd] characters + `__ -.. [8] Amend PEP 440 with Wider Feedback on Release Candidates - https://mail.python.org/pipermail/distutils-sig/2014-December/025409.html +* `Reference Implementation of PEP 440 Versions and Specifiers + `__ -.. [9] Changing the status of PEP 440 to Provisional - https://mail.python.org/pipermail/distutils-sig/2014-December/025412.html +* `Pessimistic version constraint + `__ -.. [10] PEP 440: regex should not permit Unicode [Nd] characters - https://github.com/python/peps/pull/966 +* `Changing the status of PEP 440 to Provisional + `__ Appendix A ========== @@ -1594,7 +1599,7 @@ Metadata v2.0 guidelines versus setuptools:: Appendix B : Parsing version strings with regular expressions ============================================================= -As noted earlier in the `Public version identifiers` section, published +As noted earlier in the ``Public version identifiers`` section, published version identifiers SHOULD use the canonical format. This section provides regular expressions that can be used to test whether a version is already in that form, and if it's not, extract the various components for subsequent @@ -1652,12 +1657,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-0441.txt b/peps/pep-0441.rst similarity index 98% rename from pep-0441.txt rename to peps/pep-0441.rst index 41bbad15d..a7e6c249f 100644 --- a/pep-0441.txt +++ b/peps/pep-0441.rst @@ -1,7 +1,5 @@ PEP: 441 Title: Improving Python ZIP Application Support -Version: $Revision$ -Last-Modified: $Date$ Author: Daniel Holth , Paul Moore Discussions-To: https://mail.python.org/pipermail/python-dev/2015-February/138277.html @@ -9,7 +7,8 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 30-Mar-2013 -Post-History: 30 March 2013, 1 April 2013, 16 February 2015 +Python-Version: 3.5 +Post-History: 30-Mar-2013, 01-Apr-2013, 16-Feb-2015 Resolution: https://mail.python.org/pipermail/python-dev/2015-February/138578.html Improving Python ZIP Application Support @@ -348,14 +347,3 @@ Copyright ========= This document has been placed into the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0442.txt b/peps/pep-0442.rst similarity index 99% rename from pep-0442.txt rename to peps/pep-0442.rst index 10792a74e..3b1e2aa28 100644 --- a/pep-0442.txt +++ b/peps/pep-0442.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 18-May-2013 Python-Version: 3.4 -Post-History: 2013-05-18 +Post-History: 18-May-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-June/126746.html diff --git a/pep-0443.txt b/peps/pep-0443.rst similarity index 97% rename from pep-0443.txt rename to peps/pep-0443.rst index 3fcf99302..fa800e692 100644 --- a/pep-0443.txt +++ b/peps/pep-0443.rst @@ -3,11 +3,12 @@ Title: Single-dispatch generic functions Version: $Revision$ Last-Modified: $Date$ Author: Ɓukasz Langa -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst Created: 22-May-2013 +Python-Version: 3.4 Post-History: 22-May-2013, 25-May-2013, 31-May-2013 Replaces: 245, 246, 3124 @@ -326,7 +327,7 @@ a common one, opening them up for user extensibility at the same time. Alternative approaches ====================== -In PEP 3124 [#pep-3124]_ Phillip J. Eby proposes a full-grown solution +In :pep:`3124` Phillip J. Eby proposes a full-grown solution with overloading based on arbitrary rule sets (with the default implementation dispatching on argument types), as well as interfaces, adaptation and method combining. PEAK-Rules [#peak-rules]_ is @@ -364,7 +365,7 @@ single-dispatch generics. Acknowledgements ================ -Apart from Phillip J. Eby's work on PEP 3124 [#pep-3124]_ and +Apart from Phillip J. Eby's work on :pep:`3124` and PEAK-Rules, influences include Paul Moore's original issue [#issue-5135]_ that proposed exposing ``pkgutil.simplegeneric`` as part of the ``functools`` API, Guido van Rossum's article on multimethods @@ -379,16 +380,13 @@ References .. [#ref-impl] http://hg.python.org/features/pep-443/file/tip/Lib/functools.py#l359 -.. [#pep-0008] PEP 8 states in the "Programming Recommendations" +.. [#pep-0008] :pep:`8` states in the "Programming Recommendations" section that "the Python standard library will not use function annotations as that would result in a premature commitment to a particular annotation style". - (http://www.python.org/dev/peps/pep-0008) .. [#why-c3] http://bugs.python.org/issue18244 -.. [#pep-3124] http://www.python.org/dev/peps/pep-3124/ - .. [#peak-rules] http://peak.telecommunity.com/DevCenter/PEAK_2dRules .. [#artima2005] diff --git a/pep-0444.txt b/peps/pep-0444.rst similarity index 97% rename from pep-0444.txt rename to peps/pep-0444.rst index cf5a905b5..e06bc8971 100644 --- a/pep-0444.txt +++ b/peps/pep-0444.rst @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Chris McDonough , Armin Ronacher -Discussions-To: Python Web-SIG +Discussions-To: web-sig@python.org Status: Deferred Type: Informational Content-Type: text/x-rst @@ -26,7 +26,7 @@ for lack of a current champion interested in promoting the goals of the PEP and collecting and incorporating feedback, and with sufficient available time to do so effectively. -Note that since this PEP was first created, PEP 3333 was created as a more +Note that since this PEP was first created, :pep:`3333` was created as a more incremental update that permitted use of WSGI on Python 3.2+. However, an alternative specification that furthers the Python 3 goals of a cleaner separation of binary and text data may still be valuable. @@ -35,10 +35,10 @@ Rationale and Goals =================== This protocol and specification is influenced heavily by the Web -Services Gateway Interface (WSGI) 1.0 standard described in PEP 333 -[1]_. The high-level rationale for having any standard that allows +Services Gateway Interface (WSGI) 1.0 standard described in :pep:`333`. +The high-level rationale for having any standard that allows Python-based web servers and applications to interoperate is outlined -in PEP 333. This document essentially uses PEP 333 as a template, and +in :pep:`333`. This document essentially uses :pep:`333` as a template, and changes its wording in various places for the purpose of forming a different standard. @@ -60,7 +60,7 @@ versions earlier than 2.6 nor Python 3 versions earlier than 3.1. .. note:: Whatever Python 3 version fixed http://bugs.python.org/issue4006 so - ``os.environ['foo']`` returns surrogates (ala PEP 383) when the + ``os.environ['foo']`` returns surrogates (ala :pep:`383`) when the value of 'foo' cannot be decoded using the current locale instead of failing with a KeyError is the *true* minimum Python 3 version. In particular, however, Python 3.0 is not supported. @@ -503,7 +503,7 @@ string. Each value is a bytes instance. ``SERVER_NAME``, ``SERVER_PORT`` When combined with ``SCRIPT_NAME`` and ``PATH_INFO`` (or their raw - equivalents)`, these variables can be used to complete the URL. + equivalents), these variables can be used to complete the URL. Note, however, that ``HTTP_HOST``, if present, should be used in preference to ``SERVER_NAME`` for reconstructing the request URL. See the `URL Reconstruction`_ section below for more detail. @@ -538,7 +538,7 @@ A server or gateway **should** attempt to provide as many other CGI variables as are applicable, each with a string for its key and a bytes instance for its value. In addition, if SSL is in use, the server or gateway **should** also provide as many of the Apache SSL -environment variables [5]_ as are applicable, such as ``HTTPS=on`` and +environment variables [4]_ as are applicable, such as ``HTTPS=on`` and ``SSL_PROTOCOL``. Note, however, that an application that uses any CGI variables other than the ones listed above are necessarily non-portable to web servers that do not support the relevant @@ -757,7 +757,7 @@ The ``status`` value is assumed by a gateway or server to be an HTTP "status" bytes instance like ``b'200 OK'`` or ``b'404 Not Found'``. That is, it is a string consisting of a Status-Code and a Reason-Phrase, in that order and separated by a single space, with no -surrounding whitespace or other characters. (See RFC 2616, Section +surrounding whitespace or other characters. (See :rfc:`2616`, Section 6.1.1 for more information.) The string **must not** contain control characters, and must not be terminated with a carriage return, linefeed, or combination thereof. @@ -765,7 +765,7 @@ linefeed, or combination thereof. The ``headers`` value is assumed by a gateway or server to be a literal Python list of ``(header_name, header_value)`` tuples. Each ``header_name`` must be a bytes instance representing a valid HTTP -header field-name (as defined by RFC 2616, Section 4.2), without a +header field-name (as defined by :rfc:`2616`, Section 4.2), without a trailing colon or other punctuation. Each ``header_value`` must be a bytes instance and **must not** include any control characters, including carriage returns or linefeeds, either embedded or at the @@ -941,9 +941,9 @@ The result of using a textlike object where a byteslike object is required is undefined. Values returned from a Web3 app as a status or as response headers -**must** follow RFC 2616 with respect to encoding. That is, the bytes +**must** follow :rfc:`2616` with respect to encoding. That is, the bytes returned must contain a character stream of ISO-8859-1 characters, or -the character stream should use RFC 2047 MIME encoding. +the character stream should use :rfc:`2047` MIME encoding. On Python platforms which do not have a native bytes-like type (e.g. IronPython, etc.), but instead which generally use textlike @@ -982,8 +982,8 @@ may be done in any of several ways: Note that these behavior restrictions do not apply for HTTP 1.0 requests, or for requests that are not directed to an application -object. For more information on HTTP 1.1 Expect/Continue, see RFC -2616, sections 8.2.3 and 10.1.1. +object. For more information on HTTP 1.1 Expect/Continue, see +:rfc:`2616`, sections 8.2.3 and 10.1.1. Other HTTP Features @@ -996,13 +996,14 @@ response. It is always possible for the application developer to add middleware components to supply additional features, so server/gateway developers should be conservative in their implementation. In a sense, a server should consider itself to be like an HTTP "gateway -server", with the application being an HTTP "origin server". (See RFC -2616, section 1.3, for the definition of these terms.) +server", with the application being an HTTP "origin server". (See +:rfc:`2616`, section 1.3, for the definition of these terms.) However, because Web3 servers and applications do not communicate via -HTTP, what RFC 2616 calls "hop-by-hop" headers do not apply to Web3 +HTTP, what :rfc:`2616` calls "hop-by-hop" headers do not apply to Web3 internal communications. Web3 applications **must not** generate any -"hop-by-hop" headers [4]_, attempt to use HTTP features that would +:rfc:`"hop-by-hop" headers <2616#section-13.5.1>`, +attempt to use HTTP features that would require them to generate such headers, or rely on the content of any incoming "hop-by-hop" headers in the ``environ`` dictionary. Web3 servers **must** handle any supported inbound "hop-by-hop" headers on @@ -1407,7 +1408,7 @@ Here are some alternatives to using all bytes: Applications Should be Allowed to Read ``web3.input`` Past ``CONTENT_LENGTH`` ----------------------------------------------------------------------------- -At [6]_, Graham Dumpleton makes the assertion that ``wsgi.input`` +At [5]_, Graham Dumpleton makes the assertion that ``wsgi.input`` should be required to return the empty string as a signifier of out-of-data, and that applications should be allowed to read past the number of bytes specified in ``CONTENT_LENGTH``, depending only upon @@ -1433,7 +1434,7 @@ There's no documented way to indicate that there is content in ``read()`` of ``web3.input`` Should Support No-Size Calling Convention ---------------------------------------------------------------------- -At [6]_, Graham Dumpleton makes the assertion that the ``read()`` +At [5]_, Graham Dumpleton makes the assertion that the ``read()`` method of ``wsgi.input`` should be callable without arguments, and that the result should be "all available request content". Needs discussion. @@ -1446,7 +1447,7 @@ Open for discussions though. Input Filters should set environ ``CONTENT_LENGTH`` to -1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -At [6]_, Graham Dumpleton suggests that an input filter might set +At [5]_, Graham Dumpleton suggests that an input filter might set ``environ['CONTENT_LENGTH']`` to -1 to indicate that it mutated the input. @@ -1471,7 +1472,7 @@ block iteration waiting for multiple values from an application iterable. If the middleware needs to accumulate more data from the application before it can produce any output, it **must** yield an empty string." This requirement existed to support asynchronous -applications and servers (see PEP 333's "Middleware Handling of Block +applications and servers (see :pep:`333`'s "Middleware Handling of Block Boundaries"). Asynchronous applications are now serviced explicitly by ``web3.async`` capable protocol (a Web3 application callable may itself return a callable). @@ -1491,25 +1492,25 @@ if the server performs URL rewriting. Long Response Headers --------------------- -Bob Brewer notes on Web-SIG [7]_: +Bob Brewer notes on Web-SIG [6]_: Each header_value must not include any control characters, including carriage returns or linefeeds, either embedded or at the end. (These requirements are to minimize the complexity of any parsing that must be performed by servers, gateways, and intermediate response processors that need to inspect or modify - response headers.) [1]_ + response headers.) (:pep:`333`) That's understandable, but HTTP headers are defined as (mostly) \*TEXT, and "words of \*TEXT MAY contain characters from character sets other than ISO-8859-1 only when encoded according to the rules of -RFC 2047." [2]_ And RFC 2047 specifies that "an 'encoded-word' may +:rfc:`2047`." [2]_ And :rfc:`2047` specifies that "an 'encoded-word' may not be more than 75 characters long... If it is desirable to encode more text than will fit in an 'encoded-word' of 75 characters, multiple 'encoded-word's (separated by CRLF SPACE) may be used." [3]_ This satisfies HTTP header folding rules, as well: "Header fields can be extended over multiple lines by preceding each extra line with at -least one SP or HT." [1]_ +least one SP or HT." (:pep:`333`) So in my reading of HTTP, some code somewhere should introduce newlines in longish, encoded response header values. I see three @@ -1543,25 +1544,18 @@ has all been read in. Neither WSGI nor Web3 currently supports them. References ========== -.. [1] PEP 333: Python Web Services Gateway Interface - (http://www.python.org/dev/peps/pep-0333/) - .. [2] The Common Gateway Interface Specification, v 1.1, 3rd Draft - (http://cgi-spec.golux.com/draft-coar-cgi-v11-03.txt) + (https://datatracker.ietf.org/doc/html/draft-coar-cgi-v11-03) -.. [3] "Chunked Transfer Coding" -- HTTP/1.1, section 3.6.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1) +.. [3] "Chunked Transfer Coding" -- HTTP/1.1, :rfc:`2616#section-3.6.1` -.. [4] "End-to-end and Hop-by-hop Headers" -- HTTP/1.1, Section 13.5.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1) - -.. [5] mod_ssl Reference, "Environment Variables" +.. [4] mod_ssl Reference, "Environment Variables" (http://www.modssl.org/docs/2.8/ssl_reference.html#ToC25) -.. [6] Details on WSGI 1.0 amendments/clarifications. +.. [5] Details on WSGI 1.0 amendments/clarifications. (http://blog.dscpl.com.au/2009/10/details-on-wsgi-10-amendmentsclarificat.html) -.. [7] [Web-SIG] WSGI and long response header values +.. [6] [Web-SIG] WSGI and long response header values https://mail.python.org/pipermail/web-sig/2006-September/002244.html Copyright diff --git a/pep-0445.txt b/peps/pep-0445.rst similarity index 99% rename from pep-0445.txt rename to peps/pep-0445.rst index eea0026c8..996a54cdc 100644 --- a/pep-0445.txt +++ b/peps/pep-0445.rst @@ -459,8 +459,8 @@ even in release mode: debug checks would always be compiled in, but only enabled when the environment variable is present and non-empty. This alternative was rejected because a new environment variable would -make Python initialization even more complex. `PEP 432 -`_ tries to simplify the +make Python initialization even more complex. :pep:`432` +tries to simplify the CPython startup sequence. diff --git a/pep-0446.txt b/peps/pep-0446.rst similarity index 99% rename from pep-0446.txt rename to peps/pep-0446.rst index 3b1813d46..9fd2b05fb 100644 --- a/pep-0446.txt +++ b/peps/pep-0446.rst @@ -587,7 +587,7 @@ Read the mail thread: `[Python-Dev] Proposal for a new function PEP 433 ------- -PEP 433, "Easier suppression of file descriptor inheritance", +:pep:`433`, "Easier suppression of file descriptor inheritance", was a previous attempt proposing various other alternatives, but no consensus could be reached. diff --git a/pep-0446/test_cloexec.py b/peps/pep-0446/test_cloexec.py similarity index 100% rename from pep-0446/test_cloexec.py rename to peps/pep-0446/test_cloexec.py diff --git a/pep-0447.txt b/peps/pep-0447.rst similarity index 99% rename from pep-0447.txt rename to peps/pep-0447.rst index a602f1237..626a54816 100644 --- a/pep-0447.txt +++ b/peps/pep-0447.rst @@ -1,13 +1,11 @@ PEP: 447 Title: Add __getdescriptor__ method to metaclass -Version: $Revision$ -Last-Modified: $Date$ Author: Ronald Oussoren Status: Deferred Type: Standards Track Content-Type: text/x-rst Created: 12-Jun-2013 -Post-History: 2-Jul-2013, 15-Jul-2013, 29-Jul-2013, 22-Jul-2015 +Post-History: 02-Jul-2013, 15-Jul-2013, 29-Jul-2013, 22-Jul-2015 Abstract @@ -117,7 +115,7 @@ Aside: Attribute resolution algorithm in Python ----------------------------------------------- The attribute resolution process as implemented by ``object.__getattribute__`` -(or PyObject_GenericGetAttr`` in CPython's implementation) is fairly +(or ``PyObject_GenericGetAttr`` in CPython's implementation) is fairly straightforward, but not entirely so without reading C code. The current CPython implementation of object.__getattribute__ is basically @@ -299,7 +297,7 @@ to ``tp_flags`` to indicate that new slot must be used. Use of this hook by the interpreter ----------------------------------- -The new method is required for metatypes and as such is defined on `type_`. +The new method is required for metatypes and as such is defined on ``type_``. Both ``super.__getattribute__`` and ``object.__getattribute__``/`PyObject_GenericGetAttr`_ (through ``_PyType_Lookup``) use the this ``__getdescriptor__`` method when diff --git a/pep-0448.txt b/peps/pep-0448.rst similarity index 92% rename from pep-0448.txt rename to peps/pep-0448.rst index 28a354ea5..38721e8a3 100644 --- a/pep-0448.txt +++ b/peps/pep-0448.rst @@ -1,7 +1,5 @@ PEP: 448 Title: Additional Unpacking Generalizations -Version: $Revision$ -Last-Modified: $Date$ Author: Joshua Landau Discussions-To: python-ideas@python.org Status: Final @@ -200,7 +198,7 @@ are already valid. These could be extended to:: f(**x for x in it) == f({**x for x in it}) However, it wasn't clear if this was the best behaviour or if it should -unpack into the arguments of the call to `f`. Since this is likely to be +unpack into the arguments of the call to ``f``. Since this is likely to be confusing and is of only very marginal utility, it is not included in this PEP. Instead, these will throw a ``SyntaxError`` and comprehensions with explicit brackets should be used instead. @@ -226,25 +224,14 @@ References .. [1] PEP accepted, "PEP 448 review", Guido van Rossum (https://mail.python.org/pipermail/python-dev/2015-February/138564.html) -.. [2] Issue 2292, "Missing `*`-unpacking generalizations", Thomas Wouters - (http://bugs.python.org/issue2292) +.. [2] Issue 2292, "Missing ``*``-unpacking generalizations", Thomas Wouters + (https://github.com/python/cpython/issues/46545) -.. [3] Discussion on Python-ideas list, - "list / array comprehensions extension", Alexander Heger - (https://mail.python.org/pipermail/python-ideas/2011-December/013097.html) +[3] Discussion on Python-ideas list, +\ "list / array comprehensions extension", Alexander Heger +\ (https://mail.python.org/pipermail/python-ideas/2011-December/013097.html) Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0449.txt b/peps/pep-0449.rst similarity index 94% rename from pep-0449.txt rename to peps/pep-0449.rst index 569ebc31d..bad7446c4 100644 --- a/pep-0449.txt +++ b/peps/pep-0449.rst @@ -7,6 +7,7 @@ BDFL-Delegate: Richard Jones Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 04-Aug-2013 Post-History: 04-Aug-2013 @@ -25,7 +26,7 @@ delegating a domain name under pypi.python.org to a third party. Rationale ========= -The PyPI mirroring infrastructure (defined in `PEP381`_) provides a means to +The PyPI mirroring infrastructure (defined in :pep:`381`) provides a means to mirror the content of PyPI used by the automatic installers. It also provides a method for auto discovery of mirrors and a consistent naming scheme. @@ -50,7 +51,7 @@ naming scheme: ever been implemented by one installer (pip), and its implementation, besides being insecure, has serious issues with performance and is slated for removal with its next release (1.5). -* While there are provisions in `PEP381`_ that would solve *some* of these +* While there are provisions in :pep:`381` that would solve *some* of these issues for a dedicated client it would not solve the issues that affect a users browser. Additionally these provisions have not been implemented by any installer to date. @@ -60,7 +61,7 @@ provides most of the benefit of the auto discovery and consistent naming scheme this PEP proposes to first deprecate and then remove the [a..z].pypi.python.org names for mirrors and the last.pypi.python.org name for the auto discovery protocol. The ability to mirror and the method of mirror will not be affected -and will continue to exist as written in `PEP381`_. Operators of existing +and will continue to exist as written in :pep:`381`. Operators of existing mirrors are encouraged to acquire their own domains and certificates to use for their mirrors if they wish to continue hosting them. @@ -131,13 +132,12 @@ the insecurity. Public or Private Mirrors ========================= -The mirroring protocol will continue to exist as defined in `PEP381`_ and +The mirroring protocol will continue to exist as defined in :pep:`381` and people are encouraged to host public and private mirrors if they so desire. The recommended mirroring client is `Bandersnatch`_. .. _PyPI: https://pypi.python.org/ -.. _PEP381: http://www.python.org/dev/peps/pep-0381/ .. _Bandersnatch: https://pypi.python.org/pypi/bandersnatch diff --git a/pep-0450.txt b/peps/pep-0450.rst similarity index 100% rename from pep-0450.txt rename to peps/pep-0450.rst diff --git a/pep-0451.txt b/peps/pep-0451.rst similarity index 97% rename from pep-0451.txt rename to peps/pep-0451.rst index 8a258581d..6e673e92a 100644 --- a/pep-0451.txt +++ b/peps/pep-0451.rst @@ -10,7 +10,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 08-Aug-2013 Python-Version: 3.4 -Post-History: 8-Aug-2013, 28-Aug-2013, 18-Sep-2013, 24-Sep-2013, 4-Oct-2013 +Post-History: 08-Aug-2013, 28-Aug-2013, 18-Sep-2013, 24-Sep-2013, 04-Oct-2013 Resolution: https://mail.python.org/pipermail/python-dev/2013-November/130104.html @@ -137,7 +137,7 @@ cache The import system stores compiled modules in the __pycache__ directory as an optimization. This module cache that we use today was provided by -PEP 3147. For this proposal, the relevant API for module caching is the +:pep:`3147`. For this proposal, the relevant API for module caching is the ``__cache__`` attribute of modules and the cache_from_source() function in importlib.util. Loaders are responsible for putting modules into the cache (and loading out of the cache). Currently the cache is only used @@ -160,10 +160,10 @@ Motivation ========== The import system has evolved over the lifetime of Python. In late 2002 -PEP 302 introduced standardized import hooks via finders and +:pep:`302` introduced standardized import hooks via finders and loaders and sys.meta_path. The importlib module, introduced with Python 3.1, now exposes a pure Python implementation of the APIs -described by PEP 302, as well as of the full import system. It is now +described by :pep:`302`, as well as of the full import system. It is now much easier to understand and extend the import system. While a benefit to the Python community, this greater accessibility also presents a challenge. @@ -179,7 +179,7 @@ generally only meaningful to the import system. It would be nice to have a per-module namespace in which to put future import-related information and to pass around within the import system. Secondly, there's an API void between finders and loaders that causes undue -complexity when encountered. The PEP 420 (namespace packages) +complexity when encountered. The :pep:`420` (namespace packages) implementation had to work around this. The complexity surfaced again during recent efforts on a separate proposal. [#ref_files_pep]_ @@ -217,7 +217,7 @@ those details. This is the same gap as before between finders and loaders. As an example of complexity attributable to this flaw, the -implementation of namespace packages in Python 3.3 (see PEP 420) added +implementation of namespace packages in Python 3.3 (see :pep:`420`) added FileFinder.find_loader() because there was no good way for find_module() to provide the namespace search locations. @@ -678,9 +678,9 @@ Other than adding ``__spec__``, none of the import-related module attributes will be changed or deprecated, though some of them could be; any such deprecation can wait until Python 4. -A module's spec will not be kept in sync with the corresponding import- -related attributes. Though they may differ, in practice they will -typically be the same. +A module's spec will not be kept in sync with the corresponding +import-related attributes. Though they may differ, in practice they +will typically be the same. One notable exception is that case where a module is run as a script by using the ``-m`` flag. In that case ``module.__spec__.name`` will @@ -716,8 +716,8 @@ Adding yet another similar method to loaders is a case of practicality. find_module() could be changed to return specs instead of loaders. This is tempting because the import APIs have suffered enough, especially considering PathEntryFinder.find_loader() was just -added in Python 3.3. However, the extra complexity and a less-than- -explicit method name aren't worth it. +added in Python 3.3. However, the extra complexity and a +less-than-explicit method name aren't worth it. The "target" parameter of find_spec() ------------------------------------- @@ -808,7 +808,7 @@ raising ImportError. Other changes: -PEP 420 introduced the optional module_repr() loader method to limit +:pep:`420` introduced the optional module_repr() loader method to limit the amount of special-casing in the module type's ``__repr__()``. Since this method is part of ModuleSpec, it will be deprecated on loaders. However, if it exists on a loader it will be used exclusively. @@ -856,7 +856,7 @@ Implementation Notes \* The implementation of this PEP needs to be cognizant of its impact on pkgutil (and setuptools). pkgutil has some generic function-based -extensions to PEP 302 which may break if importlib starts wrapping +extensions to :pep:`302` which may break if importlib starts wrapping loaders without the tools' knowledge. \* Other modules to look at: runpy (and pythonrun.c), pickle, pydoc, @@ -900,7 +900,7 @@ module.__spec__ is less than desirable. They would end up being an attractive nuisance, even if only exposed as "private" attributes (as they were in previous versions of this PEP). If someone finds a need for these methods later, we can expose the via an appropriate API -(separate from ModuleSpec) at that point, perhaps relative to PEP 406 +(separate from ModuleSpec) at that point, perhaps relative to :pep:`406` (import engine). Conceivably, the load() method could optionally take a list of diff --git a/pep-0452.txt b/peps/pep-0452.rst similarity index 100% rename from pep-0452.txt rename to peps/pep-0452.rst diff --git a/pep-0453.txt b/peps/pep-0453.rst similarity index 95% rename from pep-0453.txt rename to peps/pep-0453.rst index c72ef958a..cbb4ac80d 100644 --- a/pep-0453.txt +++ b/peps/pep-0453.rst @@ -1,7 +1,5 @@ PEP: 453 Title: Explicit bootstrapping of pip in Python installations -Version: $Revision$ -Last-Modified: $Date$ Author: Donald Stufft , Nick Coghlan BDFL-Delegate: Martin von Löwis @@ -31,7 +29,7 @@ PEP Acceptance This PEP was accepted for inclusion in Python 3.4 by Martin von Löwis on Tuesday 22nd October, 2013. -`Issue 19347 `__ has been created to +`Issue 19347 `__ has been created to track the implementation of this PEP. @@ -120,7 +118,7 @@ third-party packages as well as easier for the people distributing them as they should now be able to safely assume that most users will have the appropriate installation tools available (or access to clear instructions on how to obtain them). This is expected to become more important in the future -as the Wheel_ package format (deliberately) does not have a built in +as the :pep:`Wheel <427>` package format (deliberately) does not have a built in "installer" in the form of ``setup.py`` so users wishing to install from a wheel file will want an installer even in the simplest cases. @@ -435,7 +433,7 @@ into the CPython release is as follows: * ``ensurepip`` updated to use a ``pip`` 1.5 release candidate. - * PEP 101 updated to cover ensuring the bundled version of ``pip`` is up + * :pep:`101` updated to cover ensuring the bundled version of ``pip`` is up to date. * by November 24th (scheduled date of 3.4.0 beta 1) @@ -450,7 +448,7 @@ into the CPython release is as follows: subsequent maintenance release (including a suitably updated vendored copy of ``requests``) -(See PEP 429 for the current official scheduled dates of each release. Dates +(See :pep:`429` for the current official scheduled dates of each release. Dates listed above are accurate as of October 20th, 2013.) If there is no final or maintenance release of ``pip`` 1.5 with a suitable @@ -646,10 +644,9 @@ detail. Other projects which explicitly require ``setuptools`` must still provide an appropriate dependency declaration, rather than assuming ``setuptools`` will always be installed alongside ``pip``. -Once pip is able to run ``pip install --upgrade pip`` without needing -``setuptools`` installed first, then the private copy of ``setuptools`` -will be removed from ``ensurepip`` in subsequent CPython releases. - +The private copy of ``setuptools`` will be removed from ``ensurepip`` +once it is no longer needed. This is likely to be at the point when +``get-pip.py`` stops installing ``setuptools`` by default. As long as setuptools is needed, it will be a completely unmodified copy of the latest upstream setuptools release, including the ``easy_install`` script if the upstream setuptools continues to include it. The installation @@ -803,7 +800,8 @@ downstream distributors: * ``pip install --upgrade pip`` in a virtual environment should not affect the global installation. -* Migrate build systems to utilize `pip`_ and `Wheel`_ wherever feasible +* Migrate build systems to utilize `pip`_ and :pep:`Wheel <427>` + wherever feasible and avoid directly invoking ``setup.py``. * This will help ensure a smoother and more timely migration to improved @@ -944,7 +942,7 @@ invoke ``ensurepip`` by default when installing from a custom source build. Implicit bootstrap ------------------ -`PEP439`_, the predecessor for this PEP, proposes its own solution. Its +:pep:`439`, the predecessor for this PEP, proposes its own solution. Its solution involves shipping a fake ``pip`` command that when executed would implicitly bootstrap and install pip if it does not already exist. This has been rejected because it is too "magical". It hides from the end user when @@ -1013,58 +1011,45 @@ which are not handled correctly when pip is installed into the user site-packages directory rather than the system site-packages). -.. _Wheel: http://www.python.org/dev/peps/pep-0427/ .. _pip: http://www.pip-installer.org .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _PEP439: http://www.python.org/dev/peps/pep-0439/ References ========== -.. [1] Discussion thread 1 (distutils-sig) - (https://mail.python.org/pipermail/distutils-sig/2013-August/022529.html) +* `Discussion thread 1 (distutils-sig) + `_ -.. [2] Discussion thread 2 (distutils-sig) - (https://mail.python.org/pipermail/distutils-sig/2013-September/022702.html) +* `Discussion thread 2 (distutils-sig) + `_ -.. [3] Discussion thread 3 (python-dev) - (https://mail.python.org/pipermail/python-dev/2013-September/128723.html) +* `Discussion thread 3 (python-dev) + `_ -.. [4] Discussion thread 4 (python-dev) - (https://mail.python.org/pipermail/python-dev/2013-September/128780.html) +* `Discussion thread 4 (python-dev) + `_ -.. [5] Discussion thread 5 (python-dev) - (https://mail.python.org/pipermail/python-dev/2013-September/128894.html) +* `Discussion thread 5 (python-dev) + `_ -.. [#cert-verification] pip/requests certificate management concerns - (https://mail.python.org/pipermail/python-dev/2013-October/129755.html) +.. [#cert-verification] `pip/requests certificate management concerns + `_ -.. [#windows-incompatibility] Windows installer compatibility concerns - (https://mail.python.org/pipermail/distutils-sig/2013-October/022855.html) +.. [#windows-incompatibility] `Windows installer compatibility concerns + `_ -.. [#ubuntu] `Ubuntu ` -.. [#debian] `Debian ` -.. [#fedora] `Fedora ` -.. [#homebrew] `Homebrew ` -.. [#macports] `MacPorts ` -.. [#fink] `Fink ` -.. [#ContinuumIO] `Anaconda ` -.. [#ActiveState] `ActivePython ` -.. [#Enthought] `Enthought Canopy ` +.. [#ubuntu] `Ubuntu `__ +.. [#debian] `Debian `__ +.. [#fedora] `Fedora `__ +.. [#homebrew] `Homebrew `__ +.. [#macports] `MacPorts `__ +.. [#fink] `Fink `__ +.. [#ContinuumIO] `Anaconda `__ +.. [#ActiveState] `ActivePython `__ +.. [#Enthought] `Enthought Canopy `__ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0454.txt b/peps/pep-0454.rst similarity index 99% rename from pep-0454.txt rename to peps/pep-0454.rst index 895bced9b..7575386af 100644 --- a/pep-0454.txt +++ b/peps/pep-0454.rst @@ -49,7 +49,7 @@ the diagram is too huge to be analyzed manually. Proposal ======== -Using the customized allocation API from PEP 445, it becomes easy to +Using the customized allocation API from :pep:`445`, it becomes easy to set up a hook on Python memory allocators. A hook can inspect Python internals to retrieve Python tracebacks. The idea of getting the current traceback comes from the faulthandler module. The faulthandler dumps diff --git a/pep-0455.txt b/peps/pep-0455.rst similarity index 100% rename from pep-0455.txt rename to peps/pep-0455.rst diff --git a/pep-0456.txt b/peps/pep-0456.rst similarity index 98% rename from pep-0456.txt rename to peps/pep-0456.rst index a9866e780..240ada264 100644 --- a/pep-0456.txt +++ b/peps/pep-0456.rst @@ -519,7 +519,7 @@ buffer. Performance =========== -In general the PEP 456 code with SipHash24 is about as fast as the old code +In general the :pep:`456` code with SipHash24 is about as fast as the old code with FNV. SipHash24 seems to make better use of modern compilers, CPUs and large L1 cache. Several benchmarks show a small speed improvement on 64 bit CPUs such as Intel Core i5 and Intel Core i7 processes. 32 bit builds and @@ -529,7 +529,7 @@ not affect any application code. The benchmarks were conducted on CPython default branch revision b08868fd5994 and the PEP repository [pep-456-repos]_. All upstream changes were merged -into the pep-456 branch. The "performance" CPU governor was configured and +into the ``pep-456`` branch. The "performance" CPU governor was configured and almost all programs were stopped so the benchmarks were able to utilize TurboBoost and the CPU caches as much as possible. The raw benchmark results of multiple machines and platforms are made available at [benchmarks]_. @@ -633,7 +633,7 @@ for the common case. ASCII str / bytes hash collision -------------------------------- -Since the implementation of [pep-0393]_ bytes and ASCII text have the same +Since the implementation of :pep:`393`, bytes and ASCII text have the same memory layout. Because of this the new hashing API will keep the invariant:: hash("ascii string") == hash(b"ascii string") @@ -678,8 +678,6 @@ References .. [csiphash] https://github.com/majek/csiphash/ -.. [pep-0393] http://www.python.org/dev/peps/pep-0393/ - .. [aes-ni] http://en.wikipedia.org/wiki/AES_instruction_set .. [pluggable] https://mail.python.org/pipermail/python-dev/2013-October/129138.html diff --git a/pep-0457.txt b/peps/pep-0457.rst similarity index 97% rename from pep-0457.txt rename to peps/pep-0457.rst index 6e4d2e9fb..bc9730ec8 100644 --- a/pep-0457.txt +++ b/peps/pep-0457.rst @@ -3,7 +3,7 @@ Title: Notation For Positional-Only Parameters Version: $Revision$ Last-Modified: $Date$ Author: Larry Hastings -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -22,8 +22,8 @@ their position. This PEP is an Informational PEP describing the notation for use when describing APIs that use positional-only parameters (e.g. in Argument -Clinic, or in the string representation of `inspect.Signature` -objects). A separate PEP, PEP 570, proposes elevation of this notation +Clinic, or in the string representation of ``inspect.Signature`` +objects). A separate PEP, :pep:`570`, proposes elevation of this notation to full Python syntax. ========= @@ -49,7 +49,7 @@ expressible with Python syntax. This PEP proposes a notation for such signatures that could form the basis of a backwards-compatible syntax that should permit implementing -any builtin in pure Python code (see PEP 570 for that proposal). +any builtin in pure Python code (see :pep:`570` for that proposal). ----------------------------------------------------- Positional-Only Parameter Semantics In Current Python diff --git a/peps/pep-0458-1.png b/peps/pep-0458-1.png new file mode 100644 index 000000000..d455a22ac Binary files /dev/null and b/peps/pep-0458-1.png differ diff --git a/pep-0458.txt b/peps/pep-0458.rst similarity index 96% rename from pep-0458.txt rename to peps/pep-0458.rst index 6e6a274f9..f16faab27 100644 --- a/pep-0458.txt +++ b/peps/pep-0458.rst @@ -14,6 +14,7 @@ BDFL-Delegate: Donald Stufft Discussions-To: https://discuss.python.org/t/pep-458-secure-pypi-downloads-with-package-signing/2648 Status: Accepted Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 27-Sep-2013 Post-History: 06-Jan-2019, 13-Nov-2019 @@ -73,7 +74,7 @@ responsibilities by automating much of the signing process. There is no discussion in *this* PEP of support for project distributions that are signed by developers (maximum security model). This possible future extension -is covered in detail in PEP 480 [21]_. The maximum security model requires more PyPI +is covered in detail in :pep:`480`. The maximum security model requires more PyPI administrative work (though no added work for clients), and also proposes an easy-to-use key management solution for developers/publishers, ideas on how to interface with a potential future build farm on PyPI infrastructure, and the @@ -85,8 +86,7 @@ to install or update projects from PyPI with TUF metadata. Package managers interested in adopting TUF on the client side may consult its `library documentation`__, which was created for this purpose. -__ https://github.com/theupdateframework/tuf/tree/v0.11.1/tuf/client#updaterpy - +__ https://theupdateframework.readthedocs.io/en/stable/api/tuf.ngclient.html Non-goals ========= @@ -94,7 +94,7 @@ Non-goals This PEP does not eliminate any existing features from PyPI. In particular, it does not replace existing support for OpenPGP signatures. Developers can continue to upload detached OpenPGP signatures along with distributions. In the future, -PEP 480 may allow developers to directly sign TUF metadata using their OpenPGP keys. +:pep:`480` may allow developers to directly sign TUF metadata using their OpenPGP keys. PEP Status @@ -123,7 +123,7 @@ software repositories are vulnerable to an attacker on the network (MITM) intercepting and changing files. These and other attacks on software repositories are detailed here__. -This PEP, together with the follow-up proposal in PEP 480, aims to protect users +This PEP, together with the follow-up proposal in :pep:`480`, aims to protect users of PyPI from compromises of the integrity, consistency, and freshness properties of PyPI packages, and enhances compromise resilience by mitigating key risk and providing mechanisms to recover from a compromise of PyPI or its signing keys. @@ -158,11 +158,11 @@ possible through other avenues. For example, a public mirror is trusted to honestly mirror PyPI, but some mirrors may misbehave, whether by accident or through malicious intervention. Package managers such as pip are supposed to use signatures from PyPI to verify -distribution files downloaded from a public mirror [9]_, but none are known to actually +distribution files downloaded from a :pep:`public mirror <381>`, but none are known to actually do so [10]_. Therefore, it would be wise to add more security measures to detect attacks from public mirrors or content delivery networks [11]_ (CDNs). -Even though official mirrors have been deprecated on PyPI [12]_, a +Even though official mirrors have been :pep:`deprecated on PyPI <449>`, a wide variety of other attack vectors on package managers remain [13]_. These attacks can crash client systems, cause obsolete distributions to be installed, or even allow an attacker to execute arbitrary code. In `September 2013`__, a post was @@ -189,7 +189,7 @@ __ https://theupdateframework.github.io/audits.html The scope of *this* PEP is protecting users from compromises of PyPI mirrors, and PyPI's own TLS termination and content distribution infrastructure. -Protection from compromises of PyPI itself is discussed in PEP 480. +Protection from compromises of PyPI itself is discussed in :pep:`480`. Threat Model @@ -209,7 +209,7 @@ software distribution file. If the attacker is preventing the installation of updates, they do not want clients to realize there is anything wrong. This threat model describes the minimum security model. The maximum security -model described in PEP 480 also assumes that attackers can compromise PyPI's +model described in :pep:`480` also assumes that attackers can compromise PyPI's online keys. @@ -218,9 +218,7 @@ Definitions The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in RFC 2119__. - -__ http://www.ietf.org/rfc/rfc2119.txt +interpreted as described in :rfc:`2119`. This PEP focuses only on integrating TUF into PyPI. However, the reader is encouraged to review TUF design principles [2]_ and SHOULD be @@ -331,26 +329,26 @@ Integrating PyPI with TUF ========================= A software update system must complete two main tasks to integrate with TUF. -First, it must add the framework to the client side of the update system. For +First, the repository on the server side MUST be modified to provide signed +TUF metadata. This PEP is concerned with the first part of the integration, +and the changes on PyPI required to support software updates with TUF. + +Second, it must add the framework to the client side of the update system. For example, TUF MAY be integrated with the pip package manager. Thus, new versions of pip going forward SHOULD use TUF by default to download and verify distributions from PyPI before installing them. However, there may be unforeseen issues that might prevent users from installing or updating distributions, including pip itself, via TUF. Therefore, pip SHOULD provide an option e.g., -`--unsafely-disable-package-verification`, in order to work around such issues +``--unsafely-disable-package-verification``, in order to work around such issues until they are resolved. Note, the proposed option name is purposefully long, because a user must be helped to understand that the action is unsafe and not generally recommended. -Second, the repository on the server side MUST be modified to provide signed -TUF metadata. This PEP is concerned with the second part of the integration, -and the changes on PyPI required to support software updates with TUF. We assume that pip would use TUF to verify distributions downloaded only from PyPI. pip MAY support TAP 4__ in order use TUF to also verify distributions downloaded -from elsewhere__. +from :pep:`elsewhere <470>`. __ https://github.com/theupdateframework/taps/blob/master/tap4.md -__ https://www.python.org/dev/peps/pep-0470/ @@ -373,11 +371,11 @@ from PyPI. TUF downloads them and checks them against the TUF metadata that it also downloads from the repository. If the downloaded target files are trustworthy, TUF then hands them over to the package manager. -The `Metadata`__ document provides information about each type of required -metadata and its expected content. The next section covers the different -kinds of metadata RECOMMENDED for PyPI. +The `Document formats`__ section of the TUF specification provides information +about each type of required metadata and its expected content. The next +section covers the different kinds of metadata RECOMMENDED for PyPI. -__ https://github.com/theupdateframework/tuf/blob/v0.11.1/docs/METADATA.md +__ https://theupdateframework.github.io/specification/latest/#document-formats In addition, all target files SHOULD be available on disk at least two times. Once under their original filename, to provide backwards compatibility, and @@ -493,6 +491,7 @@ trusted to sign for files available on PyPI. The next two sections cover the details of signing repository files and the types of keys used for each role. .. image:: pep-0458-1.png + :class: invert-in-dark-mode Figure 1: An overview of the role metadata available on PyPI. @@ -509,11 +508,12 @@ that PyPI uses to produce snapshots that can safely coexist and be deleted independent of other snapshots [18]_. Every year, PyPI administrators SHOULD sign for *root* and *targets* role keys. -Automation will continuously sign for a timestamped snapshot of all projects. -A `repository management`__ tool is available that can sign metadata files, -generate cryptographic keys, and manage a TUF repository. +Automation will continuously sign for a timestamped snapshot of all projects. A +repository `Metadata API`__ is available that can be used to `manage a TUF +repository`__. -__ https://github.com/theupdateframework/tuf/blob/v0.11.1/docs/TUTORIAL.md#how-to-create-and-modify-a-tuf-repository +__ https://theupdateframework.readthedocs.io/en/stable/api/tuf.api.html +__ https://github.com/theupdateframework/python-tuf/blob/v0.20.0/examples/repo_example/basic_repo.py In standard operation, the *bin-n* metadata will be updated and signed as new distributions are uploaded to PyPI. However, there will also need to be a @@ -562,8 +562,8 @@ one proposed in this PEP is the minimum security model, which supports verification of PyPI distributions signed with private cryptographic keys stored on PyPI. Distributions uploaded by developers are signed by PyPI and immediately available for download. A possible future extension to this -PEP, discussed in PEP 480 [21]_, proposes the maximum security model and allows -a developer to sign for his/her project. Developer keys are not stored online: +PEP, discussed in :pep:`480`, proposes the maximum security model and allows +a developer to sign for their project. Developer keys are not stored online: therefore, projects are safe from PyPI compromises. The minimum security model requires no action from a developer and protects @@ -588,7 +588,7 @@ considered to be non-existent on PyPI. Note, the reason why *targets* does not directly delegate to *bin-n*, but instead uses the intermediary *bins* role, is so that other delegations can easily be added or removed, without affecting the *bins*-to-*bin-n* mapping. -This is crucial for the implementation of PEP 480 [21]_. +This is crucial for the implementation of :pep:`480`. Metadata Expiry Times @@ -611,14 +611,14 @@ grow correspondingly. For example, consider the *bins* role. In August 2013, it was found that the size of the *bins* metadata was about 42MB if the *bins* role itself signed for about 220K PyPI targets (which are simple indices and distributions). This PEP does not delve into the details, but TUF features a -so-called "`lazy bin walk`__" scheme that splits a large targets metadata file +so-called `"hashed bin delegation"`__ scheme that splits a large targets metadata file into many small ones. This allows a TUF client updater to intelligently download only a small number of TUF metadata files in order to update any project signed for by the *bins* role. For example, applying this scheme to the previous repository resulted in pip downloading between 1.3KB and 111KB to install or upgrade a PyPI project via TUF. -__ https://github.com/theupdateframework/tuf/blob/v0.11.1/docs/TUTORIAL.md#delegate-to-hashed-bins +__ https://github.com/theupdateframework/python-tuf/blob/v0.20.0/examples/repo_example/hashed_bin_delegation.py Based on our findings as of the time this document was updated for implementation (Nov 7 2019), summarized in Tables 2-3, PyPI SHOULD @@ -666,7 +666,7 @@ C8 was computed by querying the number of release files. C9 was derived by taking the average between a rough estimate of the average size of release files *downloaded* over the past 31 days (1,628,321 bytes), and the average size of releases files on disk (2,740,465 bytes). -Ee W. Durbin III helped to provide these numbers on November 7, 2019. +Ee Durbin helped to provide these numbers on November 7, 2019. Table 2: A list of constants used to calculate metadata overhead. @@ -722,9 +722,9 @@ overhead for returning and new users would be around 50-54% and 114% respectively, assuming that the number of bins stay fixed. If the number of bins is increased, then the cost for all users would effectively be the cost for new users, because their cost would be dominated by the (once-in-a-while) -cost of downloading the large number of delegations in the `bins` metadata. +cost of downloading the large number of delegations in the ``bins`` metadata. If the cost for new users should prove to be too much, primarily due to the -overhead of downloading the `bins` metadata, then this subject SHOULD be +overhead of downloading the ``bins`` metadata, then this subject SHOULD be revisited before that happens. Note that changes to the number of bins on the server are transparent to the @@ -1094,7 +1094,7 @@ attack, or metadata inconsistency attacks. Note that if the timestamp, snapshot, and bin-n roles are stored in the same online location, a compromise of one means they will all be compromised. Therefore, the table considers these roles together. A version of this table that considers these roles separately -is included in PEP 480 [21]_. +is included in :pep:`480`. +-----------------+-------------------+----------------+--------------------------------+ | Role Compromise | Malicious Updates | Freeze Attack | Metadata Inconsistency Attacks | @@ -1137,7 +1137,7 @@ allow the repository to recover from an attack by revoking the online key(s). The maximum security model shows how TUF mitigates online key compromises by introducing additional roles for end-to-signing. Details about how to generate -developer keys and sign upload distributions are provided in PEP 480 [21]_. +developer keys and sign upload distributions are provided in :pep:`480`. In the Event of a Key Compromise @@ -1359,12 +1359,8 @@ References .. [6] https://mail.python.org/pipermail/distutils-sig/2013-April/020596.html .. [7] https://mail.python.org/pipermail/distutils-sig/2013-May/020701.html .. [8] https://mail.python.org/pipermail/distutils-sig/2013-July/022008.html -.. [9] PEP 381, Mirroring infrastructure for PyPI, ZiadĂ©, Löwis - http://www.python.org/dev/peps/pep-0381/ .. [10] https://mail.python.org/pipermail/distutils-sig/2013-September/022773.html .. [11] https://mail.python.org/pipermail/distutils-sig/2013-May/020848.html -.. [12] PEP 449, Removal of the PyPI Mirror Auto Discovery and Naming Scheme, Stufft - http://www.python.org/dev/peps/pep-0449/ .. [13] https://theupdateframework.github.io/papers/attacks-on-package-managers-ccs2008.pdf .. [14] https://mail.python.org/pipermail/distutils-sig/2013-September/022755.html .. [15] http://ed25519.cr.yp.to/ @@ -1373,7 +1369,6 @@ References .. [18] https://en.wikipedia.org/wiki/Continuous_delivery .. [19] https://mail.python.org/pipermail/distutils-sig/2013-August/022154.html .. [20] https://en.wikipedia.org/wiki/Key-recovery_attack -.. [21] https://www.python.org/dev/peps/pep-0480/ .. [22] https://pyfound.blogspot.com/2019/09/pypi-security-q4-2019-request-for.html Acknowledgements @@ -1397,7 +1392,7 @@ Justin Samuel, Tian Tian, Santiago Torres, John Ward, and Yuyu Zheng in developing TUF. Vladimir Diaz, Monzur Muhammad, Sai Teja Peddinti, Sumana Harihareswara, -Ee W. Durbin III and Dustin Ingram helped us to review this PEP. +Ee Durbin and Dustin Ingram helped us to review this PEP. Zane Fisher helped us to review and transcribe this PEP. diff --git a/pep-0459.txt b/peps/pep-0459.rst similarity index 98% rename from pep-0459.txt rename to peps/pep-0459.rst index 66fbdbdc5..74920f90d 100644 --- a/pep-0459.txt +++ b/peps/pep-0459.rst @@ -4,24 +4,25 @@ Version: $Revision$ Last-Modified: $Date$ Author: Nick Coghlan BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG +Discussions-To: distutils-sig@python.org Status: Withdrawn Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Requires: 426 Created: 11-Nov-2013 -Post-History: 21 Dec 2013 +Post-History: 21-Dec-2013 PEP Withdrawal ============== -This PEP depends on PEP 426, which has itself been withdrawn. See the +This PEP depends on :pep:`426`, which has itself been withdrawn. See the PEP Withdrawal section in that PEP for details. In the meantime, metadata extensions will continue to be handled as they have been for past examples like ``entry_points.txt``: as additional files -installed into metadata directories alongside the main `METADATA` file. +installed into metadata directories alongside the main ``METADATA`` file. Abstract @@ -112,7 +113,7 @@ Classifiers ----------- A list of strings, with each giving a single classification value -for the distribution. Classifiers are described in PEP 301 [2]. +for the distribution. Classifiers are described in :pep:`301` [2]. Example:: @@ -202,8 +203,8 @@ If no specific role is stated, the default is ``contributor``. Email addresses must be in the form ``local-part@domain`` where the local-part may be up to 64 characters long and the entire email address contains no more than 254 characters. The formal specification of the -format is in RFC 5322 (sections 3.2.3 and 3.4.1) and RFC 5321, with a more -readable form given in the informational RFC 3696 and the associated errata. +format is in :rfc:`5322` (sections 3.2.3 and 3.4.1) and :rfc:`5321`, with a more +readable form given in the informational :rfc:`3696` and the associated errata. The defined contributor roles are as follows: diff --git a/pep-0460.txt b/peps/pep-0460.rst similarity index 100% rename from pep-0460.txt rename to peps/pep-0460.rst diff --git a/pep-0461.txt b/peps/pep-0461.rst similarity index 94% rename from pep-0461.txt rename to peps/pep-0461.rst index 7512a02a0..09036768d 100644 --- a/pep-0461.txt +++ b/peps/pep-0461.rst @@ -1,15 +1,13 @@ PEP: 461 Title: Adding % formatting to bytes and bytearray -Version: $Revision$ -Last-Modified: $Date$ Author: Ethan Furman Status: Final Type: Standards Track Content-Type: text/x-rst Created: 13-Jan-2014 Python-Version: 3.5 -Post-History: 2014-01-14, 2014-01-15, 2014-01-17, 2014-02-22, 2014-03-25, - 2014-03-27 +Post-History: 14-Jan-2014, 15-Jan-2014, 17-Jan-2014, 22-Feb-2014, 25-Mar-2014, + 27-Mar-2014 Resolution: https://mail.python.org/pipermail/python-dev/2014-March/133621.html @@ -55,14 +53,14 @@ All the numeric formatting codes (``d``, ``i``, ``o``, ``u``, ``x``, ``X``, ``e``, ``E``, ``f``, ``F``, ``g``, ``G``, and any that are subsequently added to Python 3) will be supported, and will work as they do for str, including the padding, justification and other related modifiers (currently ``#``, ``0``, -``-``, `` `` (space), and ``+`` (plus any added to Python 3)). The only +``-``, space, and ``+`` (plus any added to Python 3)). The only non-numeric codes allowed are ``c``, ``b``, ``a``, and ``s`` (which is a synonym for b). For the numeric codes, the only difference between ``str`` and ``bytes`` (or ``bytearray``) interpolation is that the results from these codes will be ASCII-encoded text, not unicode. In other words, for any numeric formatting -code `%x`:: +code ``%x``:: b"%x" % val @@ -195,7 +193,7 @@ Various new special methods were proposed, such as ``__ascii__``, ``__format_bytes__``, etc.; such methods are not needed at this time, but can be visited again later if real-world use shows deficiencies with this solution. -A competing PEP, ``PEP 460 Add binary interpolation and formatting`` [8]_, +A competing PEP, :pep:`PEP 460 Add binary interpolation and formatting <460>`, also exists. @@ -237,20 +235,9 @@ Footnotes .. [6] https://mail.python.org/pipermail/python-dev/2014-February/132750.html .. [7] http://bugs.python.org/issue23467 -- originally ``%r`` was not allowed, but was added for consistency during the 3.5 alpha stage. -.. [8] http://python.org/dev/peps/pep-0460/ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0462.txt b/peps/pep-0462.rst similarity index 97% rename from pep-0462.txt rename to peps/pep-0462.rst index 510a9842e..0c2c7d778 100644 --- a/pep-0462.txt +++ b/peps/pep-0462.rst @@ -1,7 +1,5 @@ PEP: 462 Title: Core development workflow automation for CPython -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Coghlan Status: Withdrawn Type: Process @@ -28,7 +26,7 @@ PEP Withdrawal This PEP has been `withdrawn by the author `_ -in favour of the GitLab based proposal in PEP 507. +in favour of the GitLab based proposal in :pep:`507`. If anyone else would like to take over championing this PEP, contact the `core-workflow mailing list `_ @@ -180,7 +178,7 @@ CPython core development workflow. This proposal suggests replacing the use of Rietveld for code review with the more full-featured Kallithea-based forge.python.org service proposed in -PEP 474. Guido has indicated that the original Rietveld implementation was +:pep:`474`. Guido has indicated that the original Rietveld implementation was primarily intended as a public demonstration application for Google App Engine, and switching to Kallithea will address some of the issues with identifying intended target branches that arise when working with patch files @@ -306,7 +304,7 @@ builds without errors. As yet another alternative, it may be reasonable to move some parts of the documentation (such as the tutorial and the HOWTO guides) out of the main source repository and manage them using the simpler pull request based model -described in PEP 474. +described in :pep:`474`. Perceived Benefits @@ -403,7 +401,7 @@ Mercurial vs Gerrit/git Gerrit uses git as the actual storage mechanism for patches, and automatically handles merging of approved patches. By contrast, Kallithea -use the RhodeCode created `vcs ` library as +use the RhodeCode created `vcs `_ library as an abstraction layer over specific DVCS implementations (with Mercurial and git backends currently available). @@ -511,7 +509,7 @@ Our current approach to handling NEWS file updates regularly results in spurious conflicts when merging bug fixes forward from an active maintenance branch to a later branch. -`Issue #18967* `__ discusses some +`Issue #18967* `__ discusses some possible improvements in that area, which would be beneficial regardless of whether or not we adopt Zuul as a workflow automation tool. @@ -552,7 +550,7 @@ One useful part of the OpenStack workflow is the "git review" plugin, which makes it relatively easy to push a branch from a local git clone up to Gerrit for review. -PEP 474 mentions a draft `custom Mercurial +:pep:`474` mentions a draft `custom Mercurial extension `__ that automates some aspects of the existing CPython core development workflow. @@ -596,7 +594,7 @@ infrastructure primarily due to past experience with unacceptably poor performance and inflexibility of infrastructure provided for free to the general public. CPython development was originally hosted on SourceForge, with source control moved to self hosting when SF was both slow to offer -Subversion support and suffering from CVS performance issues (see PEP 347), +Subversion support and suffering from CVS performance issues (see :pep:`347`), while issue tracking later moved to the open source Roundup issue tracker on dedicated sponsored hosting (from Upfront Systems), due to a combination of both SF performance issues and general usability issues with the SF @@ -633,7 +631,7 @@ Buildbot continuous integration fleet, have taken an extended period of time as volunteers worked their way through the many technical and social challenges involved. -Fortunately, as several aspects of this proposal and PEP 474 align with +Fortunately, as several aspects of this proposal and :pep:`474` align with various workflow improvements under consideration for Red Hat's `Beaker `__ open source hardware integration testing system and other work-related projects, I have arranged to be able @@ -673,7 +671,7 @@ Next Steps ========== If pursued, this will be a follow-on project to the Kallithea-based -forge.python.org proposal in PEP 474. Refer to that PEP for more details +forge.python.org proposal in :pep:`474`. Refer to that PEP for more details on the discussion, review and proof-of-concept pilot process currently under way. @@ -687,7 +685,7 @@ Taylor for additional technical feedback following publication of the initial draft. Thanks to Bradley Kuhn, Mads Kiellerich and other Kallithea developers for -the discussions around PEP 474 that led to a significant revision of this +the discussions around :pep:`474` that led to a significant revision of this proposal to be based on using Kallithea for the review component rather than the existing Rietveld installation. @@ -696,12 +694,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0463.txt b/peps/pep-0463.rst similarity index 99% rename from pep-0463.txt rename to peps/pep-0463.rst index 30c40d684..68f1e1f29 100644 --- a/pep-0463.txt +++ b/peps/pep-0463.rst @@ -46,7 +46,7 @@ your next PEP! Abstract ======== -Just as PEP 308 introduced a means of value-based conditions in an +Just as :pep:`308` introduced a means of value-based conditions in an expression, this system allows exception-based conditions to be used as part of an expression. @@ -833,7 +833,7 @@ that it catch BaseException and be unable to capture it with 'as'). Bare except clauses ------------------- -PEP 8 rightly advises against the use of a bare 'except'. While it is +:pep:`8` rightly advises against the use of a bare 'except'. While it is syntactically legal in a statement, and for backward compatibility must remain so, there is little value in encouraging its use. In an expression except clause, "except:" is a SyntaxError; use the equivalent long-hand @@ -919,7 +919,7 @@ This proposal simply adds a fifth: * except-expression - exception list: result -Style guides and PEP 8 should recommend not having the colon at the end of +Style guides and :pep:`8` should recommend not having the colon at the end of a wrapped line, which could potentially look like the introduction of a suite, but instead advocate wrapping before the exception list, keeping the colon clearly between two expressions. diff --git a/pep-0464.txt b/peps/pep-0464.rst similarity index 92% rename from pep-0464.txt rename to peps/pep-0464.rst index dd1bf5f20..cbc696a71 100644 --- a/pep-0464.txt +++ b/peps/pep-0464.rst @@ -7,6 +7,7 @@ BDFL-Delegate: Richard Jones Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 02-Mar-2014 Post-History: 04-Mar-2014 @@ -24,7 +25,7 @@ API, this includes the /serverkey URL and all of the URLs under /serversig. Rationale ========= -The PyPI mirroring infrastructure (defined in PEP 381) provides a means to +The PyPI mirroring infrastructure (defined in :pep:`381`) provides a means to mirror the content of PyPI used by the automatic installers, and as a component of that, it provides a method for verifying the authenticity of the mirrored content. @@ -38,7 +39,7 @@ This PEP proposes the removal of this API due to: there is *any* bias in the random nonce. * This API solves one small corner of the trust problem, however the problem itself is much larger and it would be better to have a fully fledged system, - such as `The Update Framework `_, + such as :pep:`The Update Framework <458>`, instead. Due to the issues it has and the lack of use it is the opinion of this PEP @@ -62,7 +63,7 @@ If PyPI 2.0 has not been deployed in place of PyPI 1.0 by Sept 01 2014 then this PEP will be implemented in the PyPI 1.0 code base instead (by removing the associated code). -No changes will be required in the installers, however PEP 381 compliant +No changes will be required in the installers, however :pep:`381` compliant mirroring clients, such as `bandersnatch `_ and `pep381client `_ will need to be diff --git a/pep-0465.txt b/peps/pep-0465.rst similarity index 98% rename from pep-0465.txt rename to peps/pep-0465.rst index ebed4df3b..79f3ec705 100644 --- a/pep-0465.txt +++ b/peps/pep-0465.rst @@ -9,6 +9,8 @@ Content-Type: text/x-rst Created: 20-Feb-2014 Python-Version: 3.5 Post-History: 13-Mar-2014 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/D63NDWHPF7OC2Z455MPHOW6QLLSNQUJ5/ + Abstract ======== @@ -173,7 +175,7 @@ impossible to get right at any scale. Functions that defensively try to handle both types as input and DTRT, find themselves floundering into a swamp of ``isinstance`` and ``if`` statements. -PEP 238 split ``/`` into two operators: ``/`` and ``//``. Imagine the +:pep:`238` split ``/`` into two operators: ``/`` and ``//``. Imagine the chaos that would have resulted if it had instead split ``int`` into two types: ``classic_int``, whose ``__div__`` implemented floor division, and ``new_int``, whose ``__div__`` implemented true @@ -386,8 +388,8 @@ real-world code (i.e., all the code on Github). We checked for imports of several popular stdlib modules, a variety of numerically oriented modules, and various other extremely high-profile modules like django and lxml (the latter of which is the #1 most downloaded -package on PyPI). Starred lines indicate packages which export array- -or matrix-like objects which will adopt ``@`` if this PEP is +package on PyPI). Starred lines indicate packages which export +array- or matrix-like objects which will adopt ``@`` if this PEP is approved:: Count of Python source files on Github matching given search terms @@ -795,7 +797,7 @@ expression context are: ``@``, backtick, ``$``, ``!``, and ``?``. Of these options, ``@`` is clearly the best; ``!`` and ``?`` are already heavily freighted with inapplicable meanings in the programming context, backtick has been banned from Python by BDFL pronouncement -(see PEP 3099), and ``$`` is uglier, even more dissimilar to ``*`` and +(see :pep:`3099`), and ``$`` is uglier, even more dissimilar to ``*`` and :math:`\cdot`, and has Perl/PHP baggage. ``$`` is probably the second-best option of these, though. @@ -844,7 +846,7 @@ options: be too easy to confuse with ``*+``, which is just multiplication combined with the unary ``+`` operator. -* PEP 211 suggested ``~*``. This has the downside that it sort of +* :pep:`211` suggested ``~*``. This has the downside that it sort of suggests that there is a unary ``*`` operator that is being combined with unary ``~``, but it could work. @@ -1008,12 +1010,12 @@ Rejected alternatives to adding a new operator Over the past few decades, the Python numeric community has explored a variety of ways to resolve the tension between matrix and elementwise -multiplication operations. PEP 211 and PEP 225, both proposed in 2000 +multiplication operations. :pep:`211` and :pep:`225`, both proposed in 2000 and last seriously discussed in 2008 [#threads-2008]_, were early attempts to add new operators to solve this problem, but suffered from serious flaws; in particular, at that time the Python numerical community had not yet reached consensus on the proper API for array -objects, or on what operators might be needed or useful (e.g., PEP 225 +objects, or on what operators might be needed or useful (e.g., :pep:`225` proposes 6 new operators with unspecified semantics). Experience since then has now led to consensus that the best solution, for both numeric Python and core Python, is to add a single infix operator for @@ -1046,13 +1048,13 @@ infix operators:** In addition to being generally un-Pythonic and repeatedly rejected by BDFL fiat, this would be using a sledgehammer to smash a fly. The scientific python community has consensus that adding one operator for matrix multiplication is enough to fix the one -otherwise unfixable pain point. (In retrospect, we all think PEP 225 +otherwise unfixable pain point. (In retrospect, we all think :pep:`225` was a bad idea too -- or at least far more complex than it needed to be.) **Add a new @ (or whatever) operator that has some other meaning in general Python, and then overload it in numeric code:** This was the -approach taken by PEP 211, which proposed defining ``@`` to be the +approach taken by :pep:`211`, which proposed defining ``@`` to be the equivalent of ``itertools.product``. The problem with this is that when taken on its own terms, it's pretty clear that ``itertools.product`` doesn't actually need a dedicated operator. It @@ -1134,11 +1136,11 @@ systems work. **Use a special "facade" type to support syntax like arr.M * arr:** This is very similar to the previous proposal, in that the ``.M`` -attribute would basically return the same object as ``arr *dot` would, +attribute would basically return the same object as ``arr *dot`` would, and thus suffers the same objections about 'magicalness'. This approach also has some non-obvious complexities: for example, while -``arr.M * arr`` must return an array, ``arr.M * arr.M`` and ``arr * -arr.M`` must return facade objects, or else ``arr.M * arr.M * arr`` +``arr.M * arr`` must return an array, ``arr.M * arr.M`` and +``arr * arr.M`` must return facade objects, or else ``arr.M * arr.M * arr`` and ``arr * arr.M * arr`` will not work. But this means that facade objects must be able to recognize both other array objects and other facade objects (which creates additional complexity for writing diff --git a/scan-ops.py b/peps/pep-0465/scan-ops.py old mode 100644 new mode 100755 similarity index 98% rename from scan-ops.py rename to peps/pep-0465/scan-ops.py index 79603b598..052cf63bf --- a/scan-ops.py +++ b/peps/pep-0465/scan-ops.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# http://legacy.python.org/dev/peps/pep-0465/ +# https://peps.python.org/pep-0465/ # https://gist.github.com/njsmith/9157645 # usage: diff --git a/pep-0466.txt b/peps/pep-0466.rst similarity index 100% rename from pep-0466.txt rename to peps/pep-0466.rst diff --git a/pep-0467.txt b/peps/pep-0467.rst similarity index 89% rename from pep-0467.txt rename to peps/pep-0467.rst index 773967f82..413eba6ce 100644 --- a/pep-0467.txt +++ b/peps/pep-0467.rst @@ -7,8 +7,9 @@ Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 30-Mar-2014 -Python-Version: 3.11 -Post-History: 2014-03-30 2014-08-15 2014-08-16 2016-06-07 2016-09-01 2021-04-13 2021-11-03 +Python-Version: 3.12 +Post-History: 30-Mar-2014, 15-Aug-2014, 16-Aug-2014, 07-Jun-2016, 01-Sep-2016, + 13-Apr-2021, 03-Nov-2021 Abstract @@ -30,7 +31,7 @@ During the initial development of the Python 3 language specification, the core ``bytes`` type for arbitrary binary data started as the mutable type that is now referred to as ``bytearray``. Other aspects of operating in the binary domain in Python have also evolved over the course of the Python -3 series, for example with PEP 461. +3 series, for example with :pep:`461`. Motivation @@ -127,7 +128,7 @@ Python 2 ``str``) was as simple as:: >>> str(123) '123' - + With Python 3 that became the more verbose:: >>> b'%d' % 123 @@ -143,7 +144,7 @@ to handle this use-case:: b'123' Note that ``bytes.ascii()`` would handle simple ascii-encodable text correctly, -unlike the `ascii()`` built-in:: +unlike the ``ascii()`` built-in:: >>> ascii("hello").encode('ascii') b"'hello'" @@ -198,7 +199,7 @@ Zero-initialised sequences can be created via sequence repetition:: However, this was also the case when the ``bytearray`` type was originally designed, and the decision was made to add explicit support for it in the type constructor. The immutable ``bytes`` type then inherited that feature -when it was introduced in PEP 3137. +when it was introduced in :pep:`3137`. This PEP isn't revisiting that original design decision, just changing the spelling as users sometimes find the current behavior of the binary sequence @@ -235,18 +236,12 @@ of this PEP. References ========== -.. [1] Initial March 2014 discussion thread on python-ideas - (https://mail.python.org/pipermail/python-ideas/2014-March/027295.html) -.. [2] Guido's initial feedback in that thread - (https://mail.python.org/pipermail/python-ideas/2014-March/027376.html) -.. [3] Issue proposing moving zero-initialised sequences to a dedicated API - (http://bugs.python.org/issue20895) -.. [4] Issue proposing to use calloc() for zero-initialised binary sequences - (http://bugs.python.org/issue21644) -.. [5] August 2014 discussion thread on python-dev - (https://mail.python.org/pipermail/python-ideas/2014-March/027295.html) -.. [6] June 2016 discussion thread on python-dev - (https://mail.python.org/pipermail/python-dev/2016-June/144875.html) +* `Initial March 2014 discussion thread on python-ideas `_ +* `Guido's initial feedback in that thread `_ +* `Issue proposing moving zero-initialised sequences to a dedicated API `_ +* `Issue proposing to use calloc() for zero-initialised binary sequences `_ +* `August 2014 discussion thread on python-dev `_ +* `June 2016 discussion thread on python-dev `_ Copyright diff --git a/pep-0468.txt b/peps/pep-0468.rst similarity index 97% rename from pep-0468.txt rename to peps/pep-0468.rst index de8defd98..fec816e7f 100644 --- a/pep-0468.txt +++ b/peps/pep-0468.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 05-Apr-2014 Python-Version: 3.6 -Post-History: 5-Apr-2014,8-Sep-2016 +Post-History: 05-Apr-2014, 08-Sep-2016 Resolution: https://mail.python.org/pipermail/python-dev/2016-September/146329.html @@ -201,7 +201,7 @@ idea regardless. (There is a reason those discussions were brief.) Relationship to inspect.Signature --------------------------------- -Signature objects should need no changes. The `kwargs` parameter of +Signature objects should need no changes. The ``kwargs`` parameter of inspect.BoundArguments (returned by Signature.bind() and Signature.bind_partial()) will change from a dict to an OrderedDict. @@ -321,7 +321,7 @@ application-level use of annotations. dict.__order__ -------------- -dict objects would have a new attribute, `__order__` that would default +dict objects would have a new attribute, ``__order__`` that would default to None and that in the kwargs case the interpreter would use in the same way as described above for __kworder__. @@ -329,13 +329,13 @@ Prognosis: It would mean zero impact on kwargs performance but the change would be pretty intrusive (Python uses dict a lot). Also, for the wrapper case -the interpreter would have to be careful to preserve `__order__`. +the interpreter would have to be careful to preserve ``__order__``. KWArgsDict.__order__ -------------------- -This is the same as the `dict.__order__` idea, but kwargs would be an -instance of a new minimal dict subclass that provides the `__order__` +This is the same as the ``dict.__order__`` idea, but kwargs would be an +instance of a new minimal dict subclass that provides the ``__order__`` attribute. dict would instead be unchanged. Prognosis: diff --git a/pep-0469.txt b/peps/pep-0469.rst similarity index 99% rename from pep-0469.txt rename to peps/pep-0469.rst index 5af6b8879..1307bfc08 100644 --- a/pep-0469.txt +++ b/peps/pep-0469.rst @@ -8,13 +8,13 @@ Type: Standards Track Content-Type: text/x-rst Created: 18-Apr-2014 Python-Version: 3.5 -Post-History: 2014-04-18, 2014-04-21 +Post-History: 18-Apr-2014, 21-Apr-2014 Abstract ======== -For Python 3, PEP 3106 changed the design of the ``dict`` builtin and the +For Python 3, :pep:`3106` changed the design of the ``dict`` builtin and the mapping API in general to replace the separate list based and iterator based APIs in Python 2 with a merged, memory efficient set and multiset view based API. This new style of dict iteration was also added to the Python 2.7 diff --git a/pep-0470.txt b/peps/pep-0470.rst similarity index 92% rename from pep-0470.txt rename to peps/pep-0470.rst index 87b904465..9ad243a22 100644 --- a/pep-0470.txt +++ b/peps/pep-0470.rst @@ -7,6 +7,7 @@ BDFL-Delegate: Paul Moore Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 12-May-2014 Post-History: 14-May-2014, 05-Jun-2014, 03-Oct-2014, 13-Oct-2014, 26-Aug-2015 @@ -19,7 +20,7 @@ Abstract This PEP proposes the deprecation and removal of support for hosting files externally to PyPI as well as the deprecation and removal of the functionality -added by PEP 438, particularly rel information to classify different types of +added by :pep:`438`, particularly rel information to classify different types of links and the meta-tag to indicate API version. @@ -55,14 +56,14 @@ PyPI works, and other projects works, but this one specific one does not. They oftentimes do not realize who they need to contact in order to get this fixed or what their remediation steps are. -PEP 438 attempted to solve this issue by allowing projects to explicitly +:pep:`438` attempted to solve this issue by allowing projects to explicitly declare if they were using the repository features or not, and if they were not, it had the installers classify the links it found as either "internal", -"verifiable external" or "unverifiable external". PEP 438 was accepted and +"verifiable external" or "unverifiable external". :pep:`438` was accepted and implemented in pip 1.4 (released on Jul 23, 2013) with the final transition implemented in pip 1.5 (released on Jan 2, 2014). -PEP 438 was successful in bringing about more people to utilize PyPI's +:pep:`438` was successful in bringing about more people to utilize PyPI's repository features, an altogether good thing given the global CDN powering PyPI providing speed ups for a lot of people, however it did so by introducing a new point of confusion and pain for both the end users and the authors. @@ -119,9 +120,9 @@ Why Not PEP 438 or Similar? --------------------------- While the additional search location support has existed in pip and setuptools -for quite some time support for PEP 438 has only existed in pip since the 1.4 +for quite some time support for :pep:`438` has only existed in pip since the 1.4 version, and still has yet to be implemented in setuptools. The design of -PEP 438 did mean that users still benefited for projects which did not require +:pep:`438` did mean that users still benefited for projects which did not require external files even with older installers, however for projects which *did* require external files, users are still silently being given either potentially unreliable or, even worse, unsafe files to download. This system is also unique @@ -129,7 +130,7 @@ to Python as it arises out of the history of PyPI, this means that it is almost certain that this concept will be foreign to most, if not all users, until they encounter it while attempting to use the Python toolchain. -Additionally, the classification system proposed by PEP 438 has, in practice, +Additionally, the classification system proposed by :pep:`438` has, in practice, turned out to be extremely confusing to end users, so much so that it is a position of this PEP that the situation as it stands is completely untenable. The common pattern for a user with this system is to attempt to install a @@ -146,13 +147,13 @@ This UX failure exists for several reasons. #. If pip can locate files at all for a project on the Simple API it will simply use that instead of attempting to locate more. This is generally the right thing to do as attempting to locate more would erase a large part of - the benefit of PEP 438. This means that if a project *ever* uploaded a file + the benefit of :pep:`438`. This means that if a project *ever* uploaded a file that matches what the user has requested for install that will be used regardless of how old it is. -#. PEP 438 makes an implicit assumption that most projects would either upload +#. :pep:`438` makes an implicit assumption that most projects would either upload themselves to PyPI or would update themselves to directly linking to release files. While a large number of projects did ultimately decide to upload to - PyPI, some of them did so only because the UX around what PEP 438 was so bad + PyPI, some of them did so only because the UX around what :pep:`438` was so bad that they felt forced to do so. More concerning however, is the fact that very few projects have opted to directly and safely link to files and instead they still simply link to pages which must be scraped in order to @@ -162,7 +163,7 @@ This UX failure exists for several reasons. non-obvious. It requires the inclusion of a MD5 hash (for historical reasons) in the hash of the URL. If they do not include this then their files will be considered "unverified". -#. PEP 438 takes a security centric view and disallows any form of a global opt +#. :pep:`438` takes a security centric view and disallows any form of a global opt in for unverified projects. While this is generally a good thing, it creates extremely verbose and repetitive command invocations such as:: @@ -209,7 +210,7 @@ Deprecation and Removal of Link Spidering ========================================= A new hosting mode will be added to PyPI. This hosting mode will be called -``pypi-only`` and will be in addition to the three that PEP 438 has already +``pypi-only`` and will be in addition to the three that :pep:`438` has already given us which are ``pypi-explicit``, ``pypi-scrape``, ``pypi-scrape-crawl``. This new hosting mode will modify a project's simple api page so that it only lists the files which are directly hosted on PyPI and will not link to anything @@ -254,7 +255,7 @@ Summary of Changes Repository side --------------- -#. Deprecate and remove the hosting modes as defined by PEP 438. +#. Deprecate and remove the hosting modes as defined by :pep:`438`. #. Restrict simple API to only list the files that are contained within the repository. @@ -264,7 +265,7 @@ Client side #. Implement multiple repository support. #. Implement some mechanism for removing/disabling the default repository. -#. Deprecate / Remove PEP 438 +#. Deprecate / Remove :pep:`438` Impact @@ -288,7 +289,7 @@ those, 59 of them rely on external files that are safely hosted outside of PyPI and 931 of them rely on external files which are unsafely hosted outside of PyPI. This shows us that 1.5% of projects will be affected in some way by this change while 98.5% will continue to function as they always have. In addition, -only 5% of the projects affected are using the features provided by PEP 438 to +only 5% of the projects affected are using the features provided by :pep:`438` to safely host outside of PyPI while 95% of them are exposing their users to Remote Code Execution via a Man In The Middle attack. @@ -318,9 +319,9 @@ default list of repositories. However, part of this answer will also be explaining that the previous behavior of transparently including external links was both a security hazard (given that in most cases it allowed a MITM to execute arbitrary Python code on the end users machine) and a reliability -concern and that PEP 438 attempted to resolve this by making them explicitly -opt in, but that PEP 438 brought along with it a number of serious usability -issues. PEP 470 represents a simplification of the model to a model that many +concern and that :pep:`438` attempted to resolve this by making them explicitly +opt in, but that :pep:`438` brought along with it a number of serious usability +issues. :pep:`470` represents a simplification of the model to a model that many users will be familiar with, which is common amongst Linux distributions. @@ -386,7 +387,7 @@ can add to their installer to make the installation work. This feature has been removed from the scope of the PEP because it proved too difficult to develop a solution that avoided UX issues similar to those that -caused so many problems with the PEP 438 solution. If needed, a future PEP +caused so many problems with the :pep:`438` solution. If needed, a future PEP could revisit this idea. @@ -395,7 +396,7 @@ Keep the current classification system but adjust the options This PEP rejects several related proposals which attempt to fix some of the usability problems with the current system but while still keeping the general -gist of PEP 438. +gist of :pep:`438`. This includes: @@ -405,12 +406,12 @@ This includes: * Default to disallowing safely externally hosted files with only a global flag to enable them, but disallow unsafely hosted. -* Continue on the suggested path of PEP 438 and remove the option to unsafely +* Continue on the suggested path of :pep:`438` and remove the option to unsafely host externally but continue to allow the option to safely host externally. These proposals are rejected because: -* The classification system introduced in PEP 438 in an entirely unique concept +* The classification system introduced in :pep:`438` in an entirely unique concept to PyPI which is not generically applicable even in the context of Python packaging. Adding additional concepts comes at a cost. @@ -429,7 +430,7 @@ These proposals are rejected because: determine if a link is expected to fail or not. * The mechanism paints a very broad brush when enabling an option, while - PEP 438 attempts to limit this with per package options. However a project + :pep:`438` attempts to limit this with per package options. However a project that has existed for an extended period of time may oftentimes have several different URLs listed in their simple index. It is not unusual for at least one of these to no longer be under control of the project. While an diff --git a/pep-0471.txt b/peps/pep-0471.rst similarity index 98% rename from pep-0471.txt rename to peps/pep-0471.rst index 52bde075e..3a02fda57 100644 --- a/pep-0471.txt +++ b/peps/pep-0471.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 30-May-2014 Python-Version: 3.5 -Post-History: 27-Jun-2014, 8-Jul-2014, 14-Jul-2014 +Post-History: 27-Jun-2014, 08-Jul-2014, 14-Jul-2014 Abstract @@ -46,8 +46,8 @@ better than this.) In practice, removing all those extra system calls makes ``os.walk()`` about **8-9 times as fast on Windows**, and about **2-3 times as fast -on POSIX systems**. So we're not talking about micro- -optimizations. See more `benchmarks here`_. +on POSIX systems**. So we're not talking about +micro-optimizations. See more `benchmarks here`_. .. _`benchmarks here`: https://github.com/benhoyt/scandir#benchmarks @@ -293,7 +293,7 @@ direct support for a scandir-like function from core developers and others on the python-dev and python-ideas mailing lists. A sampling: * **python-dev**: a good number of +1's and very few negatives for - scandir and PEP 471 on `this June 2014 python-dev thread + scandir and :pep:`471` on `this June 2014 python-dev thread `_ * **Nick Coghlan**, a core Python developer: "I've had the local Red @@ -562,7 +562,7 @@ methods are enough and this information is already known. See `Ben Hoyt's July 2014 reply `_ to the discussion summarizing this and detailing why he thinks the -original PEP 471 proposal is "the right one" after all. +original :pep:`471` proposal is "the right one" after all. Return values being (name, stat_result) two-tuples @@ -674,11 +674,11 @@ Previous discussion * `First July 2014 thread Ben Hoyt started on python-dev `_ - to discuss his updates to PEP 471 + to discuss his updates to :pep:`471` * `Second July 2014 thread Ben Hoyt started on python-dev `_ - to discuss the remaining decisions needed to finalize PEP 471, + to discuss the remaining decisions needed to finalize :pep:`471`, specifically whether the ``DirEntry`` methods should follow symlinks by default diff --git a/pep-0472.txt b/peps/pep-0472.rst similarity index 98% rename from pep-0472.txt rename to peps/pep-0472.rst index ddec13dce..900e51164 100644 --- a/pep-0472.txt +++ b/peps/pep-0472.rst @@ -216,7 +216,7 @@ Cons - Very strict. - Destroys ordering of the passed arguments. Preserving the - order would be possible with an OrderedDict as drafted by PEP-468 [#PEP-468]_. + order would be possible with an OrderedDict as drafted by :pep:`468`. - Does not allow use cases with mixed positional/keyword arguments such as ``a[1, 2, default=5]``. @@ -356,7 +356,7 @@ For the C2 case: P1 naturally maps to the traditional ``**kwargs`` behavior, however it breaks the convention that two or more entries for the index produce a tuple. P2 preserves this behavior, and additionally preserves the order. Preserving the -order would also be possible with an OrderedDict as drafted by PEP-468 [#PEP-468]_. +order would also be possible with an OrderedDict as drafted by :pep:`468`. The remaining cases are here shown: @@ -554,7 +554,7 @@ Relevance of ordering of keyword arguments As part of the discussion of this PEP, it's important to decide if the ordering information of the keyword arguments is important, and if indexes and keys can -be ordered in an arbitrary way (e.g. ``a[1,Z=3,2,R=4]``). PEP-468 [#PEP-468]_ +be ordered in an arbitrary way (e.g. ``a[1,Z=3,2,R=4]``). :pep:`468` tries to address the first point by proposing the use of an ordereddict, however one would be inclined to accept that keyword arguments in indexing are equivalent to kwargs in function calls, and therefore as of today equally @@ -635,9 +635,6 @@ References .. [#namedtuple] "namedtuple is not as good as it should be" (https://mail.python.org/pipermail/python-ideas/2013-June/021257.html) -.. [#PEP-468] "Preserving the order of \*\*kwargs in a function." - http://legacy.python.org/dev/peps/pep-0468/ - Copyright ========= diff --git a/pep-0473.txt b/peps/pep-0473.rst similarity index 100% rename from pep-0473.txt rename to peps/pep-0473.rst diff --git a/pep-0474.txt b/peps/pep-0474.rst similarity index 98% rename from pep-0474.txt rename to peps/pep-0474.rst index 59bce90cc..4e3490e87 100644 --- a/pep-0474.txt +++ b/peps/pep-0474.rst @@ -20,7 +20,7 @@ more accessible to new contributors, and easier to manage for core developers. This PEP does *not* propose any changes to the core development workflow -for CPython itself (see PEP 462 in relation to that). +for CPython itself (see :pep:`462` in relation to that). PEP Withdrawal @@ -28,7 +28,7 @@ PEP Withdrawal This PEP has been `withdrawn by the author `_ -in favour of the GitLab based proposal in PEP 507. +in favour of the GitLab based proposal in :pep:`507`. If anyone else would like to take over championing this PEP, contact the `core-workflow mailing list `_ @@ -102,7 +102,7 @@ but may be negotiable if a sufficiently compelling alternative is presented: the standard pull request workflows offered by those tools * SHOULD be open to customisation to meet the needs of CPython core development, including providing a potential path forward for the - proposed migration to a core reviewer model in PEP 462 + proposed migration to a core reviewer model in :pep:`462` The preference for self-hosting without ongoing fees rules out the free-as-in-beer providers like GitHub and BitBucket, in addition to the @@ -419,7 +419,7 @@ with the rollout of the following pilot deployment: services) * automatic linking of issue references in code review comments and commit messages to the corresponding issues on bugs.python.org -* draft updates to PEP 1 explaining the Kallithea-based PEP editing and +* draft updates to :pep:`1` explaining the Kallithea-based PEP editing and submission workflow The following items would be needed for a production migration, but there @@ -439,7 +439,7 @@ selected and the proposed pilot deployment is successful): * allowing the use of the GitHub and BitBucket pull request workflows to submit pull requests to the main Kallithea repo * allowing easy triggering of forced BuildBot runs based on Kallithea hosted - repos and pull requests (prior to the implementation of PEP 462, this + repos and pull requests (prior to the implementation of :pep:`462`, this would be intended for use with sandbox repos rather than the main CPython repo) @@ -449,7 +449,7 @@ Future Implications for CPython Core Development The workflow requirements for the main CPython development repository are significantly more complex than those for the repositories being discussed -in this PEP. These concerns are covered in more detail in PEP 462. +in this PEP. These concerns are covered in more detail in :pep:`462`. Given Guido's recommendation to replace Rietveld with a more actively maintained code review system, my current plan is to rewrite that PEP to diff --git a/pep-0475.txt b/peps/pep-0475.rst similarity index 100% rename from pep-0475.txt rename to peps/pep-0475.rst diff --git a/pep-0476.txt b/peps/pep-0476.rst similarity index 99% rename from pep-0476.txt rename to peps/pep-0476.rst index 95595eaf6..37fab9f8d 100644 --- a/pep-0476.txt +++ b/peps/pep-0476.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 28-Aug-2014 +Python-Version: 2.7.9, 3.4.3, 3.5 Resolution: https://mail.python.org/pipermail/python-dev/2014-October/136676.html Abstract diff --git a/pep-0477.txt b/peps/pep-0477.rst similarity index 91% rename from pep-0477.txt rename to peps/pep-0477.rst index bb7520df2..721bae141 100644 --- a/pep-0477.txt +++ b/peps/pep-0477.rst @@ -1,15 +1,13 @@ PEP: 477 Title: Backport ensurepip (PEP 453) to Python 2.7 -Version: $Revision$ -Last-Modified: $Date$ -Author: Donald Stufft +Author: Donald Stufft , Nick Coghlan BDFL-Delegate: Benjamin Peterson Status: Final Type: Standards Track Content-Type: text/x-rst Created: 26-Aug-2014 -Post-History: 1-Sep-2014 +Post-History: 01-Sep-2014 Resolution: https://mail.python.org/pipermail/python-dev/2014-September/136238.html @@ -62,14 +60,14 @@ reasons: finally actually installing it first. Furthermore, alternative implementations of Python are recognizing the benefits -of PEP 453 and both PyPy and Jython have plans to backport ensurepip to their +of :pep:`453` and both PyPy and Jython have plans to backport ensurepip to their 2.7 runtimes. Automatic Invocation ==================== -PEP 453 has ``ensurepip`` automatically invoked by default in the ``Makefile`` +:pep:`453` has ``ensurepip`` automatically invoked by default in the ``Makefile`` and the Windows and OSX Installers. This allowed it to ensure that, by default, all users would get Python with pip already installed. This PEP however believes that while this is fine for the Python 2.7 Windows and Mac OS X @@ -107,7 +105,7 @@ module it is explicitly allowed for downstream distributors to patch the If a downstream distributor wishes to disable ``ensurepip`` completely in Python 2.7, they should still at least provide the module and allow -`python -m ensurepip` style invocation. However it should raise errors or +``python -m ensurepip`` style invocation. However it should raise errors or otherwise exit with a non-zero exit code and print out an error on stderr directing users to what they can/should use instead of ``ensurepip``. @@ -127,14 +125,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0478.txt b/peps/pep-0478.rst similarity index 73% rename from pep-0478.txt rename to peps/pep-0478.rst index e974c2cda..901c8c16c 100644 --- a/pep-0478.txt +++ b/peps/pep-0478.rst @@ -3,8 +3,9 @@ Title: Python 3.5 Release Schedule Version: $Revision$ Last-Modified: $Date$ Author: Larry Hastings -Status: Active +Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 22-Sep-2014 Python-Version: 3.5 @@ -80,19 +81,19 @@ including their release dates. Features for 3.5 ================ -* PEP 441, improved Python zip application support -* PEP 448, additional unpacking generalizations -* PEP 461, "%-formatting" for bytes and bytearray objects -* PEP 465, a new operator ("@") for matrix multiplication -* PEP 471, os.scandir(), a fast new directory traversal function -* PEP 475, adding support for automatic retries of interrupted system calls -* PEP 479, change StopIteration handling inside generators -* PEP 484, the typing module, a new standard for type annotations -* PEP 485, math.isclose(), a function for testing approximate equality -* PEP 486, making the Windows Python launcher aware of virtual environments -* PEP 488, eliminating .pyo files -* PEP 489, a new and improved mechanism for loading extension modules -* PEP 492, coroutines with async and await syntax +* :pep:`441`, improved Python zip application support +* :pep:`448`, additional unpacking generalizations +* :pep:`461`, "%-formatting" for bytes and bytearray objects +* :pep:`465`, a new operator ("@") for matrix multiplication +* :pep:`471`, os.scandir(), a fast new directory traversal function +* :pep:`475`, adding support for automatic retries of interrupted system calls +* :pep:`479`, change StopIteration handling inside generators +* :pep:`484`, the typing module, a new standard for type annotations +* :pep:`485`, math.isclose(), a function for testing approximate equality +* :pep:`486`, making the Windows Python launcher aware of virtual environments +* :pep:`488`, eliminating .pyo files +* :pep:`489`, a new and improved mechanism for loading extension modules +* :pep:`492`, coroutines with async and await syntax Copyright diff --git a/pep-0479.txt b/peps/pep-0479.rst similarity index 98% rename from pep-0479.txt rename to peps/pep-0479.rst index 4bb2e3321..e5cbe823e 100644 --- a/pep-0479.txt +++ b/peps/pep-0479.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 15-Nov-2014 Python-Version: 3.5 -Post-History: 15-Nov-2014, 19-Nov-2014, 5-Dec-2014 +Post-History: 15-Nov-2014, 19-Nov-2014, 05-Dec-2014 Abstract @@ -55,11 +55,11 @@ is raised, a traceback is printed pinpointing the cause of the problem.) This is particularly pernicious in combination with the ``yield from`` -construct of PEP 380 [1]_, as it breaks the abstraction that a +construct of :pep:`380`, as it breaks the abstraction that a subgenerator may be factored out of a generator. That PEP notes this -limitation, but notes that "use cases for these [are] rare to non- -existent". Unfortunately while intentional use is rare, it is easy to -stumble on these cases by accident:: +limitation, but notes that "use cases for these [are] rare to +non-existent". Unfortunately while intentional use is rare, it is +easy to stumble on these cases by accident:: import contextlib @@ -580,9 +580,6 @@ by a simple ``return``. References ========== -.. [1] PEP 380 - Syntax for Delegating to a Subgenerator - (https://www.python.org/dev/peps/pep-0380) - .. [2] Initial mailing list comment (https://mail.python.org/pipermail/python-ideas/2014-November/029906.html) diff --git a/peps/pep-0480-1.png b/peps/pep-0480-1.png new file mode 100644 index 000000000..fe486abfd Binary files /dev/null and b/peps/pep-0480-1.png differ diff --git a/pep-0480.txt b/peps/pep-0480.rst similarity index 94% rename from pep-0480.txt rename to peps/pep-0480.rst index d5252f9a4..dbd5e3dd6 100644 --- a/pep-0480.txt +++ b/peps/pep-0480.rst @@ -1,14 +1,13 @@ PEP: 480 Title: Surviving a Compromise of PyPI: End-to-end signing of packages -Version: $Revision$ -Last-Modified: $Date$ Author: Trishank Karthik Kuppusamy , Vladimir Diaz , Justin Cappos , Marina Moore BDFL-Delegate: Donald Stufft -Discussions-To: Packaging category on Python Discourse +Discussions-To: https://discuss.python.org/t/5666 Status: Draft Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Requires: 458 Created: 08-Oct-2014 @@ -17,22 +16,22 @@ Created: 08-Oct-2014 Abstract ======== -Proposed is an extension to PEP 458 that adds support for end-to-end signing +Proposed is an extension to :pep:`458` that adds support for end-to-end signing and the maximum security model. End-to-end signing allows both PyPI and developers to sign for the distributions that are downloaded by clients. The -minimum security model proposed by PEP 458 supports continuous delivery of +minimum security model proposed by :pep:`458` supports continuous delivery of distributions (because they are signed by online keys), but that model does not protect distributions in the event that PyPI is compromised. In the minimum security model, attackers who have compromised the signing keys stored on PyPI Infrastructure may sign for malicious distributions. The maximum security model, -described in this PEP, retains the benefits of PEP 458 (e.g., immediate +described in this PEP, retains the benefits of :pep:`458` (e.g., immediate availability of distributions that are uploaded to PyPI), but additionally ensures that end-users are not at risk of installing forged software if PyPI is compromised. This PEP requires some changes to the PyPI infrastructure, and some suggested changes for developers who wish to participate in end-to-end signing. These -changes include updating the metadata layout from PEP 458 to include delegations +changes include updating the metadata layout from :pep:`458` to include delegations to developer keys, adding a process to register developer keys with PyPI, and a change in the upload workflow for developers who take advantage of end-to-end signing. All of these changes are described in detail later in this PEP. Package @@ -40,10 +39,10 @@ managers that wish to take advantage of end-to-end signing do not need to do any additional work beyond what is required to consume metadata described in PEP 458. -This PEP discusses the changes made to PEP 458 but excludes its informational +This PEP discusses the changes made to :pep:`458` but excludes its informational elements to primarily focus on the maximum security model. For example, an -overview of The Update Framework or the basic mechanisms in PEP 458 are not -covered here. The changes to PEP 458 include modifications to the snapshot +overview of The Update Framework or the basic mechanisms in :pep:`458` are not +covered here. The changes to :pep:`458` include modifications to the snapshot process, key compromise analysis, auditing snapshots, and the steps that should be taken in the event of a PyPI compromise. The signing and key management process that PyPI MAY RECOMMEND is discussed but not strictly defined. How the @@ -58,7 +57,7 @@ PEP Status The community discussed this PEP from 2014 to 2018. Due to the amount of work required to implement this PEP, discussion was deferred until -after approval for the precursor step in PEP 458. As of mid-2020 PEP +after approval for the precursor step in :pep:`458`. As of mid-2020 PEP 458 is approved and implementation is in progress, and the PEP authors aim to gain approval so they can secure appropriate funding for implementation. @@ -67,23 +66,23 @@ implementation. Rationale ========= -PEP 458 [1]_ proposes how PyPI should be integrated with The Update Framework +:pep:`458` proposes how PyPI should be integrated with The Update Framework (TUF) [2]_. It explains how modern package managers like pip can be made more secure, and the types of attacks that can be prevented if PyPI is modified on the server side to include TUF metadata. Package managers can reference the TUF metadata available on PyPI to download distributions more securely. -PEP 458 also describes the metadata layout of the PyPI repository and employs +:pep:`458` also describes the metadata layout of the PyPI repository and employs the minimum security model, which supports continuous delivery of projects and uses online cryptographic keys to sign the distributions uploaded by developers. Although the minimum security model guards against most attacks on -software updaters [5]_ [7]_, such as mix-and-match and extraneous dependencies +software updaters [5]_ [6]_, such as mix-and-match and extraneous dependencies attacks, it can be improved to support end-to-end signing and to prohibit forged distributions in the event that PyPI is compromised. -PEP 480 builds on PEP 458 by adding support for developer signing, and +:pep:`480` builds on :pep:`458` by adding support for developer signing, and reducing the reliance on online keys to prevent malicious distributions. -The main strength of PEP 458 and the minimum security model is the automated +The main strength of :pep:`458` and the minimum security model is the automated and simplified release process: developers may upload distributions and then have PyPI sign for their distributions. Much of the release process is handled in an automated fashion by online roles and this approach requires storing @@ -122,13 +121,11 @@ Definitions The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in RFC `2119`__. - -__ http://www.ietf.org/rfc/rfc2119.txt +interpreted as described in :rfc:`2119`. This PEP focuses on integrating TUF with PyPI; however, the reader is encouraged to read about TUF's design principles [2]_. It is also RECOMMENDED -that the reader be familiar with the TUF specification [3]_, and PEP 458 [1]_ +that the reader be familiar with the TUF specification [3]_, and :pep:`458` (which this PEP is extending). The following terms used in this PEP are defined in the Python Packaging @@ -200,7 +197,7 @@ developer roles are now supported and that three new delegated roles exist: *claimed*, *recently-claimed*, and *unclaimed*. The *bins* role from the minimum security model has been renamed *unclaimed* and can contain any projects that have not been added to *claimed*. The *unclaimed* role functions -just as before (i.e., as explained in PEP 458, projects added to this role are +just as before (i.e., as explained in :pep:`458`, projects added to this role are signed by PyPI with an online key). Offline keys provided by developers ensure the strength of the maximum security model over the minimum model. Although the minimum security model supports continuous delivery of projects, all @@ -209,6 +206,7 @@ packages in the minimum security model, but not in the maximum model, without also compromising a developer's key. .. image:: pep-0480-1.png + :class: invert-in-dark-mode Figure 1: An overview of the metadata layout in the maximum security model. The maximum security model supports continuous delivery and survivable key @@ -240,11 +238,13 @@ downloaded by clients. PyPI is trusted to make uploaded projects available to clients (PyPI signs the metadata for this part of the process), and developers sign the distributions that they upload to PyPI. -In order to delegate trust to a project, developers are required to submit a -public key to PyPI. PyPI takes the project's public key and adds it to parent +In order to delegate trust to a project, developers are required to submit at +least one public key to PyPI. Developers may submit multiple public keys for +the same project (for example, one key for each maintainer of the project). +PyPI takes all of the project's public keys and adds them to parent metadata that PyPI then signs. After the initial trust is established, developers are required to sign distributions that they upload to PyPI using -the public key's corresponding private key. The signed TUF metadata that +at least one public key's corresponding private key. The signed TUF metadata that developers upload to PyPI includes information like the distribution's file size and hash, which package managers use to verify distributions that are downloaded. @@ -297,12 +297,12 @@ The package manager (pip) shipped with CPython MUST work on non-CPython interpreters and cannot have dependencies that have to be compiled (i.e., the PyPI+TUF integration MUST NOT require compilation of C extensions in order to verify cryptographic signatures). Verification of signatures MUST be done in -Python, and verifying RSA [11]_ signatures in pure-Python may be impractical due +Python, and verifying RSA [8]_ signatures in pure-Python may be impractical due to speed. Therefore, PyPI MAY use the `Ed25519`__ signature scheme. __ http://ed25519.cr.yp.to/ -Ed25519 [12]_ is a public-key signature system that uses small cryptographic +Ed25519 [9]_ is a public-key signature system that uses small cryptographic signatures and keys. A `pure-Python implementation`__ of the Ed25519 signature scheme is available. Verification of Ed25519 signatures is fast even when performed in Python. @@ -369,7 +369,8 @@ A default, PyPI-mediated key management and package signing solution that is `transparent`__ to developers and does not require a key escrow (sharing of encrypted private keys with PyPI) is RECOMMENDED for the signing tools. Additionally, the signing tools SHOULD circumvent the sharing of private keys -across multiple machines of each developer. +across multiple machines of each developer. This means that the key management +solution SHOULD support multiple keys for each project. __ https://en.wikipedia.org/wiki/Transparency_%28human%E2%80%93computer_interaction%29 @@ -381,6 +382,8 @@ follow to upload a distribution to PyPI: 3. Optional: Add a new identity to the developer's PyPI user account from a second machine (after a password prompt). 4. Upload project. +5. Optional: Other maintainers associated with the project may log in and + enter a secondary password to add their identity to the project. Step 1 is the normal procedure followed by developers to `register a PyPI project`__. @@ -393,7 +396,7 @@ to PyPI, and signs the TUF metadata that is generated for the distribution. Optionally adding a new identity from a second machine, by simply entering a password, in step 3 also generates an encrypted private key file and uploads an Ed25519 public key to PyPI. Separate identities MAY be created to allow a -developer, or other project maintainers, to sign releases on multiple machines. +developer, to sign releases on multiple machines. An existing verified identity (its public key is contained in project metadata or has been uploaded to PyPI) signs for new identities. By default, project metadata has a signature threshold of "1" and other verified identities may @@ -403,11 +406,15 @@ Step 4 uploads the distribution file and TUF metadata to PyPI. The "Snapshot Process" section discusses in detail the procedure followed by developers to upload a distribution to PyPI. +Step 5 allows other maintainers to generate an encrypted key file, in a similar +manner to step 2. These keys SHOULD be uploaded to PyPI and added to the TUF +metadata. This key MAY be used to upload future releases of the project. + Generation of cryptographic files and signatures is transparent to the developers in the default case: developers need not be aware that packages are -automatically signed. However, the signing tools should be flexible; a single -project key may also be shared between multiple machines if manual key -management is preferred (e.g., ssh-copy-id). +automatically signed. However, the signing tools should be flexible; developers +may want to generate their own keys and handle the key management themselves. +In this case, the developers may simply upload their public key(s) to PyPI. The `repository`__ and `developer`__ TUF tools currently support all of the recommendations previously mentioned, except for the automated signing @@ -719,7 +726,7 @@ attacks, or metadata inconsistency attacks. Table 1: Attacks that are possible by compromising certain combinations of role keys. In `September 2013`__, it was shown how the latest version (at the time) of pip was susceptible to these attacks and how TUF could protect users against -them [8]_. Roles signed by offline keys are in **bold**. +them [7]_. Roles signed by offline keys are in **bold**. __ https://mail.python.org/pipermail/distutils-sig/2013-September/022755.html @@ -872,18 +879,14 @@ an online key. References ========== -.. [1] https://www.python.org/dev/peps/pep-0458/ .. [2] https://theupdateframework.io/papers/survivable-key-compromise-ccs2010.pdf -.. [3] https://github.com/theupdateframework/tuf/blob/develop/docs/tuf-spec.txt -.. [4] https://packaging.python.org/glossary +.. [3] https://theupdateframework.github.io/specification/latest/index.html +.. [4] https://packaging.python.org/en/latest/glossary/ .. [5] https://github.com/theupdateframework/pip/wiki/Attacks-on-software-repositories -.. [6] https://mail.python.org/pipermail/distutils-sig/2013-September/022773.html -.. [7] https://theupdateframework.io/papers/attacks-on-package-managers-ccs2008.pdf -.. [8] https://mail.python.org/pipermail/distutils-sig/2013-September/022755.html -.. [9] https://pypi.python.org/security -.. [10] https://mail.python.org/pipermail/distutils-sig/2013-August/022154.html -.. [11] https://en.wikipedia.org/wiki/RSA_%28algorithm%29 -.. [12] http://ed25519.cr.yp.to/ +.. [6] https://theupdateframework.io/papers/attacks-on-package-managers-ccs2008.pdf +.. [7] https://mail.python.org/pipermail/distutils-sig/2013-September/022755.html +.. [8] https://en.wikipedia.org/wiki/RSA_(cryptosystem) +.. [9] https://ed25519.cr.yp.to/ Acknowledgements diff --git a/pep-0481.txt b/peps/pep-0481.rst similarity index 97% rename from pep-0481.txt rename to peps/pep-0481.rst index 6bd6ede5f..ff80b0379 100644 --- a/pep-0481.txt +++ b/peps/pep-0481.rst @@ -1,7 +1,5 @@ PEP: 481 Title: Migrate CPython to Git, Github, and Phabricator -Version: $Revision$ -Last-Modified: $Date$ Author: Donald Stufft Status: Withdrawn Type: Process @@ -14,12 +12,12 @@ Abstract ======== .. note:: This PEP has been withdrawn, if you're looking for the PEP - documenting the move to Github, please refer to PEP 512. + documenting the move to Github, please refer to :pep:`512`. This PEP proposes migrating the repository hosting of CPython and the supporting repositories to Git and Github. It also proposes adding Phabricator as an alternative to Github Pull Requests to handle reviewing changes. This -particular PEP is offered as an alternative to PEP 474 and PEP 462 which aims +particular PEP is offered as an alternative to :pep:`474` and :pep:`462` which aims to achieve the same overall benefits but restricts itself to tools that support Mercurial and are completely Open Source. @@ -345,22 +343,11 @@ workflow less reusable with other projects. References ========== -.. [#openhub-stats] `Open Hub Statistics ` -.. [#hg-git] `Hg-Git mercurial plugin ` +.. [#openhub-stats] `Open Hub Statistics `_ +.. [#hg-git] `Hg-Git mercurial plugin `_ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0482.txt b/peps/pep-0482.rst similarity index 69% rename from pep-0482.txt rename to peps/pep-0482.rst index b80ca3ed9..6c0dc550a 100644 --- a/pep-0482.txt +++ b/peps/pep-0482.rst @@ -3,9 +3,10 @@ Title: Literature Overview for Type Hints Version: $Revision$ Last-Modified: $Date$ Author: Ɓukasz Langa -Discussions-To: Python-Ideas +Discussions-To: python-ideas@python.org Status: Final Type: Informational +Topic: Typing Content-Type: text/x-rst Created: 08-Jan-2015 Post-History: @@ -15,7 +16,7 @@ Abstract ======== This PEP is one of three related to type hinting. This PEP gives a -literature overview of related work. The main spec is PEP 484. +literature overview of related work. The main spec is :pep:`484`. Existing Approaches for Python @@ -25,36 +26,47 @@ Existing Approaches for Python mypy ---- -(This section is a stub, since mypy [mypy]_ is essentially what we're +(This section is a stub, since `mypy`__ is essentially what we're proposing.) +__ https://mypy-lang.org + Reticulated Python ------------------ -Reticulated Python [reticulated]_ by Michael Vitousek is an example of +`Reticulated Python`__ by Michael Vitousek is an example of a slightly different approach to gradual typing for Python. It is -described in an actual academic paper [reticulated-paper]_ written by +described in an actual `academic paper`__ written by Vitousek with Jeremy Siek and Jim Baker (the latter of Jython fame). +__ https://github.com/mvitousek/reticulated +__ http://wphomes.soic.indiana.edu/jsiek/files/2014/03/retic-python.pdf PyCharm ------- PyCharm by JetBrains has been providing a way to specify and check -types for about four years. The type system suggested by PyCharm -[pycharm]_ grew from simple class types to tuple types, generic types, +types for about four years. The type system suggested by PyCharm__ +grew from simple class types to tuple types, generic types, function types, etc. based on feedback of many users who shared their experience of using type hints in their code. +__ https://github.com/JetBrains/python-skeletons#types Others ------ -TBD: Add sections on pyflakes [pyflakes]_, pylint [pylint]_, numpy -[numpy]_, Argument Clinic [argumentclinic]_, pytypedecl [pytypedecl]_, -numba [numba]_, obiwan [obiwan]_. +TBD: Add sections on pyflakes__, pylint__, numpy__, +`Argument Clinic`__, pytypedecl__, numba__, obiwan__. +__ https://github.com/pyflakes/pyflakes/ +__ https://www.pylint.org +__ https://www.numpy.org +__ https://docs.python.org/3/howto/clinic.html +__ https://github.com/google/pytypedecl +__ https://numba.pydata.org +__ https://pypi.org/project/obiwan Existing Approaches in Other Languages ====================================== @@ -62,11 +74,13 @@ Existing Approaches in Other Languages ActionScript ------------ -ActionScript [actionscript]_ is a class-based, single inheritance, +ActionScript__ is a class-based, single inheritance, object-oriented superset of ECMAScript. It supports inferfaces and strong runtime-checked static typing. Compilation supports a “strict dialect” where type mismatches are reported at compile-time. +__ https://livedocs.adobe.com/specs/actionscript/3/ + Example code with types:: package { @@ -94,10 +108,12 @@ Example code with types:: Dart ---- -Dart [dart]_ is a class-based, single inheritance, object-oriented +Dart__ is a class-based, single inheritance, object-oriented language with C-style syntax. It supports interfaces, abstract classes, reified generics, and optional typing. +__ https://www.dartlang.org + Types are inferred when possible. The runtime differentiates between two modes of execution: *checked mode* aimed for development (catching type errors at runtime) and *production mode* recommended for speed execution @@ -120,10 +136,12 @@ Example code with types:: Hack ---- -Hack [hack]_ is a programming language that interoperates seamlessly +Hack__ is a programming language that interoperates seamlessly with PHP. It provides opt-in static type checking, type aliasing, generics, nullable types, and lambdas. +__ https://hacklang.org + Example code with types:: , Ivan Levkivskyi -Discussions-To: Python-Ideas +Discussions-To: python-ideas@python.org Status: Final Type: Informational +Topic: Typing Content-Type: text/x-rst Created: 19-Dec-2014 Post-History: @@ -14,7 +15,7 @@ Post-History: Abstract ======== -This PEP lays out the theory referenced by PEP 484. +This PEP lays out the theory referenced by :pep:`484`. Introduction @@ -38,10 +39,10 @@ Notational conventions ``ti`` or ``tj`` to refer to "any of ``t1``, ``t2``, etc." - ``T``, ``U`` etc. are type variables (defined with ``TypeVar()``, see below). - Objects, classes defined with a class statement, and instances are - denoted using standard PEP 8 conventions. + denoted using standard :pep:`8` conventions. - the symbol ``==`` applied to types in the context of this PEP means that two expressions represent the same type. -- Note that PEP 484 makes a distinction between types and classes +- Note that :pep:`484` makes a distinction between types and classes (a type is a concept for the type checker, while a class is a runtime concept). In this PEP we clarify this distinction but avoid unnecessary strictness to allow more @@ -81,7 +82,7 @@ There are several ways to define a particular type: It is important for the user to be able to define types in a form that can be understood by type checkers. The goal of this PEP is to propose such a systematic way of defining types -for type annotations of variables and functions using PEP 3107 syntax. +for type annotations of variables and functions using :pep:`3107` syntax. These annotations can be used to avoid many kind of bugs, for documentation purposes, or maybe even to increase speed of program execution. Here we only focus on avoiding bugs by using a static type checker. @@ -249,8 +250,8 @@ from building blocks described below, and are used by static type checkers. Every class is a type as discussed above. But it is tricky and error prone to implement a class that exactly represents -semantics of a given type, and it is not a goal of PEP 484. -*The static types described in PEP 484, should not be confused with +semantics of a given type, and it is not a goal of :pep:`484`. +*The static types described in* :pep:`484` *should not be confused with the runtime classes.* Examples: - ``int`` is a class and a type. @@ -441,9 +442,9 @@ replaced by the most-derived base class among ``t1``, etc. Examples: - Function type annotation with a constrained type variable:: - S = TypeVar('S', str, bytes) + AnyStr = TypeVar('AnyStr', str, bytes) - def longest(first: S, second: S) -> S: + def longest(first: AnyStr, second: AnyStr) -> AnyStr: return first if len(first) >= len(second) else second result = longest('a', 'abc') # The inferred type for result is str @@ -486,7 +487,7 @@ replaced by the most-derived base class among ``t1``, etc. Examples: Note that the type checker will reject this function:: def concat(first: U, second: U) -> U: - return x + y # Error: can't concatenate str and bytes + return first + second # Error: can't concatenate str and bytes For such cases where parameters could change their types only simultaneously one should use constrained type variables. @@ -712,7 +713,7 @@ so that a type checker will also find that, e.g., ``Derived[Manager]`` is a subtype of ``Base[Employee]``. For more information on type variables, generic types, and variance, -see PEP 484, the `mypy docs on +see :pep:`484`, the `mypy docs on generics `_, and `Wikipedia `_. @@ -760,7 +761,7 @@ are still controversial or not fully specified.) zork = cast(Any, frobozz()) -- Other things, e.g. overloading and stub modules, see PEP 484. +- Other things, e.g. overloading and stub modules, see :pep:`484`. Predefined generic types and Protocols in typing.py diff --git a/pep-0484.txt b/peps/pep-0484.rst similarity index 96% rename from pep-0484.txt rename to peps/pep-0484.rst index 1980e6612..b3c24d5ab 100644 --- a/pep-0484.txt +++ b/peps/pep-0484.rst @@ -4,20 +4,21 @@ Version: $Revision$ Last-Modified: $Date$ Author: Guido van Rossum , Jukka Lehtosalo , Ɓukasz Langa BDFL-Delegate: Mark Shannon -Discussions-To: Python-Dev -Status: Provisional +Discussions-To: python-dev@python.org +Status: Final Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 29-Sep-2014 Python-Version: 3.5 -Post-History: 16-Jan-2015,20-Mar-2015,17-Apr-2015,20-May-2015,22-May-2015 +Post-History: 16-Jan-2015, 20-Mar-2015, 17-Apr-2015, 20-May-2015, 22-May-2015 Resolution: https://mail.python.org/pipermail/python-dev/2015-May/140104.html Abstract ======== -PEP 3107 introduced syntax for function annotations, but the semantics +:pep:`3107` introduced syntax for function annotations, but the semantics were deliberately left undefined. There has now been enough 3rd party usage for static type analysis that the community would benefit from a standard vocabulary and baseline tools within the standard library. @@ -29,7 +30,7 @@ where annotations are not available. Note that this PEP still explicitly does NOT prevent other uses of annotations, nor does it require (or forbid) any particular processing of annotations, even when they conform to this specification. It -simply enables better coordination, as PEP 333 did for web frameworks. +simply enables better coordination, as :pep:`333` did for web frameworks. For example, here is a simple function whose argument and return type are declared in the annotations:: @@ -46,7 +47,7 @@ Essentially, such a type checker acts as a very powerful linter. a similar checker at run time for Design By Contract enforcement or JIT optimization, those tools are not yet as mature.) -The proposal is strongly inspired by mypy [mypy]_. For example, the +The proposal is strongly inspired by `mypy `_. For example, the type "sequence of integers" can be written as ``Sequence[int]``. The square brackets mean that no new syntax needs to be added to the language. The example here uses a custom type ``Sequence``, imported @@ -57,19 +58,19 @@ works at runtime by implementing ``__getitem__()`` in the metaclass The type system supports unions, generic types, and a special type named ``Any`` which is consistent with (i.e. assignable to and from) all types. This latter feature is taken from the idea of gradual typing. -Gradual typing and the full type system are explained in PEP 483. +Gradual typing and the full type system are explained in :pep:`483`. Other approaches from which we have borrowed or to which ours can be -compared and contrasted are described in PEP 482. +compared and contrasted are described in :pep:`482`. Rationale and Goals =================== -PEP 3107 added support for arbitrary annotations on parts of a +:pep:`3107` added support for arbitrary annotations on parts of a function definition. Although no meaning was assigned to annotations -then, there has always been an implicit goal to use them for type -hinting [gvr-artima]_, which is listed as the first possible use case +then, there has always been an `implicit goal to use them for type +hinting `_, which is listed as the first possible use case in said PEP. This PEP aims to provide a standard syntax for type annotations, @@ -138,7 +139,7 @@ decorators ``@property``, ``@staticmethod`` and ``@classmethod``. Type Definition Syntax ====================== -The syntax leverages PEP 3107-style annotations with a number of +The syntax leverages :pep:`3107`-style annotations with a number of extensions described in sections below. In its basic form, type hinting is used by filling function annotation slots with classes:: @@ -760,8 +761,8 @@ to the list, which would violate the variable's type in the caller. It turns out such an argument acts *contravariantly*, whereas the intuitive answer (which is correct in case the function doesn't mutate its argument!) requires the argument to act *covariantly*. A longer -introduction to these concepts can be found on Wikipedia -[wiki-variance]_ and in PEP 483; here we just show how to control +introduction to these concepts can be found on `Wikipedia +`_ and in :pep:`483`; here we just show how to control a type checker's behavior. By default generic types are considered *invariant* in all type variables, @@ -840,7 +841,7 @@ while the following is prohibited:: The numeric tower ----------------- -PEP 3141 defines Python's numeric tower, and the stdlib module +:pep:`3141` defines Python's numeric tower, and the stdlib module ``numbers`` implements the corresponding ABCs (``Number``, ``Complex``, ``Real``, ``Rational`` and ``Integral``). There are some issues with these ABCs, but the built-in concrete numeric classes @@ -1381,7 +1382,7 @@ return_type]`` provided by ``typing.py`` module:: res = yield round(res) return 'OK' -Coroutines introduced in PEP 492 are annotated with the same syntax as +Coroutines introduced in :pep:`492` are annotated with the same syntax as ordinary functions. However, the return type annotation corresponds to the type of ``await`` expression, not to the coroutine type:: @@ -1473,7 +1474,7 @@ Examples of type comments on ``with`` and ``for`` statements:: ... In stubs it may be useful to declare the existence of a variable -without giving it an initial value. This can be done using PEP 526 +without giving it an initial value. This can be done using :pep:`526` variable annotation syntax:: from typing import IO @@ -1517,7 +1518,7 @@ other comments and linting markers: If type hinting proves useful in general, a syntax for typing variables may be provided in a future Python version. (**UPDATE**: This syntax -was added in Python 3.6 through PEP 526.) +was added in Python 3.6 through :pep:`526`.) Casts ===== @@ -1557,7 +1558,7 @@ errors by creating simple classes. For example:: class UserId(int): pass - get_by_user_id(user_id: UserId): + def get_by_user_id(user_id: UserId): ... However, this approach introduces a runtime overhead. To avoid this, @@ -1660,7 +1661,7 @@ Additional notes on stub files: exported. (This makes it easier to re-export all objects from a given module that may vary by Python version.) -* Just like in normal Python files [importdocs]_, submodules +* Just like in `normal Python files `_, submodules automatically become exported attributes of their parent module when imported. For example, if the ``spam`` package has the following directory structure:: @@ -1764,7 +1765,7 @@ implementation using this syntax, its implementation would require using ``sys._getframe()``, which is frowned upon. Also, designing and implementing an efficient multiple dispatch mechanism is hard, which is why previous attempts were abandoned in favor of -``functools.singledispatch()``. (See PEP 443, especially its section +``functools.singledispatch()``. (See :pep:`443`, especially its section "Alternative approaches".) In the future we may come up with a satisfactory multiple dispatch design, but we don't want such a design to be constrained by the overloading syntax defined for type hints in @@ -1853,7 +1854,7 @@ Stub file package authors might use the following snippet in ``setup.py``:: (*UPDATE:* As of June 2018 the recommended way to distribute type hints for third-party packages has changed -- in addition to typeshed (see the next section) there is now a standard for distributing type -hints, PEP 561. It supports separately installable packages containing +hints, :pep:`561`. It supports separately installable packages containing stubs, stub files included in the same distribution as the executable code of a package, and inline type hints, the latter two options enabled by including a file named ``py.typed`` in the package.) @@ -1861,8 +1862,8 @@ enabled by including a file named ``py.typed`` in the package.) The Typeshed Repo ----------------- -There is a shared repository where useful stubs are being collected -[typeshed]_. Policies regarding the stubs collected here will be +There is a `shared repository `_ where useful stubs are being +collected. Policies regarding the stubs collected here will be decided separately and reported in the repo's documentation. Note that stubs for a given package will not be included here if the package owners have specifically requested that they be omitted. @@ -2238,7 +2239,7 @@ Scala also use square brackets. What about existing uses of annotations? ---------------------------------------- -One line of argument points out that PEP 3107 explicitly supports +One line of argument points out that :pep:`3107` explicitly supports the use of arbitrary expressions in function annotations. The new proposal is then considered incompatible with the specification of PEP 3107. @@ -2252,7 +2253,7 @@ We do hope that type hints will eventually become the sole use for annotations, but this will require additional discussion and a deprecation period after the initial roll-out of the typing module with Python 3.5. The current PEP will have provisional status (see -PEP 411) until Python 3.6 is released. The fastest conceivable scheme +:pep:`411`) until Python 3.6 is released. The fastest conceivable scheme would introduce silent deprecation of non-type-hint annotations in 3.6, full deprecation in 3.7, and declare type hints as the only allowed use of annotations in Python 3.8. This should give authors of @@ -2262,7 +2263,7 @@ approach, even if type hints become an overnight success. (*UPDATE:* As of fall 2017, the timeline for the end of provisional status for this PEP and for the ``typing.py`` module has changed, and so has the deprecation schedule for other uses of annotations. For -the updated schedule see PEP 563.) +the updated schedule see :pep:`563`.) Another possible outcome would be that type hints will eventually become the default meaning for annotations, but that there will always @@ -2299,7 +2300,7 @@ a problem: "use" here means "look up at runtime", and with most "forward" references there is no problem in ensuring that a name is defined before the function using it is called. -The problem with type hints is that annotations (per PEP 3107, and +The problem with type hints is that annotations (per :pep:`3107`, and similar to default values) are evaluated at the time a function is defined, and thus any names used in an annotation must be already defined when the function is being defined. A common scenario is a @@ -2345,7 +2346,7 @@ Such a ``__future__`` import statement may be proposed in a separate PEP. (*UPDATE:* That ``__future__`` import statement and its consequences -are discussed in PEP 563.) +are discussed in :pep:`563`.) The double colon @@ -2365,8 +2366,9 @@ evaluation. There are several things wrong with this idea, however. is unheard of in English, and in other languages (e.g. C++) it is used as a scoping operator, which is a very different beast. In contrast, the single colon for type hints reads naturally -- and no - wonder, since it was carefully designed for this purpose (the idea - long predates PEP 3107 [gvr-artima]_). It is also used in the same + wonder, since it was carefully designed for this purpose + (`the idea `_ + long predates :pep:`3107`). It is also used in the same fashion in other languages from Pascal to Swift. * What would you do for return type annotations? @@ -2397,8 +2399,8 @@ evaluation. There are several things wrong with this idea, however. Other forms of new syntax ------------------------- -A few other forms of alternative syntax have been proposed, e.g. the -introduction of a ``where`` keyword [roberge]_, and Cobra-inspired +A few other forms of alternative syntax have been proposed, e.g. `the +introduction `_ of a ``where`` keyword, and Cobra-inspired ``requires`` clauses. But these all share a problem with the double colon: they won't work for earlier versions of Python 3. The same would apply to a new ``__future__`` import. @@ -2410,7 +2412,7 @@ The ideas put forward include: * A decorator, e.g. ``@typehints(name=str, returns=str)``. This could work, but it's pretty verbose (an extra line, and the argument names - must be repeated), and a far cry in elegance from the PEP 3107 + must be repeated), and a far cry in elegance from the :pep:`3107` notation. * Stub files. We do want stub files, but they are primarily useful @@ -2433,12 +2435,12 @@ problem would that solve? It would just be procrastination. PEP Development Process ======================= -A live draft for this PEP lives on GitHub [github]_. There is also an -issue tracker [issues]_, where much of the technical discussion takes +A live draft for this PEP lives on `GitHub `_. There is also an +`issue tracker `_, where much of the technical discussion takes place. The draft on GitHub is updated regularly in small increments. The -official PEPS repo [peps_] is (usually) only updated when a new draft +`official PEPS repo `_ is (usually) only updated when a new draft is posted to python-dev. @@ -2451,46 +2453,37 @@ Vitousek, Andrey Vlasovskikh, Radomir Dopieralski, Peter Ludemann, and the BDFL-Delegate, Mark Shannon. Influences include existing languages, libraries and frameworks -mentioned in PEP 482. Many thanks to their creators, in alphabetical +mentioned in :pep:`482`. Many thanks to their creators, in alphabetical order: Stefan Behnel, William Edwards, Greg Ewing, Larry Hastings, Anders Hejlsberg, Alok Menghrajani, Travis E. Oliphant, Joe Pamer, Raoul-Gabriel Urma, and Julien Verlaguet. -References -========== - -.. [mypy] +.. _mypy: http://mypy-lang.org -.. [gvr-artima] - http://www.artima.com/weblogs/viewpost.jsp?thread=85551 +.. _gvr-artima: + https://www.artima.com/weblogs/viewpost.jsp?thread=85551 -.. [wiki-variance] - http://en.wikipedia.org/wiki/Covariance_and_contravariance_%28computer_science%29 +.. _wiki-variance: + https://en.wikipedia.org/wiki/Covariance_and_contravariance_%28computer_science%29 -.. [typeshed] - https://github.com/python/typeshed/ +.. _typeshed: + https://github.com/python/typeshed -.. [pyflakes] - https://github.com/pyflakes/pyflakes/ +.. _roberge: + https://aroberge.blogspot.com/2015/01/type-hinting-in-python-focus-on.html -.. [pylint] - http://www.pylint.org - -.. [roberge] - http://aroberge.blogspot.com/2015/01/type-hinting-in-python-focus-on.html - -.. [github] +.. _github: https://github.com/python/typing -.. [issues] +.. _issues: https://github.com/python/typing/issues -.. [peps] +.. _peps: https://hg.python.org/peps/file/tip/pep-0484.txt -.. [importdocs] +.. _importdocs: https://docs.python.org/3/reference/import.html#submodules @@ -2498,14 +2491,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0485.txt b/peps/pep-0485.rst similarity index 99% rename from pep-0485.txt rename to peps/pep-0485.rst index 09fb7dcf1..dd2bc3451 100644 --- a/pep-0485.txt +++ b/peps/pep-0485.rst @@ -2,7 +2,7 @@ PEP: 485 Title: A Function for testing approximate equality Version: $Revision$ Last-Modified: $Date$ -Author: Christopher Barker +Author: Christopher Barker Status: Final Type: Standards Track Content-Type: text/x-rst @@ -230,7 +230,7 @@ consideration, for instance: 4) The absolute value of the arithmetic mean of the two -These leads to the following possibilities for determining if two +These lead to the following possibilities for determining if two values, a and b, are close to each other. 1) ``abs(a-b) <= tol*abs(a)`` @@ -239,7 +239,7 @@ values, a and b, are close to each other. 3) ``abs(a-b) <= tol * min( abs(a), abs(b) )`` -4) ``abs(a-b) <= tol * (a + b)/2`` +4) ``abs(a-b) <= tol * abs(a + b)/2`` NOTE: (2) and (3) can also be written as: @@ -590,7 +590,7 @@ No absolute tolerance Given the issues with comparing to zero, another possibility would have been to only provide a relative tolerance, and let comparison to zero fail. In this case, the user would need to do a simple absolute -test: `abs(val) < zero_tol` in the case where the comparison involved +test: ``abs(val) < zero_tol`` in the case where the comparison involved zero. However, this would not allow the same call to be used for a sequence diff --git a/pep-0486.txt b/peps/pep-0486.rst similarity index 97% rename from pep-0486.txt rename to peps/pep-0486.rst index c36e4b3f4..a785a8f2a 100644 --- a/pep-0486.txt +++ b/peps/pep-0486.rst @@ -16,7 +16,7 @@ Abstract ======== The Windows installers for Python include a launcher that locates the -correct Python interpreter to run (see PEP 397). However, the +correct Python interpreter to run (see :pep:`397`). However, the launcher is not aware of virtual environments (virtualenv [1]_ or PEP 405 based), and so cannot be used to run commands from the active virtualenv. @@ -74,7 +74,7 @@ the system (i.e., when no specific version flags such as ``py -2.7`` are used) and if present, run the Python interpreter for the virtualenv rather than the default system Python. -The "default" Python interpreter referred to above is (as per PEP 397) +The "default" Python interpreter referred to above is (as per :pep:`397`) either the latest version of Python installed on the system, or a version configured via the ``py.ini`` configuration file. When the user specifies an explicit Python version on the command line, this diff --git a/pep-0487.txt b/peps/pep-0487.rst similarity index 96% rename from pep-0487.txt rename to peps/pep-0487.rst index 558afb3b8..4b63ad369 100644 --- a/pep-0487.txt +++ b/peps/pep-0487.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 27-Feb-2015 Python-Version: 3.6 -Post-History: 27-Feb-2015, 5-Feb-2016, 24-Jun-2016, 2-Jul-2016, 13-Jul-2016 +Post-History: 27-Feb-2015, 05-Feb-2016, 24-Jun-2016, 02-Jul-2016, 13-Jul-2016 Replaces: 422 Resolution: https://mail.python.org/pipermail/python-dev/2016-July/145629.html @@ -62,7 +62,7 @@ into the class creation: 2. upon class creation, a ``__set_name__`` hook is called on all the attribute (descriptors) defined in the class, and -The third category is the topic of another PEP, PEP 520. +The third category is the topic of another PEP, :pep:`520`. As an example, the first use case looks as follows:: @@ -381,7 +381,7 @@ Calling the hook on the class itself ------------------------------------ Adding an ``__autodecorate__`` hook that would be called on the class -itself was the proposed idea of PEP 422. Most examples work the same +itself was the proposed idea of :pep:`422`. Most examples work the same way or even better if the hook is called only on strict subclasses. In general, it is much easier to arrange to explicitly call the hook on the class in which it is defined (to opt-in to such a behavior) than to opt-out (by remember to check for @@ -434,14 +434,14 @@ to be detected anyway in order to give a useful error message. This decision was reinforced after noticing that the user experience of defining ``__prepare__`` and forgetting the ``@classmethod`` method -decorator is singularly incomprehensible (particularly since PEP 3115 +decorator is singularly incomprehensible (particularly since :pep:`3115` documents it as an ordinary method, and the current documentation doesn't explicitly say anything one way or the other). A more ``__new__``-like hook ---------------------------- -In PEP 422 the hook worked more like the ``__new__`` method than the +In :pep:`422` the hook worked more like the ``__new__`` method than the ``__init__`` method, meaning that it returned a class instead of modifying one. This allows a bit more flexibility, but at the cost of much harder implementation and undesired side effects. @@ -449,15 +449,15 @@ of much harder implementation and undesired side effects. Adding a class attribute with the attribute order ------------------------------------------------- -This got its own PEP 520. +This got its own :pep:`520`. History ======= -This used to be a competing proposal to PEP 422 by Nick Coghlan and Daniel -Urban. PEP 422 intended to achieve the same goals as this PEP, but with a -different way of implementation. In the meantime, PEP 422 has been withdrawn +This used to be a competing proposal to :pep:`422` by Nick Coghlan and Daniel +Urban. :pep:`422` intended to achieve the same goals as this PEP, but with a +different way of implementation. In the meantime, :pep:`422` has been withdrawn favouring this approach. References diff --git a/pep-0488.txt b/peps/pep-0488.rst similarity index 92% rename from pep-0488.txt rename to peps/pep-0488.rst index d8f7ef58f..4015d6721 100644 --- a/pep-0488.txt +++ b/peps/pep-0488.rst @@ -9,9 +9,9 @@ Content-Type: text/x-rst Created: 20-Feb-2015 Python-Version: 3.5 Post-History: - 2015-03-06 - 2015-03-13 - 2015-03-20 + 06-Mar-2015, + 13-Mar-2015, + 20-Mar-2015 Abstract ======== @@ -48,7 +48,7 @@ of reading PYO files, this can lead to an interpreter using a mixture of optimization levels with its code if the user was not careful to make sure all PYO files were generated using the same optimization level (typically done by blindly deleting all PYO files and then -using the `compileall` module to compile all-new PYO files [1]_). +using the ``compileall`` module to compile all-new PYO files [1]_). This issue is only compounded when people optimize Python code beyond what the interpreter natively supports, e.g., using the astoptimizer project [2]_. @@ -76,7 +76,7 @@ To eliminate the ambiguity that PYO files present, this PEP proposes eliminating the concept of PYO files and their accompanying ``.pyo`` file extension. To allow for the optimization level to be unambiguous as well as to avoid having to regenerate optimized bytecode files -needlessly in the `__pycache__` directory, the optimization level +needlessly in the ``__pycache__`` directory, the optimization level used to generate the bytecode file will be incorporated into the bytecode file name. When no optimization level is specified, the pre-PEP ``.pyc`` file name will be used (i.e., no optimization level @@ -91,7 +91,7 @@ based on the interpreter's optimization level (none, ``-O``, and Currently bytecode file names are created by ``importlib.util.cache_from_source()``, approximately using the -following expression defined by PEP 3147 [3]_, [4]_, [5]_:: +following expression defined by :pep:`3147` [3]_, [4]_:: '{name}.{cache_tag}.pyc'.format(name=module_name, cache_tag=sys.implementation.cache_tag) @@ -126,7 +126,7 @@ instead of ``importlib.cpython-35.pyo`` the file name would be Leaving out the new ``opt-`` tag when no optimization level is applied should increase backwards-compatibility. This is also more understanding of Python implementations which have no use for -optimization levels (e.g., PyPy[10]_). +optimization levels (e.g., PyPy [10]_). It should be noted that this change in no way affects the performance of import. Since the import system looks for a single bytecode file @@ -236,7 +236,7 @@ of any use. Since people typically only distribute bytecode files for code obfuscation purposes or smaller distribution size then only having to distribute a single ``.pyc`` should actually be beneficial to these use-cases. And since the magic number for bytecode files -changed in Python 3.5 to support PEP 465 there is no need to support +changed in Python 3.5 to support :pep:`465` there is no need to support pre-existing ``.pyo`` files [8]_. @@ -307,50 +307,36 @@ References ========== .. [1] The compileall module - (https://docs.python.org/3/library/compileall.html#module-compileall) + (https://docs.python.org/3.5/library/compileall.html) .. [2] The astoptimizer project - (https://pypi.python.org/pypi/astoptimizer) + (https://web.archive.org/web/20150909225454/https://pypi.python.org/pypi/astoptimizer) .. [3] ``importlib.util.cache_from_source()`` (https://docs.python.org/3.5/library/importlib.html#importlib.util.cache_from_source) .. [4] Implementation of ``importlib.util.cache_from_source()`` from CPython 3.4.3rc1 - (https://hg.python.org/cpython/file/038297948389/Lib/importlib/_bootstrap.py#l437) - -.. [5] PEP 3147, PYC Repository Directories, Warsaw - (http://www.python.org/dev/peps/pep-3147) + (https://github.com/python/cpython/blob/e55181f517bbfc875065ce86ed3e05cf0e0246fa/Lib/importlib/_bootstrap.py#L437) .. [6] The py_compile module - (https://docs.python.org/3/library/compileall.html#module-compileall) + (https://docs.python.org/3.5/library/compileall.html) .. [7] The importlib.machinery module - (https://docs.python.org/3/library/importlib.html#module-importlib.machinery) + (https://docs.python.org/3.5/library/importlib.html#module-importlib.machinery) .. [8] ``importlib.util.MAGIC_NUMBER`` - (https://docs.python.org/3/library/importlib.html#importlib.util.MAGIC_NUMBER) + (https://docs.python.org/3.5/library/importlib.html#importlib.util.MAGIC_NUMBER) .. [9] Informal poll of file name format options on Google+ - (https://plus.google.com/u/0/+BrettCannon/posts/fZynLNwHWGm) + (https://web.archive.org/web/20160925163500/https://plus.google.com/+BrettCannon/posts/fZynLNwHWGm) .. [10] The PyPy Project - (http://pypy.org/) + (https://www.pypy.org/) .. [11] Implementation of PEP 488 - (http://bugs.python.org/issue23731) + (https://github.com/python/cpython/issues/67919) Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0489.txt b/peps/pep-0489.rst similarity index 95% rename from pep-0489.txt rename to peps/pep-0489.rst index 0daf966d7..4b08d104e 100644 --- a/pep-0489.txt +++ b/peps/pep-0489.rst @@ -12,7 +12,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 11-Aug-2013 Python-Version: 3.5 -Post-History: 23-Aug-2013, 20-Feb-2015, 16-Apr-2015, 7-May-2015, 18-May-2015 +Post-History: 23-Aug-2013, 20-Feb-2015, 16-Apr-2015, 07-May-2015, 18-May-2015 Resolution: https://mail.python.org/pipermail/python-dev/2015-May/140108.html @@ -24,9 +24,9 @@ interact with the import machinery. This was last revised for Python 3.0 in PEP 3121, but did not solve all problems at the time. The goal is to solve import-related problems by bringing extension modules closer to the way Python modules behave; specifically to hook into the ModuleSpec-based loading -mechanism introduced in PEP 451. +mechanism introduced in :pep:`451`. -This proposal draws inspiration from PyType_Spec of PEP 384 to allow extension +This proposal draws inspiration from PyType_Spec of :pep:`384` to allow extension authors to only define features they need, and to allow future additions to extension module declarations. @@ -41,7 +41,7 @@ when using the new API. The proposal also allows extension modules with non-ASCII names. -Not all problems tackled in PEP 3121 are solved in this proposal. +Not all problems tackled in :pep:`3121` are solved in this proposal. In particular, problems with run-time module lookup (PyState_FindModule) are left to a future PEP. @@ -51,8 +51,8 @@ Motivation Python modules and extension modules are not being set up in the same way. For Python modules, the module object is created and set up first, then the -module code is being executed (PEP 302). -A ModuleSpec object (PEP 451) is used to hold information about the module, +module code is being executed (:pep:`302`). +A ModuleSpec object (:pep:`451`) is used to hold information about the module, and passed to the relevant hooks. For extensions (i.e. shared libraries) and built-in modules, the module @@ -76,7 +76,7 @@ Furthermore, the majority of currently existing extension modules has problems with sub-interpreter support and/or interpreter reloading, and, while it is possible with the current infrastructure to support these features, it is neither easy nor efficient. -Addressing these issues was the goal of PEP 3121, but many extensions, +Addressing these issues was the goal of :pep:`3121`, but many extensions, including some in the standard library, took the least-effort approach to porting to Python 3, leaving these issues unresolved. This PEP keeps backwards compatibility, which should reduce pressure and give @@ -119,7 +119,7 @@ object, will still be accepted, so existing code will work unchanged, including binary compatibility. The PyModuleDef structure will be changed to contain a list of slots, -similarly to PEP 384's PyType_Spec for types. +similarly to :pep:`384`'s PyType_Spec for types. To keep binary compatibility, and avoid needing to introduce a new structure (which would introduce additional supporting functions and per-module storage), the currently unused m_reload pointer of PyModuleDef will be changed to @@ -188,8 +188,8 @@ Here is an overview of how the modified importers will operate. Details such as logging or handling of errors and invalid states are left out, and C code is presented with a concise Python-like syntax. -The framework that calls the importers is explained in PEP 451 -[#pep-0451-loading]_. +The framework that calls the importers is explained in +:pep:`451#how-loading-will-work`. importlib/_bootstrap.py: @@ -347,13 +347,13 @@ The value pointer must point to a function with the following signature:: PyObject* (*PyModuleCreateFunction)(PyObject *spec, PyModuleDef *def) -The function receives a ModuleSpec instance, as defined in PEP 451, +The function receives a ModuleSpec instance, as defined in :pep:`451`, and the PyModuleDef structure. It should return a new module object, or set an error and return NULL. This function is not responsible for setting import-related attributes -specified in PEP 451 [#pep-0451-attributes]_ (such as ``__name__`` or +specified in :pep:`451#attributes` (such as ``__name__`` or ``__loader__``) on the new module. There is no requirement for the returned object to be an instance of @@ -412,7 +412,7 @@ PyModule_GetDef would fail), the execution phase is skipped. Execution slots may be specified multiple times, and are processed in the order they appear in the slots array. When using the default import machinery, they are processed after -import-related attributes specified in PEP 451 [#pep-0451-attributes]_ +import-related attributes specified in :pep:`451#attributes` (such as ``__name__`` or ``__loader__``) are set and the module is added to sys.modules. @@ -620,7 +620,7 @@ PyInit_, where is the name of the module. For module names containing non-ASCII characters, the import hook is named PyInitU_, where the name is encoded using CPython's -"punycode" encoding (Punycode [#rfc-3492]_ with a lowercase suffix), +"punycode" encoding (:rfc:`Punycode <3492>` with a lowercase suffix), with hyphens ("-") replaced by underscores ("_"). @@ -768,7 +768,7 @@ Internal functions of Python/import.c and Python/importdl.c will be removed. Possible Future Extensions ========================== -The slots mechanism, inspired by PyType_Slot from PEP 384, +The slots mechanism, inspired by PyType_Slot from :pep:`384`, allows later extensions. Some extension modules exports many constants; for example _ssl has @@ -790,7 +790,7 @@ beneficial, though.) Another possibility is providing a "main" function that would be run when the module is given to Python's -m switch. For this to work, the runpy module will need to be modified to take -advantage of ModuleSpec-based loading introduced in PEP 451. +advantage of ModuleSpec-based loading introduced in :pep:`451`. Also, it will be necessary to add a mechanism for setting up a module according to slots it wasn't originally defined with. @@ -808,13 +808,13 @@ Previous Approaches Stefan Behnel's initial proto-PEP [#stefans_protopep]_ had a "PyInit_modulename" hook that would create a module class, whose ``__init__`` would be then called to create the module. -This proposal did not correspond to the (then nonexistent) PEP 451, +This proposal did not correspond to the (then nonexistent) :pep:`451`, where module creation and initialization is broken into distinct steps. It also did not support loading an extension into pre-existing module objects. Nick Coghlan proposed "Create" and "Exec" hooks, and wrote a prototype implementation [#nicks-prototype]_. -At this time PEP 451 was still not implemented, so the prototype +At this time :pep:`451` was still not implemented, so the prototype does not use ModuleSpec. The original version of this PEP used Create and Exec hooks, and allowed @@ -834,18 +834,12 @@ exporting a definition, yielded a much simpler solution. References ========== -.. [#pep-0451-attributes] - https://www.python.org/dev/peps/pep-0451/#attributes - .. [#stefans_protopep] https://mail.python.org/pipermail/python-dev/2013-August/128087.html .. [#nicks-prototype] https://mail.python.org/pipermail/python-dev/2013-August/128101.html -.. [#rfc-3492] - http://tools.ietf.org/html/rfc3492 - .. [#gh-repo] https://github.com/encukou/cpython/commits/pep489 @@ -855,9 +849,6 @@ References .. [#findmodule-discussion] https://mail.python.org/pipermail/import-sig/2015-April/000959.html -.. [#pep-0451-loading] - https://www.python.org/dev/peps/pep-0451/#how-loading-will-work] - .. [#subinterpreter-docs] https://docs.python.org/3/c-api/init.html#sub-interpreter-support diff --git a/pep-0490.txt b/peps/pep-0490.rst similarity index 96% rename from pep-0490.txt rename to peps/pep-0490.rst index 3105f1b3a..3514860d3 100644 --- a/pep-0490.txt +++ b/peps/pep-0490.rst @@ -20,7 +20,7 @@ Rationale ========= Python 3 introduced a new killer feature: exceptions are chained by -default, PEP 3134. +default, :pep:`3134`. Example:: @@ -215,15 +215,14 @@ Appendix PEPs ---- -* `PEP 3134 -- Exception Chaining and Embedded Tracebacks - `_ (Python 3.0): +* :pep:`3134` -- Exception Chaining and Embedded Tracebacks + (Python 3.0): new ``__context__`` and ``__cause__`` attributes for exceptions -* `PEP 415 - Implement context suppression with exception attributes - `_ (Python 3.3): +* :pep:`415` -- Implement context suppression with exception attributes + (Python 3.3): ``raise exc from None`` -* `PEP 409 - Suppressing exception context - `_ - (superseded by the PEP 415) +* :pep:`409` -- Suppressing exception context + (superseded by the :pep:`415`) Python C API diff --git a/pep-0491.txt b/peps/pep-0491.rst similarity index 95% rename from pep-0491.txt rename to peps/pep-0491.rst index 71a77b99c..b4c06cac1 100644 --- a/pep-0491.txt +++ b/peps/pep-0491.rst @@ -1,11 +1,10 @@ PEP: 491 Title: The Wheel Binary Package Format 1.9 -Version: $Revision$ -Last-Modified: $Date$ Author: Daniel Holth -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Deferred Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 16-Apr-2015 @@ -19,7 +18,7 @@ re-building from source each time. A wheel is a ZIP-format archive with a specially formatted file name and the ``.whl`` extension. It contains a single distribution nearly as it -would be installed according to PEP 376 with a particular installation +would be installed according to :pep:`376` with a particular installation scheme. Simple wheels can be unpacked onto ``sys.path`` and used directly but wheels are usually installed with a specialized installer. @@ -39,11 +38,11 @@ Some specific elements to be addressed when work on this PEP is resumed in the future: - migrating the official wheel format definition to - https://packaging.python.org/specifications/ (similar to what PEP 566 did for + https://packaging.python.org/specifications/ (similar to what :pep:`566` did for https://packaging.python.org/specifications/core-metadata/) - updating the PEP itself to focus on the *changes* being made between the two versions of the format and the rationale for those changes, rather than - having to repeat all the information that is unchanged from PEP 427 + having to repeat all the information that is unchanged from :pep:`427` - clarifying that the PEP is deliberately written to allow existing installers to be compliant with the specification when using existing install scheme definitions, while also allowing the creation of new install scheme @@ -169,7 +168,7 @@ on any CPU architecture. The last three components of the filename before the extension are called "compatibility tags." The compatibility tags express the -package's basic interpreter requirements and are detailed in PEP 425. +package's basic interpreter requirements and are detailed in :pep:`425`. Escaping and Unicode '''''''''''''''''''' @@ -249,12 +248,12 @@ The .dist-info directory found at the root of sdists. #. WHEEL is the wheel metadata specific to a build of the package. #. RECORD is a list of (almost) all the files in the wheel and their - secure hashes. Unlike PEP 376, every file except RECORD, which + secure hashes. Unlike :pep:`376`, every file except RECORD, which cannot contain a hash of itself, must include its hash. The hash algorithm must be sha256 or better; specifically, md5 and sha1 are not permitted, as signed wheel files rely on the strong hashes in RECORD to validate the integrity of the archive. -#. PEP 376's INSTALLER and REQUESTED are not included in the archive. +#. :pep:`376`'s INSTALLER and REQUESTED are not included in the archive. #. RECORD.jws is used for digital signatures. It is not mentioned in RECORD. #. RECORD.p7s is allowed as a courtesy to anyone who would prefer to @@ -360,7 +359,7 @@ Signed wheel files ------------------ Wheel files include an extended RECORD that enables digital -signatures. PEP 376's RECORD is altered to include a secure hash +signatures. :pep:`376`'s RECORD is altered to include a secure hash ``digestname=urlsafe_b64encode_nopad(digest)`` (urlsafe base64 encoding with no trailing = characters) as the second column instead of an md5sum. All possible entries are hashed, including any @@ -394,10 +393,10 @@ checker only needs to establish that RECORD matches the signature. See -- http://self-issued.info/docs/draft-ietf-jose-json-web-signature.html -- http://self-issued.info/docs/draft-jones-jose-jws-json-serialization.html -- http://self-issued.info/docs/draft-ietf-jose-json-web-key.html -- http://self-issued.info/docs/draft-jones-jose-json-private-key.html +- :rfc:`7515` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-jws-json-serialization.html +- :rfc:`7517` +- https://datatracker.ietf.org/doc/html/draft-jones-jose-json-private-key.html Comparison to .egg @@ -530,13 +529,6 @@ Is it possible to import Python code directly from a wheel file? a fully installed package before accepting it as a genuine bug. -References -========== - -.. [1] PEP acceptance - (https://mail.python.org/pipermail/python-dev/2013-February/124103.html) - - Appendix ======== @@ -557,14 +549,3 @@ Copyright ========= This document has been placed into the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0492.txt b/peps/pep-0492.rst similarity index 97% rename from pep-0492.txt rename to peps/pep-0492.rst index e5122b479..2bab371de 100644 --- a/pep-0492.txt +++ b/peps/pep-0492.rst @@ -1,9 +1,7 @@ PEP: 492 Title: Coroutines with async and await syntax -Version: $Revision$ -Last-Modified: $Date$ Author: Yury Selivanov -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -112,9 +110,9 @@ This proposal introduces new syntax and semantics to enhance coroutine support in Python. This specification presumes knowledge of the implementation of -coroutines in Python (PEP 342 and PEP 380). Motivation for the syntax -changes proposed here comes from the asyncio framework (PEP 3156) and -the "Cofunctions" proposal (PEP 3152, now rejected in favor of this +coroutines in Python (:pep:`342` and :pep:`380`). Motivation for the syntax +changes proposed here comes from the asyncio framework (:pep:`3156`) and +the "Cofunctions" proposal (:pep:`3152`, now rejected in favor of this specification). From this point in this document we use the word *native coroutine* to @@ -154,7 +152,7 @@ Key properties of *coroutines*: * ``StopIteration`` exceptions are not propagated out of coroutines, and are replaced with a ``RuntimeError``. For regular generators - such behavior requires a future import (see PEP 479). + such behavior requires a future import (see :pep:`479`). * When a *native coroutine* is garbage collected, a ``RuntimeWarning`` is raised if it was never awaited on (see also @@ -175,8 +173,8 @@ in asyncio and *native coroutines* introduced by this PEP:: data = yield from read_data(db) ... -The function applies ``CO_ITERABLE_COROUTINE`` flag to generator- -function's code object, making it return a *coroutine* object. +The function applies ``CO_ITERABLE_COROUTINE`` flag to +generator-function's code object, making it return a *coroutine* object. If ``fn`` is not a *generator function*, it is wrapped. If it returns a *generator*, it will be wrapped in an *awaitable* proxy object @@ -217,7 +215,7 @@ can be one of: fundamental mechanism of how *Futures* are implemented. Since, internally, coroutines are a special kind of generators, every ``await`` is suspended by a ``yield`` somewhere down the chain of - ``await`` calls (please refer to PEP 3156 for a detailed + ``await`` calls (please refer to :pep:`3156` for a detailed explanation). To enable this behavior for coroutines, a new magic method called @@ -558,7 +556,7 @@ an asynchronous one. While this is not a very useful thing to do, the code illustrates the relationship between regular and asynchronous iterators. -:: +.. code:: python class AsyncIteratorWrapper: def __init__(self, obj): @@ -584,7 +582,7 @@ Why StopAsyncIteration? Coroutines are still based on generators internally. So, before PEP 479, there was no fundamental difference between -:: +.. code:: python def g1(): yield from fut @@ -592,17 +590,17 @@ Coroutines are still based on generators internally. So, before PEP and -:: +.. code:: python def g2(): yield from fut raise StopIteration('spam') -And since PEP 479 is accepted and enabled by default for coroutines, +And since :pep:`479` is accepted and enabled by default for coroutines, the following example will have its ``StopIteration`` wrapped into a ``RuntimeError`` -:: +.. code:: python async def a1(): await fut @@ -612,7 +610,7 @@ The only way to tell the outside code that the iteration has ended is to raise something other than ``StopIteration``. Therefore, a new built-in exception class ``StopAsyncIteration`` was added. -Moreover, with semantics from PEP 479, all ``StopIteration`` exceptions +Moreover, with semantics from :pep:`479`, all ``StopIteration`` exceptions raised in coroutines are wrapped in ``RuntimeError``. @@ -643,7 +641,7 @@ generators are treated as distinct concepts: doing so will result in a ``TypeError``. 3. *generator-based coroutines* (for asyncio code must be decorated - with ``@asyncio.coroutine``) can ``yield from`` *native coroutine + with ``@asyncio.coroutine`` [1]_) can ``yield from`` *native coroutine objects*. 4. ``inspect.isgenerator()`` and ``inspect.isgeneratorfunction()`` @@ -658,7 +656,7 @@ Coroutines are based on generators internally, thus they share the implementation. Similarly to generator objects, *coroutines* have ``throw()``, ``send()`` and ``close()`` methods. ``StopIteration`` and ``GeneratorExit`` play the same role for coroutines (although -PEP 479 is enabled by default for coroutines). See PEP 342, PEP 380, +:pep:`479` is enabled by default for coroutines). See :pep:`342`, :pep:`380`, and Python Documentation [11]_ for details. ``throw()``, ``send()`` methods for *coroutines* are used to push @@ -943,7 +941,7 @@ Design Considerations PEP 3152 -------- -PEP 3152 by Gregory Ewing proposes a different mechanism for coroutines +:pep:`3152` by Gregory Ewing proposes a different mechanism for coroutines (called "cofunctions"). Some key points: 1. A new keyword ``codef`` to declare a *cofunction*. *Cofunction* is @@ -980,7 +978,7 @@ Differences from this proposal: ``async def``). It is possible to simply write ``await future``, whereas ``cocall`` always requires parentheses. -3. To make asyncio work with PEP 3152 it would be required to modify +3. To make asyncio work with :pep:`3152` it would be required to modify ``@asyncio.coroutine`` decorator to wrap all functions in an object with a ``__cocall__`` method, or to implement ``__cocall__`` on generators. To call *cofunctions* from existing generator-based @@ -998,8 +996,8 @@ Differences from this proposal: coroutines with ``yield`` or ``async yield`` expressions -- we wouldn't need a ``cocall`` keyword to call them. So we'll end up having ``__cocall__`` and no ``__call__`` for regular coroutines, - and having ``__call__`` and no ``__cocall__`` for coroutine- - generators. + and having ``__call__`` and no ``__cocall__`` for + coroutine-generators. 6. Requiring parentheses grammatically also introduces a whole lot of new problems. @@ -1065,11 +1063,11 @@ project easier (Python with ECMAScript 7 for instance). Why "__aiter__" does not return an awaitable -------------------------------------------- -PEP 492 was accepted in CPython 3.5.0 with ``__aiter__`` defined as +:pep:`492` was accepted in CPython 3.5.0 with ``__aiter__`` defined as a method, that was expected to return an awaitable resolving to an asynchronous iterator. -In 3.5.2 (as PEP 492 was accepted on a provisional basis) the +In 3.5.2 (as :pep:`492` was accepted on a provisional basis) the ``__aiter__`` protocol was updated to return asynchronous iterators directly. @@ -1218,7 +1216,7 @@ Overall Impact This proposal introduces no observable performance impact. Here is an output of python's official set of benchmarks [4]_: -:: +.. code:: text python perf.py -r -b default ../cpython/python.exe ../cpython-aw/python.exe @@ -1318,8 +1316,8 @@ List of high-level changes and new protocols associated protocol with ``__aenter__`` and ``__aexit__`` methods. 4. New syntax for asynchronous iteration: ``async for``. And - associated protocol with ``__aiter__``, ``__aexit__`` and new built- - in exception ``StopAsyncIteration``. New ``tp_as_async.am_aiter`` + associated protocol with ``__aiter__``, ``__aexit__`` and new built-in + exception ``StopAsyncIteration``. New ``tp_as_async.am_aiter`` and ``tp_as_async.am_anext`` slots in ``PyTypeObject``. 5. New AST nodes: ``AsyncFunctionDef``, ``AsyncFor``, ``AsyncWith``, @@ -1356,7 +1354,7 @@ Working example All concepts proposed in this PEP are implemented [3]_ and can be tested. -:: +.. code:: python import asyncio @@ -1388,7 +1386,7 @@ tested. Acceptance ========== -PEP 492 was accepted by Guido, Tuesday, May 5, 2015 [14]_. +:pep:`492` was accepted by Guido, Tuesday, May 5, 2015 [14]_. Implementation @@ -1456,12 +1454,3 @@ Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0493.txt b/peps/pep-0493.rst similarity index 95% rename from pep-0493.txt rename to peps/pep-0493.rst index 5947180ad..0cf4c5341 100644 --- a/pep-0493.txt +++ b/peps/pep-0493.rst @@ -18,7 +18,7 @@ Resolution: https://mail.python.org/pipermail/python-dev/2016-March/143450.html Abstract ======== -PEP 476 updated Python's default handling of HTTPS certificates in client +:pep:`476` updated Python's default handling of HTTPS certificates in client modules to align with certificate handling in web browsers, by validating that the certificates received belonged to the server the client was attempting to contact. The Python 2.7 long term maintenance series was judged to be in @@ -35,7 +35,7 @@ from the decision to update to newer Python 2.7 maintenance releases. Rationale ========= -PEP 476 changed Python's default behaviour to align with expectations +:pep:`476` changed Python's default behaviour to align with expectations established by web browsers in regards to the semantics of HTTPS URLs: starting with Python 2.7.9 and 3.4.3, HTTPS clients in the standard library validate server certificates by default. @@ -52,13 +52,13 @@ offer options to switch off certificate checking entirely (by way of ``curl --insecure`` and ``wget --no-check-certificate``, respectively). At a different layer of the technology stack, Linux security modules like -`SELinux` and `AppArmor`, while enabled by default by distribution vendors, +``SELinux`` and ``AppArmor``, while enabled by default by distribution vendors, offer relatively straightforward mechanisms for turning them off. At the moment, no such convenient mechanisms exist to disable Python's default certificate checking for a whole process. -PEP 476 did attempt to address this question, by covering how to revert to the +:pep:`476` did attempt to address this question, by covering how to revert to the old settings process wide by monkeypatching the ``ssl`` module to restore the old behaviour. Unfortunately, the ``sitecustomize.py`` based technique proposed to allow system administrators to disable the feature by default in their @@ -86,7 +86,7 @@ redistributors will still make their own design decisions in the interests of their customers. The main approaches available are: * Continuing to rebase on new Python 2.7.x releases, while providing no - additional assistance beyond the mechanisms defined in PEP 476 in migrating + additional assistance beyond the mechanisms defined in :pep:`476` in migrating from unchecked to checked hostnames in standard library HTTPS clients * Gating availability of the changes in default handling of HTTPS connections on upgrading from Python 2 to Python 3 @@ -121,14 +121,14 @@ or absence of the feature to be determined using the following technique:: python -c "import ssl; ssl.<_relevant_attribute>" -This will fail with `AttributeError` (and hence a non-zero return code) if the -relevant capability is not available. +This will fail with ``AttributeError`` (and hence a non-zero return code) if +the relevant capability is not available. The feature detection attributes defined by this PEP are: * ``ssl._https_verify_certificates``: runtime configuration API * ``ssl._https_verify_envvar``: environment based configuration -* ``ssl._cert_verification_config``: file based configuration (PEP 476 opt-in) +* ``ssl._cert_verification_config``: file based configuration (:pep:`476` opt-in) The marker attributes are prefixed with an underscore to indicate the implementation dependent and security sensitive nature of these capabilities. @@ -281,7 +281,7 @@ Backporting this PEP to earlier Python versions If this PEP is accepted, then commercial Python redistributors may choose to backport the per-process configuration mechanisms defined in this PEP to base -versions older than Python 2.7.9, *without* also backporting PEP 476's change +versions older than Python 2.7.9, *without* also backporting :pep:`476`'s change to the default behaviour of the overall Python installation. Such a backport would differ from the mechanism proposed in this PEP solely in @@ -303,7 +303,7 @@ is nominally older than Python 2.7.12. Specification ------------- -Implementing this backport involves backporting the changes in PEP 466, 476 and +Implementing this backport involves backporting the changes in :pep:`466`, 476 and this PEP, with the following change to the handling of the ``PYTHONHTTPSVERIFY`` environment variable in the ``ssl`` module: @@ -342,7 +342,7 @@ Security Considerations This change would be a strict security upgrade for any Python version that currently defaults to skipping certificate validation in standard library HTTPS clients. The technical trade-offs to be taken into account relate largely -to the magnitude of the PEP 466 backport also required rather than to anything +to the magnitude of the :pep:`466` backport also required rather than to anything security related. Interaction with Python virtual environments @@ -408,9 +408,9 @@ relevant configuration file name. Recommended modifications to the Python standard library -------------------------------------------------------- -The recommended approach to backporting the PEP 476 modifications to an earlier +The recommended approach to backporting the :pep:`476` modifications to an earlier point release is to implement the following changes relative to the default -PEP 476 behaviour implemented in Python 2.7.9+: +:pep:`476` behaviour implemented in Python 2.7.9+: * modify the ``ssl`` module to read a system wide configuration file when the module is first imported into a Python process @@ -544,7 +544,7 @@ Recommendation for combined feature backports If a redistributor chooses to backport the environment variable based configuration setting from this PEP to a modified Python version that also -implements the configuration file based PEP 476 backport, then the environment +implements the configuration file based :pep:`476` backport, then the environment variable should take precedence over the system-wide configuration setting. This allows the setting to be changed for a given user or application, regardless of the installation-wide default behaviour. diff --git a/pep-0494.txt b/peps/pep-0494.rst similarity index 90% rename from pep-0494.txt rename to peps/pep-0494.rst index 7cce1fbe4..e358410a1 100644 --- a/pep-0494.txt +++ b/peps/pep-0494.rst @@ -3,8 +3,9 @@ Title: Python 3.6 Release Schedule Version: $Revision$ Last-Modified: $Date$ Author: Ned Deily -Status: Active +Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 30-May-2015 Python-Version: 3.6 @@ -17,10 +18,6 @@ This document describes the development and release schedule for Python 3.6. The schedule primarily concerns itself with PEP-sized items. -.. Small features may be added up to the first beta - release. Bugs may be fixed until the final release, - which is planned for December 2016. - Release Manager and Crew ======================== @@ -42,6 +39,12 @@ After that, it is expected that (source only) will be released as needed until 5 years after the release of 3.6 final, so until approximately 2021-12. +As of 2021-12-23, 3.6 has reached the +`end-of-life phase `_ +of its release cycle. 3.6.15 was the final security release. The code base for +3.6 is now frozen and no further updates will be provided nor issues of any +kind will be accepted on the bug tracker. + Release Schedule ================ @@ -150,18 +153,11 @@ Source only - 3.6.14 final: 2021-06-28 -3.6.15 schedule ---------------- +3.6.15 schedule (last security-only release) +-------------------------------------------- - 3.6.15 final: 2021-09-04 -3.6.16 and beyond schedule --------------------------- - -Security fixes only, as needed, until 2021-12 - -- TBD - Features for 3.6 ================ diff --git a/peps/pep-0495-daylightsavings.png b/peps/pep-0495-daylightsavings.png new file mode 100644 index 000000000..913d67de5 Binary files /dev/null and b/peps/pep-0495-daylightsavings.png differ diff --git a/pep-0495-fold.svg b/peps/pep-0495-fold.svg similarity index 100% rename from pep-0495-fold.svg rename to peps/pep-0495-fold.svg diff --git a/peps/pep-0495-gap.svg b/peps/pep-0495-gap.svg new file mode 100644 index 000000000..090213a30 --- /dev/null +++ b/peps/pep-0495-gap.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/pep-0495.txt b/peps/pep-0495.rst similarity index 99% rename from pep-0495.txt rename to peps/pep-0495.rst index 197e78a81..dc88ca603 100644 --- a/pep-0495.txt +++ b/peps/pep-0495.rst @@ -3,7 +3,7 @@ Title: Local Time Disambiguation Version: $Revision$ Last-Modified: $Date$ Author: Alexander Belopolsky , Tim Peters -Discussions-To: Datetime-SIG +Discussions-To: datetime-sig@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -143,7 +143,7 @@ Access macros will be defined to extract the value of ``fold`` from .. code:: - int PyDateTime_GET_FOLD(PyDateTime_DateTime *o) + int PyDateTime_DATE_GET_FOLD(PyDateTime_DateTime *o) Return the value of ``fold`` as a C ``int``. @@ -384,9 +384,10 @@ smaller after any transition that creates a fold. The values returned by ``tzname()`` and ``dst()`` may or may not depend on the value of the ``fold`` attribute depending on the kind of the transition. -.. image:: pep-0495-fold-2.png +.. image:: pep-0495-fold.svg :align: center :width: 60% + :class: invert-in-dark-mode The sketch above illustrates the relationship between the UTC and local time around a fall-back transition. The zig-zag line is a graph @@ -409,9 +410,10 @@ local time that falls in a gap, the rules in effect before the transition should be used if ``fold=0``. Otherwise, the rules in effect after the transition should be used. -.. image:: pep-0495-gap.png +.. image:: pep-0495-gap.svg :align: center :width: 60% + :class: invert-in-dark-mode The sketch above illustrates the relationship between the UTC and local time around a spring-forward transition. At the transition, the @@ -687,7 +689,7 @@ The following alternative names have also been considered: the original time is invalid. **which** - The `original`_ placeholder name for the `localtime` function + The `original`_ placeholder name for the ``localtime`` function branch index was `independently proposed`_ for the name of the disambiguation attribute and received `some support`_. diff --git a/pep-0496.txt b/peps/pep-0496.rst similarity index 86% rename from pep-0496.txt rename to peps/pep-0496.rst index 528267119..53068cfc2 100644 --- a/pep-0496.txt +++ b/peps/pep-0496.rst @@ -6,16 +6,17 @@ Author: James Polley BDFL-Delegate: Nick Coghlan Status: Rejected Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 03-Jul-2015 PEP Status ========== -After this PEP was initially drafted, PEP 508 was developed and submitted to +After this PEP was initially drafted, :pep:`508` was developed and submitted to fully specify the dependency declaration syntax, including environment markers. As a result, this PEP ended up being rejected in favour of the more comprehensive -PEP 508. +:pep:`508`. Abstract ======== @@ -26,8 +27,8 @@ required in particular environments, and to indicate supported platforms for distributions with additional constraints beyond the availability of a Python runtime. -Environment markers were first specified in PEP-0345 [#pep345]_. PEP-0426 -[#pep426]_ (which would replace PEP-0345) proposed extensions to the markers. +Environment markers were first specified in :pep:`345`. :pep:`426` +(which would replace :pep:`345`) proposed extensions to the markers. When 2.7.10 was released, even these extensions became insufficient due to their reliance on simple lexical comparisons, and thus this PEP has been born. @@ -97,9 +98,9 @@ The pseudo-grammar is :: ``SUBEXPR`` is either a Python string (such as ``'win32'``) or one of the ``Strings`` marker variables listed below. -``VEREXPR`` is a PEP-0440 [#pep440]_ version identifier, or one of the +``VEREXPR`` is a :pep:`440` version identifier, or one of the ``Version number`` marker variables listed below. Comparisons between -version numbers are done using PEP-0440 semantics. +version numbers are done using :pep:`440` semantics. Strings @@ -147,18 +148,6 @@ respectively, in accordance with the following algorithm:: ``python_full_version`` will typically correspond to ``sys.version.split()[0]``. -References -========== - -.. [#pep345] PEP 345, Metadata for Python Software Packages 1.2, Jones - (http://www.python.org/dev/peps/pep-0345) - -.. [#pep426] PEP 426, Metadata for Python Software Packages 2.0, Coghlan, Holth, Stufft - (http://www.python.org/dev/peps/pep-0426) - -.. [#pep440] PEP 440, Version Identification and Dependency Specification, Coghlan, Stufft - (https://www.python.org/dev/peps/pep-0440/) - Copyright ========= diff --git a/pep-0497.txt b/peps/pep-0497.rst similarity index 90% rename from pep-0497.txt rename to peps/pep-0497.rst index bc6fa24b5..18d352eb7 100644 --- a/pep-0497.txt +++ b/peps/pep-0497.rst @@ -3,7 +3,7 @@ Title: A standard mechanism for backward compatibility Version: $Revision$ Last-Modified: $Date$ Author: Ed Schofield -BDFL-Delegate: Brett Cannon (on behalf of the steering council) +PEP-Delegate: Brett Cannon Status: Rejected Type: Process Content-Type: text/x-rst @@ -12,9 +12,9 @@ Created: 04-Aug-2015 Rejection Notice ================ -The steering council decided that the `__past__` aspect of this proposal +The steering council decided that the ``__past__`` aspect of this proposal was too complicated for the potential benefit. The other aspect of stronger -requirements for backwards-compatibility should be addressed by PEP 387. +requirements for backwards-compatibility should be addressed by :pep:`387`. Scope @@ -24,14 +24,14 @@ This PEP is complementary to PEPs 5, 236, and 387, and shares similar goals. This PEP explains the need for an additional compatibility mechanism -in support of PEP 5, "Guidelines for Language Evolution". PEP 236, -"Back to the __future__", introduced a mechanism for forward -compatibility in support of PEP 5 but noted that a new mechanism for +in support of :pep:`5`, "Guidelines for Language Evolution". :pep:`236`, +"Back to the ``__future__``", introduced a mechanism for forward +compatibility in support of :pep:`5` but noted that a new mechanism for backward compatibility was outside the scope of that PEP. A related PEP (in progress) introduces such a mechanism for backward compatibility. -PEP 5, "Guidelines for Language Evolution", notes that "This PEP [PEP 5] +:pep:`5`, "Guidelines for Language Evolution", notes that "This PEP [:pep:`5`] does not replace or preclude other compatibility strategies such as dynamic loading of backwards-compatible parsers." @@ -39,16 +39,16 @@ dynamic loading of backwards-compatible parsers." Context ======= -From PEP 236: "From time to time, Python makes an incompatible change +From :pep:`236`: "From time to time, Python makes an incompatible change to the advertised semantics of core language constructs, or changes their accidental (implementation-dependent) behavior in some way. While this is never done capriciously, and is always done with the aim of improving the language over the long term, over the short term it's -contentious and disrupting. PEP 5, Guidelines for Language Evolution, -suggests ways to ease the pain, and this PEP [236] introduces some +contentious and disrupting. :pep:`5`, Guidelines for Language Evolution, +suggests ways to ease the pain, and this PEP [:pep:`236`] introduces some machinery in support of that." -Also from PEP 236: "The purpose of future_statement is to make life +Also from :pep:`236`: "The purpose of future_statement is to make life easier for people who keep current with the latest release in a timely fashion. We don't hate you if you don't, but your problems are much harder to solve, and somebody with those problems will need to write a @@ -77,8 +77,8 @@ the newer interpreter is only capable of using code that has been upgraded to support the changed feature. As an example, consider the changes to the division operator -introduced in PEP 238 in 2001, soon after PEP 236 introduced the -future_statement mechanism. PEP 238 outlines a suite of useful +introduced in :pep:`238` in 2001, soon after :pep:`236` introduced the +future_statement mechanism. :pep:`238` outlines a suite of useful forward-compatibility mechanisms for "true division" in the Python 2.x series but omits to include any backward-compatibility mechanisms for after "true division" was first enforced in Python 3.0. Python versions @@ -127,14 +127,14 @@ Proposal - part 1 This PEP makes two specific, related proposals. The first is that: - PEP 5 be augmented with a 6th step in the section "Steps for + :pep:`5` be augmented with a 6th step in the section "Steps for Introducing Backwards-Incompatible Features" to indicate that, when an incompatible change to core language syntax or semantics is being made, Python-dev's policy is to prefer and expect that, wherever possible, a mechanism for backward compatibility be considered and provided for future Python versions after the breaking change is adopted by default, in addition to any mechanisms proposed for forward - compatibility such as new future_statements. Furthermore, PEP 387, + compatibility such as new future_statements. Furthermore, :pep:`387`, "Backwards Compatibility Policy" (if accepted) would be augmented with the same 6th step. @@ -144,7 +144,7 @@ Example As an example of how this PEP is to be applied, if the latest revision of the "true division" PEP (238) were proposed today, it would be -considered incomplete. PEP 238 notes the "severe backwards +considered incomplete. :pep:`238` notes the "severe backwards compatibility issues" raised by the proposal and describes several measures for forward compatibility in the Abstract and API Changes sections. It also mentions some backward compatibility ideas raised on @@ -153,7 +153,7 @@ classic division semantics in a module", but it does not put forward any backward compatibility plan as part of the proposal. If this PEP is accepted, it would be expected that a proposal such as -PEP 238, because of its large-scale compatibility implications, would +:pep:`238`, because of its large-scale compatibility implications, would also be accompanied by a backward compatibility plan that enables users of future Python versions after the breaking change has come into effect to re-enable the classic division behaviour easily in @@ -176,7 +176,7 @@ of the ``__future__`` module and ``future_statement`` mechanism. The specific form and implementation of the ``__past__`` mechanism is the subject of a separate PEP (in progress). However, this PEP recommends that this ``__past__`` mechanism be designed to meet -similar criteria to those outlined in PEP 296 for ``__future__``. +similar criteria to those outlined in :pep:`296` for ``__future__``. Specifically: a. It should enable individual modules to specify obsolete behaviours diff --git a/pep-0498.txt b/peps/pep-0498.rst similarity index 98% rename from pep-0498.txt rename to peps/pep-0498.rst index c0f03f683..22fc26892 100644 --- a/pep-0498.txt +++ b/peps/pep-0498.rst @@ -42,10 +42,10 @@ their values. Some examples are:: >>> f'He said his name is {name!r}.' "He said his name is 'Fred'." -A similar feature was proposed in PEP 215. PEP 215 proposed to support +A similar feature was proposed in :pep:`215`. :pep:`215` proposed to support a subset of Python expressions, and did not support the type-specific string formatting (the ``__format__()`` method) which was introduced -with PEP 3101. +with :pep:`3101`. Rationale ========= @@ -78,9 +78,9 @@ To be defensive, the following code should be used:: ``str.format()`` was added to address some of these problems with %-formatting. In particular, it uses normal function call syntax (and -therefor supports multiple parameters) and it is extensible through +therefore supports multiple parameters) and it is extensible through the ``__format__()`` method on the object being converted to a -string. See PEP 3101 for a detailed rationale. This PEP reuses much of +string. See :pep:`3101` for a detailed rationale. This PEP reuses much of the ``str.format()`` syntax and machinery, in order to provide continuity with an existing Python string formatting mechanism. @@ -613,7 +613,7 @@ is not compatible with a bytes string. Binary f-strings would first require a solution for ``bytes.format()``. This idea has been proposed in the past, most -recently in PEP 461 [#]_. The discussions of such a feature usually +recently in :pep:`461#proposed-variations`. The discussions of such a feature usually suggest either - adding a method such as ``__bformat__()`` so an object can control @@ -667,7 +667,7 @@ If you feel you must use lambdas, they may be used inside of parentheses:: Can't combine with 'u' -------------------------- -The 'u' prefix was added to Python 3.3 in PEP 414 as a means to ease +The 'u' prefix was added to Python 3.3 in :pep:`414` as a means to ease source compatibility with Python 2.7. Because Python 2.7 will never support f-strings, there is nothing to be gained by being able to combine the 'f' prefix with 'u'. @@ -734,9 +734,6 @@ References .. [#] Differences in str.format() and f-string expressions (https://mail.python.org/pipermail/python-ideas/2015-July/034726.html) -.. [#] PEP 461 rejects bytes.format() - (https://www.python.org/dev/peps/pep-0461/#proposed-variations) - Copyright ========= diff --git a/pep-0499.txt b/peps/pep-0499.rst similarity index 84% rename from pep-0499.txt rename to peps/pep-0499.rst index b75f527ef..8a4862af7 100644 --- a/pep-0499.txt +++ b/peps/pep-0499.rst @@ -1,7 +1,5 @@ PEP: 499 -Title: ``python -m foo`` should bind ``sys.modules['foo']`` in addition to ``sys.modules['__main__']`` -Version: $Revision$ -Last-Modified: $Date$ +Title: ``python -m foo`` should also bind ``'foo'`` in ``sys.modules`` Author: Cameron Simpson , Chris Angelico , Joseph Jevnik BDFL-Delegate: Nick Coghlan Status: Deferred @@ -147,7 +145,7 @@ Therefore, ``__name__`` is no longer the canonical name for some normal imports. Some counter arguments follow: -- As of PEP 451 a module's canonical name is stored at ``__spec__.name``. +- As of :pep:`451` a module's canonical name is stored at ``__spec__.name``. - Very little code should actually care about ``__name__`` being the canonical name and any that does should arguably be updated to consult ``__spec__.name`` with fallback to ``__name__`` for older Pythons, should that be relevant. @@ -182,29 +180,30 @@ Pickle compatibility If no changes are made to the pickle module, then pickles that were previously being written with the correct module name (due to a dual import) may start -being written with `__main__` as their module name instead, and hence fail to be -loaded correctly by other projects. +being written with ``__main__`` as their module name instead, and hence fail +to be loaded correctly by other projects. Scenarios to be checked: -* `python script.py` writing, `python -m script` reading -* `python -m script` writing, `python script.py` reading -* `python -m script` writing, `python some_other_app.py` reading -* `old_python -m script` writing, `new_python -m script` reading -* `new_python -m script` writing, `old_python -m script` reading +* ``python script.py`` writing, ``python -m script`` reading +* ``python -m script`` writing, ``python script.py`` reading +* ``python -m script`` writing, ``python some_other_app.py`` reading +* ``old_python -m script`` writing, ``new_python -m script`` reading +* ``new_python -m script`` writing, ``old_python -m script`` reading -Projects that special-case `__main__` -------------------------------------- +Projects that special-case ``__main__`` +--------------------------------------- In order to get the regression test suite to pass, the current reference -implementation had to patch `pdb` to avoid destroying its own global namespace. +implementation had to patch ``pdb`` to avoid destroying its own global +namespace. This suggests there may be a broader compatibility issue where some scripts are relying on direct execution and import giving different namespaces (just as -package execution keeps the two separate by executing the `__main__` submodule -in the `__main__` namespace, while the package name references the `__init__` -file as usual. +package execution keeps the two separate by executing the ``__main__`` +submodule in the ``__main__`` namespace, while the package name references +the ``__init__`` file as usual. Background @@ -219,9 +218,10 @@ module instance, not the running module instance. However, the problem has been around as long as the ``-m`` command line option and is encountered regularly, if infrequently, by others. -In addition to `issue 19702`_, the discrepancy around `__main__` -is alluded to in PEP 451 and a similar proposal (predating PEP 451) -is described in PEP 395 under `Fixing dual imports of the main module`_. +In addition to `issue 19702`_, the discrepancy around ``__main__`` +is alluded to in :pep:`451` and a similar proposal (predating :pep:`451`) +is described in :pep:`395` under +:pep:`Fixing dual imports of the main module <395#fixing-dual-imports-of-the-main-module>`. References @@ -231,20 +231,8 @@ References .. _I tripped over this issue: https://mail.python.org/pipermail/python-list/2015-August/694905.html -.. _Fixing dual imports of the main module: https://www.python.org/dev/peps/pep-0395/#fixing-dual-imports-of-the-main-module - Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0500.txt b/peps/pep-0500.rst similarity index 97% rename from pep-0500.txt rename to peps/pep-0500.rst index 947a41e8c..b96f48a42 100644 --- a/pep-0500.txt +++ b/peps/pep-0500.rst @@ -1,10 +1,9 @@ PEP: 500 -Title: A protocol for delegating datetime methods to their - tzinfo implementations +Title: A protocol for delegating datetime methods to their tzinfo implementations Version: $Revision$ Last-Modified: $Date$ Author: Alexander Belopolsky , Tim Peters -Discussions-To: Datetime-SIG +Discussions-To: datetime-sig@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0501.txt b/peps/pep-0501.rst similarity index 90% rename from pep-0501.txt rename to peps/pep-0501.rst index e623c28ed..f368a54bd 100644 --- a/pep-0501.txt +++ b/peps/pep-0501.rst @@ -14,7 +14,7 @@ Post-History: 08-Aug-2015, 23-Aug-2015, 30-Aug-2015 Abstract ======== -PEP 498 proposes new syntactic support for string interpolation that is +:pep:`498` proposes new syntactic support for string interpolation that is transparent to the compiler, allow name references from the interpolation operation full access to containing namespaces (as with any other expression), rather than being limited to explicit name references. These are referred @@ -47,7 +47,7 @@ Some possible examples of the proposed syntax:: PEP Deferral ============ -This PEP is currently deferred pending further experience with PEP 498's +This PEP is currently deferred pending further experience with :pep:`498`'s simpler approach of only supporting eager rendering without the additional complexity of also supporting deferred rendering. @@ -55,7 +55,7 @@ complexity of also supporting deferred rendering. Summary of differences from PEP 498 =================================== -The key additions this proposal makes relative to PEP 498: +The key additions this proposal makes relative to :pep:`498`: * the "i" (interpolation template) prefix indicates delayed rendering, but otherwise uses the same syntax and semantics as formatted strings @@ -70,7 +70,7 @@ The key additions this proposal makes relative to PEP 498: NOTE: This proposal spells out a draft API for ``types.InterpolationTemplate``. The precise details of the structures and methods exposed by this type would -be informed by the reference implementation of PEP 498, so it makes sense to +be informed by the reference implementation of :pep:`498`, so it makes sense to gain experience with that as an internal API before locking down a public API (if this extension proposal is accepted). @@ -120,7 +120,7 @@ strings that are not present directly in the source code of the application. Rationale ========= -PEP 498 makes interpolating values into strings with full access to Python's +:pep:`498` makes interpolating values into strings with full access to Python's lexical namespace semantics simpler, but it does so at the cost of creating a situation where interpolating values into sensitive targets like SQL queries, shell commands and HTML templates will enjoy a much cleaner syntax when handled @@ -148,7 +148,7 @@ permitted), and string literal concatenation operates as normal, with the entire combined literal forming the interpolation template. The template string is parsed into literals, expressions and format specifiers -as described for f-strings in PEP 498. Conversion specifiers are handled +as described for f-strings in :pep:`498`. Conversion specifiers are handled by the compiler, and appear as part of the field text in interpolation templates. @@ -264,7 +264,7 @@ NOTE: interpolation templates. The ``!a``, ``!r`` and ``!s`` conversion specifiers supported by ``str.format`` -and hence PEP 498 are handled in interpolation templates as follows: +and hence :pep:`498` are handled in interpolation templates as follows: * they're included unmodified in the raw template to ensure no information is lost @@ -334,7 +334,7 @@ Is essentially equivalent to:: Handling code injection attacks ------------------------------- -The PEP 498 formatted string syntax makes it potentially attractive to write +The :pep:`498` formatted string syntax makes it potentially attractive to write code like the following:: runquery(f"SELECT {column} FROM {table};") @@ -396,7 +396,7 @@ Invalid expressions:: SyntaxError: invalid syntax Run time errors occur when evaluating the expressions inside a -template string before creating the interpolation template object. See PEP 498 +template string before creating the interpolation template object. See :pep:`498` for some examples. Different renderers may also impose additional runtime @@ -456,7 +456,7 @@ that happen to be passed in as data values in a normal field. Discussion ========== -Refer to PEP 498 for additional discussion, as several of the points there +Refer to :pep:`498` for additional discussion, as several of the points there also apply to this PEP. Deferring support for binary interpolation @@ -480,7 +480,7 @@ For interoperability with interfaces that only accept strings, interpolation templates can still be prerendered with ``format``, rather than delegating the rendering to the called function. -This reflects the key difference from PEP 498, which *always* eagerly applies +This reflects the key difference from :pep:`498`, which *always* eagerly applies the default rendering, without any way to delegate the choice of renderer to another section of the code. @@ -503,10 +503,9 @@ made it much easier to support customisation of the semantics of interpolation. Building atop PEP 498, rather than competing with it ---------------------------------------------------- - Earlier versions of this PEP attempted to serve as a complete substitute for -PEP 498, rather than building a more flexible delayed rendering capability on -top of PEP 498's eager rendering. +:pep:`498`, rather than building a more flexible delayed rendering capability on +top of :pep:`498`'s eager rendering. Assuming the presence of f-strings as a supporting capability simplified a number of aspects of the proposal in this PEP (such as how to handle substitution @@ -527,9 +526,9 @@ contexts (like HTML, system shells, and database queries), or producing application debugging messages in the preferred language of the development team (rather than the native language of end users). -Due to the original design of the ``str.format`` substitution syntax in PEP -3101 being inspired by C#'s string formatting syntax, the specific field -substitution syntax used in PEP 498 is consistent not only with Python's own ``str.format`` syntax, but also with string formatting in C#, including the +Due to the original design of the ``str.format`` substitution syntax in :pep:`3101` +being inspired by C#'s string formatting syntax, the specific field +substitution syntax used in :pep:`498` is consistent not only with Python's own ``str.format`` syntax, but also with string formatting in C#, including the native "$-string" interpolation syntax introduced in C# 6.0 (released in July 2015). The related ``IFormattable`` interface in C# forms the basis of a `number of elements `__ of C#'s internationalization and localization @@ -545,7 +544,7 @@ format for translating C# applications). Acknowledgements ================ -* Eric V. Smith for creating PEP 498 and demonstrating the feasibility of +* Eric V. Smith for creating :pep:`498` and demonstrating the feasibility of arbitrary expression substitution in string interpolation * Barry Warsaw, Armin Ronacher, and Mike Miller for their contributions to exploring the feasibility of using this model of delayed rendering in i18n @@ -555,47 +554,33 @@ Acknowledgements References ========== -.. [#] %-formatting - (https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting) +* `%-formatting + `_ -.. [#] str.format - (https://docs.python.org/3/library/string.html#formatstrings) +* `str.format + `_ -.. [#] string.Template documentation - (https://docs.python.org/3/library/string.html#template-strings) +* `string.Template documentation + `_ -.. [#] PEP 215: String Interpolation - (https://www.python.org/dev/peps/pep-0215/) +* :pep:`215`: String Interpolation -.. [#] PEP 292: Simpler String Substitutions - (https://www.python.org/dev/peps/pep-0292/) +* :pep:`292`: Simpler String Substitutions -.. [#] PEP 3101: Advanced String Formatting - (https://www.python.org/dev/peps/pep-3101/) +* :pep:`3101`: Advanced String Formatting -.. [#] PEP 498: Literal string formatting - (https://www.python.org/dev/peps/pep-0498/) +* :pep:`498`: Literal string formatting -.. [#] FormattableString and C# native string interpolation - (https://msdn.microsoft.com/en-us/library/dn961160.aspx) +* `FormattableString and C# native string interpolation + `_ -.. [#] IFormattable interface in C# (see remarks for globalization notes) - (https://msdn.microsoft.com/en-us/library/system.iformattable.aspx) +* `IFormattable interface in C# (see remarks for globalization notes) + `_ -.. [#] Running external commands in Julia - (http://julia.readthedocs.org/en/latest/manual/running-external-programs/) +* `Running external commands in Julia + `_ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0502.txt b/peps/pep-0502.rst similarity index 90% rename from pep-0502.txt rename to peps/pep-0502.rst index 5e6855d3f..429c2be7e 100644 --- a/pep-0502.txt +++ b/peps/pep-0502.rst @@ -1,7 +1,5 @@ PEP: 502 Title: String Interpolation - Extended Discussion -Version: $Revision$ -Last-Modified: $Date$ Author: Mike G. Miller Status: Rejected Type: Informational @@ -13,7 +11,7 @@ Python-Version: 3.6 Abstract ======== -PEP 498: *Literal String Interpolation*, which proposed "formatted strings" was +:pep:`498`: *Literal String Interpolation*, which proposed "formatted strings" was accepted September 9th, 2015. Additional background and rationale given during its design phase is detailed below. @@ -22,7 +20,7 @@ To recap that PEP, a string prefix was introduced that marks the string as a template to be rendered. These formatted strings may contain one or more expressions -built on `the existing syntax`_ of ``str.format()``. +built on `the existing syntax`_ of ``str.format()``. [10]_ [11]_ The formatted string expands at compile-time into a conventional string format operation, with the given expressions from its text extracted and passed instead as @@ -46,7 +44,7 @@ PEP Status ========== This PEP was rejected based on its using an opinion-based tone rather than a factual one. -This PEP was also deemed not critical as PEP 498 was already written and should be the place +This PEP was also deemed not critical as :pep:`498` was already written and should be the place to house design decision details. @@ -64,10 +62,10 @@ or identifier duplication. These difficulties are described at moderate length in the original `post to python-ideas`_ -that started the snowball (that became PEP 498) rolling. [1]_ +that started the snowball (that became :pep:`498`) rolling. [1]_ Furthermore, replacement of the print statement with the more consistent print -function of Python 3 (PEP 3105) has added one additional minor burden, +function of Python 3 (:pep:`3105`) has added one additional minor burden, an additional set of parentheses to type and read. Combined with the verbosity of current string formatting solutions, this puts an otherwise simple language at an unfortunate disadvantage to its @@ -192,7 +190,7 @@ or forget the trailing type, e.g. (``s`` or ``d``). string.Template Class ''''''''''''''''''''' -The ``string.Template`` `class from`_ PEP 292 +The ``string.Template`` `class from`_ :pep:`292` (Simpler String Substitutions) is a purposely simplified design, using familiar shell interpolation syntax, @@ -220,7 +218,7 @@ unless encapsulated in a `convenience library`_ such as ``flufl.i18n``. PEP 215 - String Interpolation '''''''''''''''''''''''''''''' -PEP 215 was a former proposal of which this one shares a lot in common. +:pep:`215` was a former proposal of which this one shares a lot in common. Apparently, the world was not ready for it at the time, but considering recent support in a number of other languages, its day may have come. @@ -234,7 +232,7 @@ It was superseded by the following proposal. str.format() Method ''''''''''''''''''' -The ``str.format()`` `syntax of`_ PEP 3101 is the most recent and modern of the +The ``str.format()`` `syntax of`_ :pep:`3101` is the most recent and modern of the existing options. It is also more powerful and usually easier to read than the others. It avoids many of the drawbacks and limits of the previous techniques. @@ -260,7 +258,7 @@ The verbosity of the method-based approach is illustrated here. PEP 498 -- Literal String Formatting '''''''''''''''''''''''''''''''''''' -PEP 498 defines and discusses format strings, +:pep:`498` defines and discusses format strings, as also described in the `Abstract`_ above. It also, somewhat controversially to those first exposed, @@ -273,7 +271,7 @@ Restricting Syntax section under PEP 501 -- Translation ready string interpolation ''''''''''''''''''''''''''''''''''''''''''''''''' -The complimentary PEP 501 brings internationalization into the discussion as a +The complimentary :pep:`501` brings internationalization into the discussion as a first-class concern, with its proposal of the i-prefix, ``string.Template`` syntax integration compatible with ES6 (Javascript), deferred rendering, @@ -463,7 +461,7 @@ the existing choices, >>> f'Hello, {location} !' # new prefix: f'' 'Hello, World !' # interpolated result -PEP 498 -- Literal String Formatting, delves into the mechanics and +:pep:`498` -- Literal String Formatting, delves into the mechanics and implementation of this design. @@ -565,7 +563,7 @@ Internationalization '''''''''''''''''''' Though it was highly desired to integrate internationalization support, -(see PEP 501), +(see :pep:`501`), the finer details diverge at almost every point, making a common solution unlikely: [15]_ @@ -640,7 +638,7 @@ which could encourage bad habits. [13]_ Acknowledgements ================ -* Eric V. Smith for the authoring and implementation of PEP 498. +* Eric V. Smith for the authoring and implementation of :pep:`498`. * Everyone on the python-ideas mailing list for rejecting the various crazy ideas that came up, helping to keep the final design in focus. @@ -650,78 +648,52 @@ References ========== .. [1] Briefer String Format - - (https://mail.python.org/pipermail/python-ideas/2015-July/034659.html) + (https://mail.python.org/pipermail/python-ideas/2015-July/034659.html) .. [2] Briefer String Format - - (https://mail.python.org/pipermail/python-ideas/2015-July/034669.html) + (https://mail.python.org/pipermail/python-ideas/2015-July/034669.html) .. [3] Briefer String Format - - (https://mail.python.org/pipermail/python-ideas/2015-July/034701.html) + (https://mail.python.org/pipermail/python-ideas/2015-July/034701.html) .. [4] Bash Docs - - (http://www.tldp.org/LDP/abs/html/arithexp.html) + (https://tldp.org/LDP/abs/html/arithexp.html) .. [5] Bash Docs - - (http://www.tldp.org/LDP/abs/html/commandsub.html) + (https://tldp.org/LDP/abs/html/commandsub.html) .. [6] Perl Cookbook - - (http://docstore.mik.ua/orelly/perl/cookbook/ch01_11.htm) + (https://docstore.mik.ua/orelly/perl/cookbook/ch01_11.htm) .. [7] Perl Docs - - (http://perl6maven.com/perl6-scalar-array-and-hash-interpolation) + (https://web.archive.org/web/20121025185907/https://perl6maven.com/perl6-scalar-array-and-hash-interpolation) .. [8] Ruby Docs - - (http://ruby-doc.org/core-2.1.1/doc/syntax/literals_rdoc.html#label-Strings) + (http://ruby-doc.org/core-2.1.1/doc/syntax/literals_rdoc.html#label-Strings) .. [9] Ruby Docs - - (https://en.wikibooks.org/wiki/Ruby_Programming/Syntax/Literals#Interpolation) + (https://en.wikibooks.org/wiki/Ruby_Programming/Syntax/Literals#Interpolation) .. [10] Python Str.Format Syntax - - (https://docs.python.org/3/library/string.html#format-string-syntax) + (https://docs.python.org/3.6/library/string.html#format-string-syntax) .. [11] Python Format-Spec Mini Language - - (https://docs.python.org/3/library/string.html#format-specification-mini-language) + (https://docs.python.org/3.6/library/string.html#format-specification-mini-language) .. [12] Escaping of Input Variables - - (https://mail.python.org/pipermail/python-ideas/2015-August/035532.html) + (https://mail.python.org/pipermail/python-ideas/2015-August/035532.html) .. [13] Environment Access and Command Substitution - - (https://mail.python.org/pipermail/python-ideas/2015-August/035554.html) + (https://mail.python.org/pipermail/python-ideas/2015-August/035554.html) .. [14] Extensible String Prefixes - - (https://mail.python.org/pipermail/python-ideas/2015-August/035336.html) - + (https://mail.python.org/pipermail/python-ideas/2015-August/035336.html) .. [15] Literal String Formatting - - (https://mail.python.org/pipermail/python-dev/2015-August/141289.html) + (https://mail.python.org/pipermail/python-dev/2015-August/141289.html) Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0503.txt b/peps/pep-0503.rst similarity index 98% rename from pep-0503.txt rename to peps/pep-0503.rst index ad8e42d30..922afe70c 100644 --- a/pep-0503.txt +++ b/peps/pep-0503.rst @@ -5,8 +5,9 @@ Last-Modified: $Date$ Author: Donald Stufft BDFL-Delegate: Donald Stufft Discussions-To: distutils-sig@python.org -Status: Accepted -Type: Informational +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 04-Sep-2015 Post-History: 04-Sep-2015 @@ -98,7 +99,7 @@ In addition to the above, the following constraints are placed on the API: GPG signature. Repositories that do this **SHOULD** include it on every link. * A repository **MAY** include a ``data-requires-python`` attribute on a file - link. This exposes the *Requires-Python* metadata field, specified in PEP 345, + link. This exposes the *Requires-Python* metadata field, specified in :pep:`345`, for the corresponding release. Where this is present, installer tools **SHOULD** ignore the download when installing to a Python version that doesn't satisfy the requirement. For example:: @@ -111,7 +112,7 @@ In addition to the above, the following constraints are placed on the API: Normalized Names ---------------- -This PEP references the concept of a "normalized" project name. As per PEP 426 +This PEP references the concept of a "normalized" project name. As per :pep:`426` the only valid characters in a name are the ASCII alphabet, ASCII numbers, ``.``, ``-``, and ``_``. The name should be lowercased with all runs of the characters ``.``, ``-``, or ``_`` replaced with a single ``-`` character. This diff --git a/pep-0504.txt b/peps/pep-0504.rst similarity index 99% rename from pep-0504.txt rename to peps/pep-0504.rst index a1317b589..d893596b6 100644 --- a/pep-0504.txt +++ b/peps/pep-0504.rst @@ -50,7 +50,7 @@ Steven's proposal has the desired effect of aligning the easy way to generate such tokens and the right way to generate them, without introducing any compatibility risks for the existing ``random`` module API, so this PEP has been withdrawn in favour of further work on refining Steven's proposal as -PEP 506. +:pep:`506`. Proposal diff --git a/pep-0505.rst b/peps/pep-0505.rst similarity index 98% rename from pep-0505.rst rename to peps/pep-0505.rst index 108efead9..9669d3d6f 100644 --- a/pep-0505.rst +++ b/peps/pep-0505.rst @@ -1,7 +1,5 @@ PEP: 505 Title: None-aware operators -Version: $Revision$ -Last-Modified: $Date$ Author: Mark E. Haase , Steve Dower Status: Deferred Type: Standards Track @@ -236,7 +234,7 @@ conversion to use ``None``-aware operators may look like. Standard Library ---------------- -Using the ``find-pep505.py`` script[5]_ an analysis of the Python 3.7 standard +Using the ``find-pep505.py`` script [5]_ an analysis of the Python 3.7 standard library discovered up to 678 code snippets that could be replaced with use of one of the ``None``-aware operators:: @@ -572,8 +570,8 @@ Rejected Ideas The first three ideas in this section are oft-proposed alternatives to treating ``None`` as special. For further background on why these are rejected, see their -treatment in `PEP 531 `_ and -`PEP 532 `_ and the associated +treatment in :pep:`531` and +:pep:`532` and the associated discussions. No-Value Protocol @@ -751,7 +749,7 @@ Going back to one of the motivating examples above, consider the following:: >>> import json >>> created = None - >>> json.dumps({'created': created?.isoformat()})`` + >>> json.dumps({'created': created?.isoformat()}) The JSON serializer does not know how to serialize ``NoneQuestion``, nor will any other API. This proposal actually requires *lots of specialized logic* @@ -844,7 +842,7 @@ References ========== .. [1] C# Reference: Operators - (https://msdn.microsoft.com/en-us/library/6a71f45d.aspx) + (https://learn.microsoft.com/en/dotnet/csharp/language-reference/operators/) .. [2] A Tour of the Dart Language: Operators (https://www.dartlang.org/docs/dart-up-and-running/ch02.html#operators) @@ -862,14 +860,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0505/find-pep505.out b/peps/pep-0505/find-pep505.out similarity index 100% rename from pep-0505/find-pep505.out rename to peps/pep-0505/find-pep505.out diff --git a/pep-0505/find-pep505.py b/peps/pep-0505/find-pep505.py similarity index 100% rename from pep-0505/find-pep505.py rename to peps/pep-0505/find-pep505.py diff --git a/pep-0505/test.py b/peps/pep-0505/test.py similarity index 100% rename from pep-0505/test.py rename to peps/pep-0505/test.py diff --git a/pep-0506.txt b/peps/pep-0506.rst similarity index 98% rename from pep-0506.txt rename to peps/pep-0506.rst index ab4f8d5f2..bd170f4ae 100644 --- a/pep-0506.txt +++ b/peps/pep-0506.rst @@ -226,8 +226,9 @@ module [#]_. This received considerable scepticism and outright opposition: without a proven attack against Python applications, many people object to a backwards-incompatible change. -Nick Coghlan made an earlier suggestion for a globally configurable PRNG -which uses the system CSPRNG by default [#]_, but has since withdrawn it +Nick Coghlan made an :pep:`earlier suggestion <504>` +for a globally configurable PRNG +which uses the system CSPRNG by default, but has since withdrawn it in favour of this proposal. @@ -412,8 +413,6 @@ References output from the system CSPRNG, which is believed to be much harder to exploit. -.. [#] http://legacy.python.org/dev/peps/pep-0504/ - .. [#] http://php.net/manual/en/function.uniqid.php .. [#] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php diff --git a/pep-0507.txt b/peps/pep-0507.rst similarity index 97% rename from pep-0507.txt rename to peps/pep-0507.rst index 9c363fba6..e4ace277d 100644 --- a/pep-0507.txt +++ b/peps/pep-0507.rst @@ -1,7 +1,5 @@ PEP: 507 Title: Migrate CPython to Git and GitLab -Version: $Revision$ -Last-Modified: $Date$ Author: Barry Warsaw Status: Rejected Type: Process @@ -17,10 +15,10 @@ Abstract This PEP proposes migrating the repository hosting of CPython and the supporting repositories to Git. Further, it proposes adopting a hosted GitLab instance as the primary way of handling merge requests, -code reviews, and code hosting. It is similar in intent to PEP 481 +code reviews, and code hosting. It is similar in intent to :pep:`481` but proposes an open source alternative to GitHub and omits the -proposal to run Phabricator. As with PEP 481, this particular PEP is -offered as an alternative to PEP 474 and PEP 462. +proposal to run Phabricator. As with :pep:`481`, this particular PEP is +offered as an alternative to :pep:`474` and :pep:`462`. Rationale @@ -310,23 +308,12 @@ Open issues References ========== -.. [#openhub-stats] `Open Hub Statistics ` -.. [#hg-git] `Hg-Git mercurial plugin ` -.. [#GitLab] `https://about.gitlab.com/` +.. [#openhub-stats] `Open Hub Statistics `_ +.. [#hg-git] `Hg-Git mercurial plugin `_ +.. [#GitLab] ``_ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0508.txt b/peps/pep-0508.rst similarity index 92% rename from pep-0508.txt rename to peps/pep-0508.rst index 19e855355..acc97cc8a 100644 --- a/pep-0508.txt +++ b/peps/pep-0508.rst @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Robert Collins BDFL-Delegate: Donald Stufft -Discussions-To: distutils-sig -Status: Active -Type: Informational +Discussions-To: distutils-sig@python.org +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 11-Nov-2015 Post-History: 05-Nov-2015, 16-Nov-2015 @@ -31,8 +32,8 @@ acceptable, so the language permits describing all these cases. The language defined is a compact line based format which is already in widespread use in pip requirements files, though we do not specify the command line option handling that those files permit. There is one caveat - the -URL reference form, specified in PEP-440 [#pep440]_ is not actually -implemented in pip, but since PEP-440 is accepted, we use that format rather +URL reference form, specified in :pep:`440` is not actually +implemented in pip, but since :pep:`440` is accepted, we use that format rather than pip's current native format. Motivation @@ -40,7 +41,7 @@ Motivation Any specification in the Python packaging ecosystem that needs to consume lists of dependencies needs to build on an approved PEP for such, but -PEP-426 [#pep426]_ is mostly aspirational - and there are already existing +:pep:`426` is mostly aspirational - and there are already existing implementations of the dependency specification which we can instead adopt. The existing implementations are battle proven and user friendly, so adopting them is arguably much better than approving an aspirational, unconsumed, format. @@ -82,8 +83,8 @@ as comments, multiple line support via continuations, or other such features. The full grammar including annotations to build a useful parse tree is included at the end of the PEP. -Versions may be specified according to the PEP-440 [#pep440]_ rules. (Note: -URI is defined in std-66 [#std66]_):: +Versions may be specified according to the :pep:`440` rules. (Note: +URI is defined in :rfc:`std-66 <3986>`):: version_cmp = wsp* '<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '===' version = wsp* ( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+ @@ -151,7 +152,7 @@ sole exception is detecting the end of a URL requirement. Names ----- -Python distribution names are currently defined in PEP-345 [#pep345]_. Names +Python distribution names are currently defined in :pep:`345`. Names act as the primary identifier for distributions. They are present in all dependency specifications, and are sufficient to be a specification on their own. However, PyPI places strict restrictions on names - they must match a @@ -182,12 +183,12 @@ If multiple extras are listed, all the dependencies are unioned together. Versions -------- -See PEP-440 [#pep440]_ for more detail on both version numbers and version +See :pep:`440` for more detail on both version numbers and version comparisons. Version specifications limit the versions of a distribution that can be used. They only apply to distributions looked up by name, rather than via a URL. Version comparison are also used in the markers feature. The -optional brackets around a version are present for compatibility with PEP-345 -[#pep345]_ but should not be generated, only accepted. +optional brackets around a version are present for compatibility with :pep:`345` +but should not be generated, only accepted. Environment Markers ------------------- @@ -204,15 +205,14 @@ False, the dependency specification should be ignored. The marker language is inspired by Python itself, chosen for the ability to safely evaluate it without running arbitrary code that could become a security -vulnerability. Markers were first standardised in PEP-345 [#pep345]_. This PEP -fixes some issues that were observed in the design described in PEP-426 -[#pep426]_. +vulnerability. Markers were first standardised in :pep:`345`. This PEP +fixes some issues that were observed in the design described in :pep:`426`. Comparisons in marker expressions are typed by the comparison operator. The operators that are not in perform the same as they -do for strings in Python. The operators use the PEP-440 -[#pep440]_ version comparison rules when those are defined (that is when both -sides have a valid version specifier). If there is no defined PEP-440 +do for strings in Python. The operators use the :pep:`440` +version comparison rules when those are defined (that is when both +sides have a valid version specifier). If there is no defined :pep:`440` behaviour and the operator exists in Python, then the operator falls back to the Python behaviour. Otherwise an error should be raised. e.g. the following will result in errors:: @@ -241,7 +241,7 @@ variables. The "extra" variable is special. It is used by wheels to signal which specifications apply to a given extra in the wheel ``METADATA`` file, but -since the ``METADATA`` file is based on a draft version of PEP-426, there is +since the ``METADATA`` file is based on a draft version of :pep:`426`, there is no current specification for this. Regardless, outside of a context where this special handling is taking place, the "extra" variable should result in an error like all other unknown variables. @@ -316,13 +316,13 @@ concerns. There are however a few points where the PEP differs from the deployed base. -Firstly, PEP-440 direct references haven't actually been deployed in the wild, +Firstly, :pep:`440` direct references haven't actually been deployed in the wild, but they were designed to be compatibly added, and there are no known obstacles to adding them to pip or other tools that consume the existing dependency metadata in distributions - particularly since they won't be permitted to be present in PyPI uploaded distributions anyway. -Secondly, PEP-426 markers which have had some reasonable deployment, +Secondly, :pep:`426` markers which have had some reasonable deployment, particularly in wheels and pip, will handle version comparisons with ``python_full_version`` "2.7.10" differently. Specifically in 426 "2.7.10" is less than "2.7.9". This backward incompatibility is deliberate. We are also @@ -334,8 +334,8 @@ both features will need to make a judgement as to when support has become sufficiently widespread in the ecosystem that using them will not cause compatibility issues. -Thirdly, PEP-345 required brackets around version specifiers. In order to -accept PEP-345 dependency specifications, brackets are accepted, but they +Thirdly, :pep:`345` required brackets around version specifiers. In order to +accept :pep:`345` dependency specifications, brackets are accepted, but they should not be generated. Rationale @@ -547,18 +547,6 @@ References .. [#pip] pip, the recommended installer for Python packages (http://pip.readthedocs.org/en/stable/) -.. [#pep345] PEP-345, Python distribution metadata version 1.2. - (https://www.python.org/dev/peps/pep-0345/) - -.. [#pep426] PEP-426, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0426/) - -.. [#pep440] PEP-440, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0440/) - -.. [#std66] The URL specification. - (https://tools.ietf.org/html/rfc3986) - .. [#parsley] The parsley PEG library. (https://pypi.python.org/pypi/parsley/) diff --git a/pep-0509.txt b/peps/pep-0509.rst similarity index 95% rename from pep-0509.txt rename to peps/pep-0509.rst index b9e94a012..6f30f5ac7 100644 --- a/pep-0509.txt +++ b/peps/pep-0509.rst @@ -1,13 +1,17 @@ PEP: 509 Title: Add a private version to dict -Version: $Revision$ -Last-Modified: $Date$ Author: Victor Stinner -Status: Final +Status: Superseded Type: Standards Track Content-Type: text/x-rst Created: 04-Jan-2016 Python-Version: 3.6 +Post-History: `08-Jan-2016 `__, + `11-Jan-2016 `__, + `14-Apr-2016 `__, + `19-Apr-2016 `__ +Superseded-By: 699 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/QFVJV6YQOUSWIYY4FBORY647YCBSCIMQ/ Abstract @@ -55,8 +59,8 @@ is modified, the function uses a regular lookup, and maybe also deoptimizes the function (to remove the overhead of the guard check for next function calls). -See the `PEP 510 -- Specialized functions with guards -`_ for concrete usage of +See the :pep:`510 -- Specialized functions with guards <510>` +for concrete usage of guards to specialize functions and for a more general rationale on Python static optimizers. @@ -122,8 +126,7 @@ must also be invalidated. Specialized functions using guards ---------------------------------- -The `PEP 510 -- Specialized functions with guards -`_ proposes an API to support +:pep:`510` proposes an API to support specialized functions with guards. It allows to implement static optimizers for Python without breaking the Python semantics. @@ -408,10 +411,8 @@ example, it increases the size of each dictionary entry by 8 bytes on In Python, the memory footprint matters and the trend is to reduce it. Examples: -* `PEP 393 -- Flexible String Representation - `_ -* `PEP 412 -- Key-Sharing Dictionary - `_ +* :pep:`393` -- Flexible String Representation +* :pep:`412` -- Key-Sharing Dictionary Add a new dict subtype diff --git a/pep-0510.txt b/peps/pep-0510.rst similarity index 98% rename from pep-0510.txt rename to peps/pep-0510.rst index 70746c9ef..e96ba19e3 100644 --- a/pep-0510.txt +++ b/peps/pep-0510.rst @@ -125,11 +125,11 @@ Hypothetical myoptimizer module Examples in this PEP uses a hypothetical ``myoptimizer`` module which provides the following functions and types: -* ``specialize(func, code, guards)``: add the specialized code `code` - with guards `guards` to the function `func` +* ``specialize(func, code, guards)``: add the specialized code ``code`` + with guards ``guards`` to the function ``func`` * ``get_specialized(func)``: get the list of specialized codes as a list - of ``(code, guards)`` tuples where `code` is a callable or code object - and `guards` is a list of a guards + of ``(code, guards)`` tuples where ``code`` is a callable or code object + and ``guards`` is a list of a guards * ``GuardBuiltins(name)``: guard watching for ``builtins.__dict__[name]`` and ``globals()[name]``. The guard fails if ``builtins.__dict__[name]`` is replaced, or if ``globals()[name]`` @@ -323,7 +323,7 @@ The ``check()`` function checks a guard: *stack* is an array of arguments: indexed arguments followed by (*key*, *value*) pairs of keyword arguments. *na* is the number of indexed arguments. *nk* is the number of keyword arguments: the number of (*key*, -*value*) pairs. `stack` contains ``na + nk * 2`` objects. +*value*) pairs. ``stack`` contains ``na + nk * 2`` objects. Specialized code diff --git a/pep-0511.txt b/peps/pep-0511.rst similarity index 97% rename from pep-0511.txt rename to peps/pep-0511.rst index 29282b95c..223680716 100644 --- a/pep-0511.txt +++ b/peps/pep-0511.rst @@ -79,8 +79,7 @@ Example of optimizations which can be implemented with an AST optimizer: * `Dead code elimination `_ -Using guards (see the `PEP 510 -`_), it is possible to +Using guards (see :pep:`510`), it is possible to implement a much wider choice of optimizations. Examples: * Simplify iterable: replace ``range(3)`` with ``(0, 1, 2)`` when used @@ -137,11 +136,11 @@ Some examples: `_ * Domain Specific Language (DSL) like SQL queries. The Python language itself doesn't need to be modified. Previous attempts - to implement DSL for SQL like `PEP 335 - Overloadable Boolean - Operators `_ was rejected. + to implement DSL for SQL like :pep:`PEP 335 - Overloadable Boolean + Operators <335>` was rejected. * Pattern Matching of functional languages -* String Interpolation, but `PEP 498 -- Literal String Interpolation - `_ was merged into Python +* String Interpolation, but :pep:`498` + was merged into Python 3.6. `MacroPy `_ has a long list of @@ -549,7 +548,7 @@ Output:: Other Python implementations ============================ -The PEP 511 should be implemented by all Python implementation, but the +The :pep:`511` should be implemented by all Python implementation, but the bytecode and the AST are not standardized. By the way, even between minor version of CPython, there are changes on diff --git a/pep-0512.txt b/peps/pep-0512.rst similarity index 98% rename from pep-0512.txt rename to peps/pep-0512.rst index d61773be4..4284f089c 100644 --- a/pep-0512.txt +++ b/peps/pep-0512.rst @@ -44,7 +44,7 @@ the basic steps were: Rietveld code review tool [#rietveld]_. 6. Download the patch to make sure it still applies cleanly. 7. Run the test suite manually. -8. Update the `NEWS`, `ACKS`, and "What's New" document as necessary +8. Update the ``NEWS``, ``ACKS``, and "What's New" document as necessary 9. Pull changes to avoid a merge race. 10. Commit the change manually. 11. If the change was for a bugfix release, merge into the @@ -69,7 +69,7 @@ burdensome as it gets for a core developer to work with. Hence the decision was made in late 2014 that a move to a new development process was needed. A request for PEPs proposing new workflows was made, in the end leading to two: -PEP 481 and PEP 507 proposing GitHub [#github]_ and +:pep:`481` and :pep:`507` proposing GitHub [#github]_ and GitLab [#gitlab]_, respectively. The year 2015 was spent off-and-on working on those proposals and @@ -240,7 +240,7 @@ in the Knights Who Say Ni project [#ni]_. Make old repository read-only ''''''''''''''''''''''''''''' -Updating `.hg/hgrc` in the now-old Mercurial repository in the `[hooks]` +Updating ``.hg/hgrc`` in the now-old Mercurial repository in the ``[hooks]`` section with:: pretxnchangegroup.reject = echo " * This repo has been migrated to github.com/python/peps and does not accept new commits in Mercurial!" 2>&1; exit 1 @@ -262,8 +262,8 @@ Requirements for the cpython Repository Obviously the most active and important repository currently hosted at hg.python.org [#h.p.o]_ is the cpython -repository [#cpython-repo]_. Because of its importance and high- -frequency use, it requires more tooling before being moved to GitHub +repository [#cpython-repo]_. Because of its importance and +high-frequency use, it requires more tooling before being moved to GitHub compared to the other repositories mentioned in this PEP. @@ -435,7 +435,7 @@ http://docs.openstack.org/developer/reno/. Discussions at the Sep 2016 Python core-dev sprints led to this decision compared to the rejected approaches outlined in the -`Rejected Ideas` section of this PEP. The separate files approach +``Rejected Ideas`` section of this PEP. The separate files approach seems to have the right balance of flexibility and potential tooling out of the various options while solving the motivating problem. @@ -465,8 +465,8 @@ Work for this is being tracked at https://github.com/python/core-workflow/issues/7. -Create https://git.python.org -''''''''''''''''''''''''''''' +Create ``https://git.python.org`` +''''''''''''''''''''''''''''''''' Just as hg.python.org [#h.p.o]_ currently points to the Mercurial repository for Python, git.python.org should do the equivalent for @@ -739,7 +739,7 @@ Required: - Message #python-dev for each commit (PR or direct) (https://github.com/python/cpython/settings/hooks/new?service=irc) - Get docs built from git - (https://github.com/python/docsbuild-scripts/blob/master/build_docs.py already + (https://github.com/python/docsbuild-scripts/blob/main/build_docs.py already updated; https://github.com/python/psf-salt/pull/91 to switch) - Migrate buildbots to be triggered and pull from GitHub @@ -772,7 +772,7 @@ Optional features: - `Link web content back to files that it is generated from`_ - `Handling Misc/NEWS`_ - `Bot to generate cherry-pick pull requests`_ - - Write `.github/CONTRIBUTING.md` + - Write ``.github/CONTRIBUTING.md`` (to prevent PRs that are inappropriate from even showing up and pointing to the devguide) diff --git a/pep-0513.txt b/peps/pep-0513.rst similarity index 98% rename from pep-0513.txt rename to peps/pep-0513.rst index dd1f5f93f..bae964a83 100644 --- a/pep-0513.txt +++ b/peps/pep-0513.rst @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Robert T. McGibbon , Nathaniel J. Smith BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG +Discussions-To: distutils-sig@python.org Status: Superseded Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 19-Jan-2016 Post-History: 19-Jan-2016, 25-Jan-2016, 29-Jan-2016 @@ -29,7 +30,8 @@ Rationale ========= Currently, distribution of binary Python extensions for Windows and OS X is -straightforward. Developers and packagers build wheels [1]_ [2]_, which are +straightforward. Developers and packagers build wheels (:pep:`427`, :pep:`491`), +which are assigned platform tags such as ``win32`` or ``macosx_10_6_intel``, and upload these wheels to PyPI. Users can download and install these wheels using tools such as ``pip``. @@ -39,7 +41,7 @@ extension modules built on one Linux distribution will not work on other Linux distributions, or even on different machines running the same Linux distribution with different system libraries installed. -Build tools using PEP 425 platform tags [3]_ do not track information about the +Build tools using :pep:`425` platform tags do not track information about the particular Linux distribution or installed system libraries, and instead assign all wheels the too-vague ``linux_i686`` or ``linux_x86_64`` tags. Because of this ambiguity, there is no expectation that ``linux``-tagged built @@ -593,12 +595,6 @@ defer specifying these until we have more experience with the initial References ========== -.. [1] PEP 427 -- The Wheel Binary Package Format 1.0 - (https://www.python.org/dev/peps/pep-0427/) -.. [2] PEP 491 -- The Wheel Binary Package Format 1.9 - (https://www.python.org/dev/peps/pep-0491/) -.. [3] PEP 425 -- Compatibility Tags for Built Distributions - (https://www.python.org/dev/peps/pep-0425/) .. [4] Enthought Canopy Python Distribution (https://store.enthought.com/downloads/) .. [5] Continuum Analytics Anaconda Python Distribution diff --git a/pep-0514.txt b/peps/pep-0514.rst similarity index 99% rename from pep-0514.txt rename to peps/pep-0514.rst index 98ad939a7..dd1b63a0a 100644 --- a/pep-0514.txt +++ b/peps/pep-0514.rst @@ -38,7 +38,7 @@ Motivation When installed on Windows, the official Python installer creates a registry key for discovery and detection by other applications. This allows tools such as installers or IDEs to automatically detect and display a user's Python -installations. For example, the PEP 397 ``py.exe`` launcher and editors such as +installations. For example, the :pep:`397` ``py.exe`` launcher and editors such as PyCharm and Visual Studio already make use of this information. Third-party installers, such as those used by distributions, typically create @@ -171,7 +171,7 @@ name (preferred), a hostname, or as a last resort, a UUID would be appropriate:: HKEY_CURRENT_USER\Software\Python\www.example.com HKEY_CURRENT_USER\Software\Python\6C465E66-5A8C-4942-9E6A-D29159480C60 -The company name ``PyLauncher`` is reserved for the PEP 397 launcher +The company name ``PyLauncher`` is reserved for the :pep:`397` launcher (``py.exe``). It does not follow this convention and should be ignored by tools. If a string value named ``DisplayName`` exists, it should be used to identify diff --git a/pep-0515.txt b/peps/pep-0515.rst similarity index 100% rename from pep-0515.txt rename to peps/pep-0515.rst diff --git a/pep-0516.txt b/peps/pep-0516.rst similarity index 88% rename from pep-0516.txt rename to peps/pep-0516.rst index ee267aebc..1f66518e1 100644 --- a/pep-0516.txt +++ b/peps/pep-0516.rst @@ -1,13 +1,12 @@ PEP: 516 Title: Build system abstraction for pip/conda etc -Version: $Revision$ -Last-Modified: $Date$ Author: Robert Collins , - Nathaniel Smith + Nathaniel J. Smith BDFL-Delegate: Nick Coghlan -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 26-Oct-2015 Resolution: https://mail.python.org/pipermail/distutils-sig/2017-May/030517.html @@ -36,14 +35,14 @@ install build time requirements for packages which is an important step in getting pip to full feature parity with the installation components of easy-install. -As PEP-426 [#pep426]_ is draft, we cannot utilise the metadata format it -defined. However PEP-427 wheels are in wide use and fairly well specified, so +As :pep:`426` is draft, we cannot utilise the metadata format it +defined. However :pep:`427` wheels are in wide use and fairly well specified, so we have adopted the METADATA format from that for specifying distribution -dependencies and general project metadata. PEP-0508 [#pep508] provides a +dependencies and general project metadata. :pep:`508` provides a self-contained language for describing a dependency, which we encapsulate in a thin JSON schema to describe bootstrap dependencies. -Since Python sdists specified in PEP-0314 [#pep314] are also source trees, this +Since Python sdists specified in :pep:`314` are also source trees, this PEP is updating the definition of sdists. @@ -51,7 +50,7 @@ PEP Rejection ============= The CLI based approach proposed in this PEP has been rejected in favour of the -Python API based approach proposed in PEP 517. The specific CLI used to +Python API based approach proposed in :pep:`517`. The specific CLI used to communicate with build backends running as isolated subprocesses will be considered an implementation detail of front-end developer tool implementations. @@ -97,7 +96,7 @@ schema schema version. bootstrap_requires - Optional list of dependency specifications [#pep508] that must be + Optional list of :pep:`508` dependency specifications that must be installed before running the build tool. For instance, if using flit, then the requirements might be:: @@ -164,7 +163,7 @@ The examples below use a build_command of ``flit`` for illustrative purposes. build_requires Query build requirements. Build requirements are returned as a UTF-8 encoded JSON document with one key ``build_requires`` consisting of a list - of dependency specifications [#pep508]_. Additional keys must be + of :pep:`508` dependency specifications. Additional keys must be ignored. The build_requires command is the only command run without setting up a build environment. @@ -176,7 +175,7 @@ metadata Query project metadata. The metadata and only the metadata should be output on stdout in UTF-8 encoding. pip would run metadata just once to determine what other packages need to be downloaded and installed. The - metadata is output as a wheel METADATA file per PEP-427 [#pep427]_. + metadata is output as a wheel METADATA file per :pep:`427`. Note that the metadata generated by the metadata command, and the metadata present in a generated wheel must be identical. @@ -217,9 +216,10 @@ develop [--prefix PREFIX] flit develop --root /tmp/ --prefix /usr/local - Should install scripts within `/tmp/usr/local/bin`, even if the Python - environment in use reports that the sys.prefix is `/usr/` which would lead - to using `/tmp/usr/bin/`. Similar logic applies for package files etc. + Should install scripts within ``/tmp/usr/local/bin``, even if the Python + environment in use reports that the sys.prefix is ``/usr/`` which would + lead to using ``/tmp/usr/bin/``. + Similar logic applies for package files etc. The build environment --------------------- @@ -308,15 +308,15 @@ When 'pip' reads this it would prepare an environment with flit in it before trying to use flit. Because flit doesn't have setup-requires support today, -`flit build_requires` would just output a constant string:: +``flit build_requires`` would just output a constant string:: {"build_requires": []} -`flit metadata` would interrogate `flit.ini` and marshal the metadata into +``flit metadata`` would interrogate ``flit.ini`` and marshal the metadata into a wheel METADATA file and output that on stdout. -`flit wheel` would need to accept a `-d` parameter that tells it where to output the -wheel (pip needs this). +``flit wheel`` would need to accept a ``-d`` parameter that tells it where to +output the wheel (pip needs this). Backwards Compatibility ======================= @@ -350,7 +350,7 @@ run by pip, and that will try and fail to install A. As such we recommend that tools which are currently used as setup-requires either ensure that they keep a `setuptools shim`_ or find their consumers and -get them all to upgrade to the use of a `pypa.json` in advance of moving +get them all to upgrade to the use of a ``pypa.json`` in advance of moving themselves. Pragmatically that is impossible, so the advice is to keep a setuptools shim indefinitely - both for projects like pbr, setuptools_scm and also projects like numpy. @@ -382,7 +382,7 @@ a dependency, but otherwise deferred such to future iterations. We chose wheel METADATA files rather than defining a new specification, because pip can already handle wheel .dist-info directories which encode all -the necessary data in a METADATA file. PEP-426 can't be used as it's still +the necessary data in a METADATA file. :pep:`426` can't be used as it's still draft, and defining a new metadata format, while we should do that, is a separate problem. Using a directory on disk would not add any value to the interface (pip has to do that today due to limitations in the setuptools @@ -418,7 +418,7 @@ learning how to invoke a custom build tool. The metadata and wheel commands are required to have consistent metadata to avoid a race condition that could otherwise happen where pip reads the metadata, acts on it, and then the resulting wheel has incompatible -requirements. That race is exploited today by packages using PEP-426 +requirements. That race is exploited today by packages using :pep:`426` environment markers, to work with older pip versions that do not support environment markers. That exploit is not needed with this PEP, because either the setuptools shim is in use (with older pip versions), or an environment @@ -434,10 +434,10 @@ does today, and while there is a PR to do that as part of building from source, it is contentious and lacks consensus. Rather than impose a requirement on all build systems, we are treating it as a YAGNI, and will add such a verb in a future version of the interface if required. The existing -PEP-314 [#pep314] requirements for sdists still apply, and distutils or setuptools +:pep:`314` requirements for sdists still apply, and distutils or setuptools users can use ``setup.py sdist`` to create an sdist. Other tools should create -sdists compatible with PEP-314 [#pep314]. Note that pip itself does not require -PEP-314 compatibility - it does not use any of the metadata from sdists - they +sdists compatible with :pep:`314`. Note that pip itself does not require +:pep:`314` compatibility - it does not use any of the metadata from sdists - they are treated like source trees from disk or version control. References @@ -458,12 +458,6 @@ References .. [#shellvars] Shellvars, an implementation of shell variable rules for Python. (https://github.com/testing-cabal/shellvars) -.. [#pep426] PEP-426, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0426/) - -.. [#pep427] PEP-427, Python distribution metadata. - (https://www.python.org/dev/peps/pep-0427/) - .. [#thread] The kick-off thread. (https://mail.python.org/pipermail/distutils-sig/2015-October/026925.html) @@ -473,25 +467,8 @@ References .. [#strformat] The Python string formatting syntax. (https://docs.python.org/3.1/library/string.html#format-string-syntax) -.. [#pep314] Metadata for Python Software Packages v1.1 - (https://www.python.org/dev/peps/pep-0314/) - -.. [#pep508] Dependency specification language PEP. - (https://www.python.org/dev/peps/pep-0508/) - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0517.txt b/peps/pep-0517.rst similarity index 97% rename from pep-0517.txt rename to peps/pep-0517.rst index 7b32471ff..1ad562030 100644 --- a/pep-0517.txt +++ b/peps/pep-0517.rst @@ -5,12 +5,13 @@ Last-Modified: $Date$ Author: Nathaniel J. Smith , Thomas Kluyver BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 30-Sep-2015 -Post-History: 1 Oct 2015, 25 Oct 2015, 19 May 2017, 11 Sep 2017 +Post-History: 01-Oct-2015, 25-Oct-2015, 19-May-2017, 11-Sep-2017 Resolution: https://mail.python.org/pipermail/distutils-sig/2017-September/031548.html ========== @@ -50,16 +51,6 @@ We therefore propose a new, relatively minimal interface for installation tools like ``pip`` to interact with package source trees and source distributions. -======================= - Provisional acceptance -======================= - -In accordance with the PyPA's specification process, this PEP has been -`provisionally accepted `_ -for initial implementation in ``pip`` and other PyPA tools. - -During this time, the specification is still subject to revision based -on real world experience with those initial implementations. ======================= Terminology and goals @@ -111,7 +102,7 @@ specification is encoded in the source code and documentation of to it as the ``setup.py``\-style. Here we define a new style of source tree based around the -``pyproject.toml`` file defined in PEP 518, extending the +``pyproject.toml`` file defined in :pep:`518`, extending the ``[build-system]`` table in that file with one additional key, ``build-backend``. Here's an example of how it would look:: @@ -220,7 +211,7 @@ build_wheel :: - build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): ... Must build a .whl file, and place it in the specified ``wheel_directory``. It @@ -273,7 +264,7 @@ A .tar.gz source distribution (sdist) contains a single top-level directory call package. This directory must also contain the ``pyproject.toml`` from the build directory, and a PKG-INFO file containing metadata in the format described in -`PEP 345 `_. Although historically +:pep:`345`. Although historically zip files have also been used as sdists, this hook should produce a gzipped tarball. This is already the more common format for sdists, and having a consistent format makes for simpler tooling. @@ -305,7 +296,7 @@ get_requires_for_build_wheel def get_requires_for_build_wheel(config_settings=None): ... -This hook MUST return an additional list of strings containing PEP 508 +This hook MUST return an additional list of strings containing :pep:`508` dependency specifications, above and beyond those specified in the ``pyproject.toml`` file, to be installed when calling the ``build_wheel`` or ``prepare_metadata_for_build_wheel`` hooks. @@ -350,9 +341,9 @@ get_requires_for_build_sdist :: def get_requires_for_build_sdist(config_settings=None): - ... + ... -This hook MUST return an additional list of strings containing PEP 508 +This hook MUST return an additional list of strings containing :pep:`508` dependency specifications, above and beyond those specified in the ``pyproject.toml`` file. These dependencies will be installed when calling the ``build_sdist`` hook. @@ -383,9 +374,11 @@ dictionary provided as an "escape hatch" for users to pass ad-hoc configuration into individual package builds. Build backends MAY assign any semantics they like to this dictionary. Build frontends SHOULD provide some mechanism for users to specify arbitrary -string-key/string-value pairs to be placed in this dictionary. For -example, they might support some syntax like ``--package-config -CC=gcc``. Build frontends MAY also provide arbitrary other mechanisms +string-key/string-value pairs to be placed in this dictionary. +For example, they might support some syntax like ``--package-config CC=gcc``. +In case a user provides duplicate string-keys, build frontends SHOULD +combine the corresponding string-values into a list of strings. +Build frontends MAY also provide arbitrary other mechanisms for users to place entries in this dictionary. For example, ``pip`` might choose to map a mix of modern and legacy command line arguments like:: @@ -600,7 +593,7 @@ Integration frontends require that an sdist named ``{NAME}-{VERSION}.{EXT}`` will generate a wheel named ``{NAME}-{VERSION}-{COMPAT-INFO}.whl``. -The new restrictions for sdists built by PEP 517 backends are: +The new restrictions for sdists built by :pep:`517` backends are: - They will be gzipped tar archives, with the ``.tar.gz`` extension. Zip archives, or other compression formats for tarballs, are not allowed at @@ -698,7 +691,7 @@ implementation was released in pip 19.0. * Support for in-tree backends and self-hosting of backends was added by the introduction of the ``backend-path`` key in the ``[build-system]`` table. -* Clarified that the ``setuptools.build_meta:__legacy__`` PEP 517 backend is +* Clarified that the ``setuptools.build_meta:__legacy__`` :pep:`517` backend is an acceptable alternative to directly invoking ``setup.py`` for source trees that don't specify ``build-backend`` explicitly. @@ -712,7 +705,7 @@ has now been rejected in favour of this PEP. The primary difference is that our build backend is defined via a Python hook-based interface rather than a command-line based interface. -This appendix documents the arguments advanced for this PEP over PEP 516. +This appendix documents the arguments advanced for this PEP over :pep:`516`. We do *not* expect that specifying Python hooks rather than command line interfaces will, by itself, reduce the @@ -811,6 +804,7 @@ build backend:: import os.path import pathlib import shutil + import tarfile SDIST_NAME = "mypackage-0.1" SDIST_FILENAME = SDIST_NAME + ".tar.gz" @@ -940,7 +934,7 @@ process, it can easily write it to do something like:: In the alternative where the public interface boundary is placed at the subprocess call, this is not possible -- either we need to spawn an extra process just to query what interfaces are supported (as was -included in an earlier draft of PEP 516, an alternative to this), or +included in an earlier draft of :pep:`516`, an alternative to this), or else we give up on autonegotiation entirely (as in the current version of that PEP), meaning that any changes in the interface will require N individual packages to update their ``pyproject.toml`` files before diff --git a/pep-0518.txt b/peps/pep-0518.rst similarity index 97% rename from pep-0518.txt rename to peps/pep-0518.rst index 820d018d8..a2ce91924 100644 --- a/pep-0518.txt +++ b/peps/pep-0518.rst @@ -1,14 +1,13 @@ PEP: 518 Title: Specifying Minimum Build System Requirements for Python Projects -Version: $Revision$ -Last-Modified: $Date$ Author: Brett Cannon , - Nathaniel Smith , + Nathaniel J. Smith , Donald Stufft BDFL-Delegate: Nick Coghlan -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 10-May-2016 Post-History: 10-May-2016, @@ -119,7 +118,7 @@ To provide more context and motivation for this PEP, think of the 2. Installation of the build system. 3. Execute the build system. -This PEP covers step #2. PEP 517 covers step #3, including how to have +This PEP covers step #2. :pep:`517` covers step #3, including how to have the build system dynamically specify more dependencies that the build system requires to perform its job. The purpose of this PEP though, is to specify the minimal set of requirements for the build system to @@ -157,7 +156,7 @@ build-system table The ``[build-system]`` table is used to store build-related data. Initially only one key of the table will be valid and is mandatory for the table: ``requires``. This key must have a value of a list -of strings representing PEP 508 dependencies required to execute the +of strings representing :pep:`508` dependencies required to execute the build system (currently that means what dependencies are required to execute a ``setup.py`` file). @@ -268,7 +267,7 @@ Other table names Another name proposed for the ``[build-system]`` table was ``[build]``. The alternative name is shorter, but doesn't convey as -much of the intention of what information is store in the table. After +much of the intention of what information is stored in the table. After a vote on the distutils-sig mailing list, the current name won out. @@ -558,14 +557,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0519.txt b/peps/pep-0519.rst similarity index 99% rename from pep-0519.txt rename to peps/pep-0519.rst index af3583c5d..576c19e50 100644 --- a/pep-0519.txt +++ b/peps/pep-0519.rst @@ -44,7 +44,7 @@ file system path whether it actually represents a path or not. To help elevate the representation of file system paths from their representation as strings and bytes to a richer object representation, the pathlib module [#pathlib]_ was provisionally introduced in -Python 3.4 through PEP 428. While considered by some as an improvement +Python 3.4 through :pep:`428`. While considered by some as an improvement over strings and bytes for file system paths, it has suffered from a lack of adoption. Typically the key issue listed for the low adoption rate has been the lack of support in the standard library. This lack @@ -415,7 +415,7 @@ Much of the discussion that led to this PEP revolved around whether ``__fspath__()`` should be polymorphic and return ``bytes`` as well as ``str`` or only return ``str``. The general sentiment for this view was that ``bytes`` are difficult to work with due to their -inherent lack of information about their encoding and PEP 383 makes +inherent lack of information about their encoding and :pep:`383` makes it possible to represent all file system paths using ``str`` with the ``surrogateescape`` handler. Thus, it would be better to forcibly promote the use of ``str`` as the low-level path representation for diff --git a/pep-0520.txt b/peps/pep-0520.rst similarity index 91% rename from pep-0520.txt rename to peps/pep-0520.rst index 6fd52fdb3..978456b90 100644 --- a/pep-0520.txt +++ b/peps/pep-0520.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 07-Jun-2016 Python-Version: 3.6 -Post-History: 7-Jun-2016, 11-Jun-2016, 20-Jun-2016, 24-Jun-2016 +Post-History: 07-Jun-2016, 11-Jun-2016, 20-Jun-2016, 24-Jun-2016 Resolution: https://mail.python.org/pipermail/python-dev/2016-June/145442.html .. note:: @@ -37,8 +37,8 @@ This allows introspection of the original definition order, e.g. by class decorators. Additionally, this PEP requires that the default class definition -namespace be ordered (e.g. ``OrderedDict``) by default. The long- -lived class namespace (``__dict__``) will remain a ``dict``. +namespace be ordered (e.g. ``OrderedDict``) by default. The +long-lived class namespace (``__dict__``) will remain a ``dict``. Motivation @@ -93,7 +93,7 @@ with that: First, it requires the use of a metaclass. Metaclasses introduce an extra level of complexity to code and in some cases (e.g. conflicts) are a problem. So reducing the need for them is worth doing when the -opportunity presents itself. PEP 422 and PEP 487 discuss this at +opportunity presents itself. :pep:`422` and :pep:`487` discuss this at length. We have such an opportunity by using an ordered mapping (e.g. ``OrderedDict`` for CPython at least) for the default class definition namespace, virtually eliminating the need for ``__prepare__()``. @@ -185,10 +185,10 @@ the well-worn precedent found in Python. Per Guido:: Also, note that a writeable ``__definition_order__`` allows dynamically created classes (e.g. by Cython) to still have ``__definition_order__`` -properly set. That could certainly be handled through specific class- -creation tools, such as ``type()`` or the C-API, without the need to -lose the semantics of a read-only attribute. However, with a writeable -attribute it's a moot point. +properly set. That could certainly be handled through specific +class-creation tools, such as ``type()`` or the C-API, without the need +to lose the semantics of a read-only attribute. However, with a +writeable attribute it's a moot point. Why not "__attribute_order__"? @@ -344,8 +344,8 @@ be minimal. All conforming implementations are expected to set Implementation ============== -The implementation is found in the tracker. [impl_] - +The implementation is found in the +`tracker `__. Alternatives ============ @@ -373,9 +373,10 @@ as a pain point. A "namespace" Keyword Arg for Class Definition ---------------------------------------------- -PEP 422 introduced a new "namespace" keyword arg to class definitions -that effectively replaces the need to ``__prepare__()``. [pep422_] -However, the proposal was withdrawn in favor of the simpler PEP 487. +:pep:`PEP 422 <422#order-preserving-classes>` +introduced a new "namespace" keyword arg to class definitions +that effectively replaces the need to ``__prepare__()``. +However, the proposal was withdrawn in favor of the simpler :pep:`487`. A stdlib Metaclass that Implements __prepare__() with OrderedDict ----------------------------------------------------------------- @@ -409,27 +410,17 @@ discovery is almost entirely an implementation detail. References ========== -.. [impl] issue #24254 - (https://bugs.python.org/issue24254) +* `Original discussion + `__ -.. [nick_concern] Nick's concerns about mutability - (https://mail.python.org/pipermail/python-dev/2016-June/144883.html) +* `Follow-up 1 + `__ -.. [pep422] PEP 422 - (https://www.python.org/dev/peps/pep-0422/#order-preserving-classes) - -.. [pep487] PEP 487 - (https://www.python.org/dev/peps/pep-0487/#defining-arbitrary-namespaces) - -.. [orig] original discussion - (https://mail.python.org/pipermail/python-ideas/2013-February/019690.html) - -.. [followup1] follow-up 1 - (https://mail.python.org/pipermail/python-dev/2013-June/127103.html) - -.. [followup2] follow-up 2 - (https://mail.python.org/pipermail/python-dev/2015-May/140137.html) +* `Follow-up 2 + `__ +* `Nick Coghlan's concerns about mutability + `__ Copyright =========== diff --git a/pep-0521.txt b/peps/pep-0521.rst similarity index 99% rename from pep-0521.txt rename to peps/pep-0521.rst index fc6715702..e31cf36cd 100644 --- a/pep-0521.txt +++ b/peps/pep-0521.rst @@ -13,7 +13,7 @@ Post-History: 29-Apr-2015 PEP Withdrawal ============== -Withdrawn in favor of PEP 567. +Withdrawn in favor of :pep:`567`. Abstract @@ -68,7 +68,7 @@ More formally, consider the following code:: f((yield foo)) PARTIAL-BLOCK-2 -Currently this is equivalent to the following code (copied from PEP 343):: +Currently this is equivalent to the following code (copied from :pep:`343`):: mgr = (EXPR) exit = type(mgr).__exit__ # Not calling it yet @@ -308,7 +308,7 @@ down to a single C-level ``if (frame->needs_suspend_resume_calls) { Interaction with PEP 492 ======================== -PEP 492 added new asynchronous context managers, which are like +:pep:`492` added new asynchronous context managers, which are like regular context managers, but instead of having regular methods ``__enter__`` and ``__exit__`` they have coroutine methods ``__aenter__`` and ``__aexit__``. diff --git a/pep-0522.txt b/peps/pep-0522.rst similarity index 98% rename from pep-0522.txt rename to peps/pep-0522.rst index f60a26b12..be3690e75 100644 --- a/pep-0522.txt +++ b/peps/pep-0522.rst @@ -60,9 +60,9 @@ potentially be encountered in the following situations: Relationship with other PEPs ============================ -This PEP depends on the Accepted PEP 506, which adds the ``secrets`` module. +This PEP depends on the Accepted :pep:`506`, which adds the ``secrets`` module. -This PEP competes with Victor Stinner's PEP 524, which proposes to make +This PEP competes with Victor Stinner's :pep:`524`, which proposes to make ``os.urandom`` itself implicitly block when the system RNG is not ready. @@ -70,7 +70,7 @@ PEP Rejection ============= For the reference implementation, Guido rejected this PEP in favour of the -unconditional implicit blocking proposal in PEP 524 (which brings CPython's +unconditional implicit blocking proposal in :pep:`524` (which brings CPython's behaviour on Linux into line with its behaviour on other operating systems). This means any further discussion of appropriate default behaviour for @@ -86,7 +86,7 @@ CPython interpreter initialization and ``random`` module initialization have already been updated to gracefully fall back to alternative seeding options if the system random number generator is not ready. -This PEP does not compete with the proposal in PEP 524 to add an +This PEP does not compete with the proposal in :pep:`524` to add an ``os.getrandom()`` API to expose the ``getrandom`` syscall on platforms that offer it. There is sufficient motive for adding that API in the ``os`` module's role as a thin wrapper around potentially platform dependent operating system @@ -109,7 +109,7 @@ This behaviour will then propagate through to the existing ``random.SystemRandom``, which provides a relatively thin wrapper around ``os.urandom()`` that matches the ``random.Random()`` API. -However, the new ``secrets`` module introduced by PEP 506 will be updated to +However, the new ``secrets`` module introduced by :pep:`506` will be updated to catch the new exception and implicitly wait for the system random number generator if the exception is ever encountered. @@ -288,7 +288,7 @@ security sensitive operations in Python. To help improve API discoverability and make it clearer that secrecy and simulation are not the same problem (even though they both involve -random numbers), PEP 506 collected several of the one line recipes based +random numbers), :pep:`506` collected several of the one line recipes based on the lower level ``os.urandom()`` API into a new ``secrets`` module. However, this guidance has also come with a longstanding caveat: developers @@ -355,7 +355,7 @@ might unexpectedly start blocking waiting for the system RNG to be available. Backwards Compatibility Impact Assessment ========================================= -Similar to PEP 476, this is a proposal to turn a previously silent security +Similar to :pep:`476`, this is a proposal to turn a previously silent security failure into a noisy exception that requires the application developer to make an explicit decision regarding the behaviour they desire. @@ -675,7 +675,7 @@ decision has to be made: imported) * servicing user calls to the ``os.urandom`` public API * the higher level ``random.SystemRandom`` public API -* the new ``secrets`` module public API added by PEP 506 +* the new ``secrets`` module public API added by :pep:`506` Previously, these five places all used the same underlying code, and thus made this decision in the same way. @@ -714,7 +714,7 @@ as a side-effect of importing the random module. Independently of this PEP, the first two cases have already been updated to never block, regardless of the behaviour of ``os.urandom()``. -Where PEP 524 proposes to make all 3 of the latter cases block implicitly, +Where :pep:`524` proposes to make all 3 of the latter cases block implicitly, this PEP proposes that approach only for the last case (the ``secrets``) module, with ``os.urandom()`` and ``random.SystemRandom()`` instead raising an exception when they detect that the underlying operating system call diff --git a/pep-0523.txt b/peps/pep-0523.rst similarity index 98% rename from pep-0523.txt rename to peps/pep-0523.rst index 43c43f419..4ba6fb4b8 100644 --- a/pep-0523.txt +++ b/peps/pep-0523.rst @@ -1,13 +1,12 @@ PEP: 523 Title: Adding a frame evaluation API to CPython -Version: $Revision$ -Last-Modified: $Date$ Author: Brett Cannon , Dino Viehland Status: Final Type: Standards Track Content-Type: text/x-rst Created: 16-May-2016 +Python-Version: 3.6 Post-History: 16-May-2016 Resolution: https://mail.python.org/pipermail/python-dev/2016-August/145937.html @@ -324,7 +323,7 @@ mutable. The thinking seemed to be that having a field that was mutated after the creation of the code object made the object seem mutable, even though no other aspect of code objects changed. -The view of this PEP is that the `co_extra` field doesn't change the +The view of this PEP is that the ``co_extra`` field doesn't change the fact that code objects are immutable. The field is specified in this PEP to not contain information required to make the code object usable, making it more of a caching field. It could be viewed as @@ -389,14 +388,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0524.txt b/peps/pep-0524.rst similarity index 98% rename from pep-0524.txt rename to peps/pep-0524.rst index 6719fcd6a..03ff8d89c 100644 --- a/pep-0524.txt +++ b/peps/pep-0524.rst @@ -46,8 +46,8 @@ counter-measure against the hash denial-of-service (hash DoS), see: * `Issue #13703: Hash collision security issue `_ -* `PEP 456: Secure and interchangeable hash algorithm - `_ +* :pep:`PEP 456: Secure and interchangeable hash algorithm + <456>` Importing the ``random`` module creates an instance of ``random.Random``: ``random._inst``. On Python 3.5, random.Random @@ -405,8 +405,8 @@ Raise BlockingIOError in os.urandom() Proposition ^^^^^^^^^^^ -`PEP 522: Allow BlockingIOError in security sensitive APIs on Linux -`_. +:pep:`PEP 522: Allow BlockingIOError in security sensitive APIs on Linux +<522>`. Python should not decide for the developer how to handle `The bug`_: raising immediately a ``BlockingIOError`` if ``os.urandom()`` is going to diff --git a/peps/pep-0525-1.png b/peps/pep-0525-1.png new file mode 100644 index 000000000..356386fbb Binary files /dev/null and b/peps/pep-0525-1.png differ diff --git a/pep-0525.txt b/peps/pep-0525.rst similarity index 97% rename from pep-0525.txt rename to peps/pep-0525.rst index e8e64179a..dc7613b39 100644 --- a/pep-0525.txt +++ b/peps/pep-0525.rst @@ -3,7 +3,7 @@ Title: Asynchronous Generators Version: $Revision$ Last-Modified: $Date$ Author: Yury Selivanov -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -15,7 +15,7 @@ Post-History: 02-Aug-2016, 23-Aug-2016, 01-Sep-2016, 06-Sep-2016 Abstract ======== -PEP 492 introduced support for native coroutines and ``async``/``await`` +:pep:`492` introduced support for native coroutines and ``async``/``await`` syntax to Python 3.5. It is proposed here to extend Python's asynchronous capabilities by adding support for *asynchronous generators*. @@ -24,7 +24,7 @@ asynchronous capabilities by adding support for Rationale and Goals =================== -Regular generators (introduced in PEP 255) enabled an elegant way of +Regular generators (introduced in :pep:`255`) enabled an elegant way of writing complex *data producers* and have them behave like an iterator. However, currently there is no equivalent concept for the *asynchronous @@ -33,7 +33,7 @@ data producers unnecessarily complex, as one must define a class that implements ``__aiter__`` and ``__anext__`` to be able to use it in an ``async for`` statement. -Essentially, the goals and rationale for PEP 255, applied to the +Essentially, the goals and rationale for :pep:`255`, applied to the asynchronous execution case, hold true for this proposal as well. Performance is an additional point for this proposal: in our testing of @@ -80,7 +80,7 @@ This proposal introduces the concept of *asynchronous generators* to Python. This specification presumes knowledge of the implementation of -generators and coroutines in Python (PEP 342, PEP 380 and PEP 492). +generators and coroutines in Python (:pep:`342`, :pep:`380` and :pep:`492`). Asynchronous Generators @@ -107,7 +107,7 @@ We propose to use the same approach to define The result of calling an *asynchronous generator function* is an *asynchronous generator object*, which implements the asynchronous -iteration protocol defined in PEP 492. +iteration protocol defined in :pep:`492`. It is a ``SyntaxError`` to have a non-empty ``return`` statement in an asynchronous generator. @@ -143,7 +143,7 @@ iterate over a simple asynchronous generator:: Finalization ------------ -PEP 492 requires an event loop or a scheduler to run coroutines. +:pep:`492` requires an event loop or a scheduler to run coroutines. Because asynchronous generators are meant to be used from coroutines, they also require an event loop to run and finalize them. @@ -389,6 +389,7 @@ iterated: .. image:: pep-0525-1.png :align: center :width: 80% + :class: invert-in-dark-mode PyAsyncGenASend and PyAsyncGenAThrow @@ -517,7 +518,7 @@ Design Considerations ``aiter()`` and ``anext()`` builtins ------------------------------------ -Originally, PEP 492 defined ``__aiter__`` as a method that should +Originally, :pep:`492` defined ``__aiter__`` as a method that should return an *awaitable* object, resulting in an asynchronous iterator. However, in CPython 3.5.2, ``__aiter__`` was redefined to return @@ -618,7 +619,7 @@ print numbers from 0 to 9 with one second delay):: Acceptance ========== -PEP 525 was accepted by Guido, September 6, 2016 [2]_. +:pep:`525` was accepted by Guido, September 6, 2016 [2]_. Implementation diff --git a/pep-0526.txt b/peps/pep-0526.rst similarity index 96% rename from pep-0526.txt rename to peps/pep-0526.rst index 6308b3290..6529afbe2 100644 --- a/pep-0526.txt +++ b/peps/pep-0526.rst @@ -5,6 +5,7 @@ Last-Modified: $Date$ Author: Ryan Gonzalez , Philip House , Ivan Levkivskyi , Lisa Roach , Guido van Rossum Status: Final Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 09-Aug-2016 Python-Version: 3.6 @@ -30,13 +31,13 @@ There was preliminary discussion on python-ideas and at https://github.com/python/typing/issues/258. Before you bring up an objection in a public forum please at least -read the summary of `rejected`_ ideas listed at the end of this PEP. +read the summary of `rejected ideas `_ listed at the end of this PEP. Abstract ======== -PEP 484 introduced type hints, a.k.a. type annotations. While its +:pep:`484` introduced type hints, a.k.a. type annotations. While its main focus was function annotations, it also introduced the notion of type comments to annotate variables:: @@ -61,7 +62,7 @@ instead of expressing them through comments:: class Starship: stats: ClassVar[Dict[str, int]] = {} -PEP 484 explicitly states that type comments are intended to help with +:pep:`484` explicitly states that type comments are intended to help with type inference in complex cases, and this PEP does not change this intention. However, since in practice type comments have also been adopted for class variables and instance variables, this PEP also @@ -107,7 +108,7 @@ The majority of these issues can be alleviated by making the syntax a core part of the language. Moreover, having a dedicated annotation syntax for class and instance variables (in addition to method annotations) will pave the way to static duck-typing as a complement to nominal typing defined -by PEP 484. +by :pep:`484`. Non-goals ********* @@ -141,7 +142,7 @@ party type checker:: other_var: int = 'a' # Flagged as error by type checker, # but OK at runtime. -This syntax does not introduce any new semantics beyond PEP 484, so that +This syntax does not introduce any new semantics beyond :pep:`484`, so that the following three statements are equivalent:: var = value # type: annotation @@ -153,7 +154,7 @@ in different contexts and their runtime effects. We also suggest how type checkers might interpret annotations, but compliance to these suggestions is not mandatory. (This is in line -with the attitude towards compliance in PEP 484.) +with the attitude towards compliance in :pep:`484`.) Global and local variable annotations ************************************* @@ -394,8 +395,8 @@ Changes to Standard Library and Documentation instances. This restriction is ensured by static checkers, but not at runtime. See the `classvar`_ section for examples and explanations for the usage of - ``ClassVar``, and see the `rejected`_ section for more information on - the reasoning behind ``ClassVar``. + ``ClassVar``, and see the `rejected `_ section + for more information on the reasoning behind ``ClassVar``. - Function ``get_type_hints`` in the ``typing`` module will be extended, so that one can retrieve type annotations at runtime from modules @@ -407,7 +408,7 @@ Changes to Standard Library and Documentation - Recommended guidelines for using annotations will be added to the documentation, containing a pedagogical recapitulation of specifications - described in this PEP and in PEP 484. In addition, a helper script for + described in this PEP and in :pep:`484`. In addition, a helper script for translating type comments into type annotations will be published separately from the standard library. @@ -463,7 +464,7 @@ culprit, is accepted by the Python interpreter without questioning it The recommended way of getting annotations at runtime is by using ``typing.get_type_hints`` function; as with all dunder attributes, -any undocummented use of ``__annotations__`` is subject to breakage +any undocumented use of ``__annotations__`` is subject to breakage without warning:: from typing import Dict, ClassVar, get_type_hints @@ -510,14 +511,14 @@ These stored annotations might be used for other purposes, but with this PEP we explicitly recommend type hinting as the preferred use of annotations. -.. _rejected: +.. _PEP 526 rejected: Rejected/Postponed Proposals ============================ - **Should we introduce variable annotations at all?** Variable annotations have *already* been around for almost two years - in the form of type comments, sanctioned by PEP 484. They are + in the form of type comments, sanctioned by :pep:`484`. They are extensively used by third party type checkers (mypy, pytype, PyCharm, etc.) and by projects using the type checkers. However, the comment syntax has many downsides listed in Rationale. This PEP is @@ -632,11 +633,11 @@ Rejected/Postponed Proposals - **Do not evaluate annotations, treat them as strings:** This would be inconsistent with the behavior of function annotations that are always evaluated. Although this might be reconsidered in future, - it was decided in PEP 484 that this would have to be a separate PEP. + it was decided in :pep:`484` that this would have to be a separate PEP. - **Annotate variable types in class docstring:** Many projects already use various docstring conventions, often without - much consistency and generally without conforming to the PEP 484 annotation + much consistency and generally without conforming to the :pep:`484` annotation syntax yet. Also this would require a special sophisticated parser. This, in turn, would defeat the purpose of the PEP -- collaborating with the third party type checking tools. diff --git a/pep-0527.txt b/peps/pep-0527.rst similarity index 98% rename from pep-0527.txt rename to peps/pep-0527.rst index 53205c730..fbe682f77 100644 --- a/pep-0527.txt +++ b/peps/pep-0527.rst @@ -6,7 +6,8 @@ Author: Donald Stufft BDFL-Delegate: Nick Coghlan Discussions-To: distutils-sig@python.org Status: Final -Type: Process +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 23-Aug-2016 Post-History: 23-Aug-2016 @@ -96,7 +97,7 @@ a library using Python 3.5, then ``bdist_dumb`` will produce a ``.tar.gz`` file named something like ``exampleproject-1.0.macosx-10.11-x86_64.tar.gz``. Right off the bat this file name is somewhat difficult to differentiate from an ``sdist`` since they both use the same file extension (and with the legacy pre -PEP 440 versions, ``1.0-macosx-10.11-x86_64`` is a valid, although quite silly, +:pep:`440` versions, ``1.0-macosx-10.11-x86_64`` is a valid, although quite silly, version number). However, once you open up the created ``.tar.gz``, you'd find that there is no metadata inside that could be used for things like dependency discovery and in fact, it is quite simply a tarball containing hardcoded paths diff --git a/pep-0528.txt b/peps/pep-0528.rst similarity index 100% rename from pep-0528.txt rename to peps/pep-0528.rst diff --git a/pep-0529.txt b/peps/pep-0529.rst similarity index 99% rename from pep-0529.txt rename to peps/pep-0529.rst index 2e7264c1e..13c32f50c 100644 --- a/pep-0529.txt +++ b/peps/pep-0529.rst @@ -126,7 +126,7 @@ The use of utf-8 will not be configurable, except for the provision of a "legacy mode" flag to revert to the previous behaviour. The ``surrogateescape`` error mode does not apply here, as the concern is not -about retaining non-sensical bytes. Any path returned from the operating system +about retaining nonsensical bytes. Any path returned from the operating system will be valid Unicode, while invalid paths created by the user should raise a decoding error (currently these would raise ``OSError`` or a subclass). diff --git a/pep-0530.txt b/peps/pep-0530.rst similarity index 94% rename from pep-0530.txt rename to peps/pep-0530.rst index 34b62b3f2..a646f9f4c 100644 --- a/pep-0530.txt +++ b/peps/pep-0530.rst @@ -3,7 +3,7 @@ Title: Asynchronous Comprehensions Version: $Revision$ Last-Modified: $Date$ Author: Yury Selivanov -Discussions-To: +Discussions-To: python-dev@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -15,7 +15,7 @@ Post-History: 03-Sep-2016 Abstract ======== -PEP 492 and PEP 525 introduce support for native coroutines and +:pep:`492` and :pep:`525` introduce support for native coroutines and asynchronous generators using ``async`` / ``await`` syntax. This PEP proposes to add asynchronous versions of list, set, dict comprehensions and generator expressions. @@ -55,7 +55,7 @@ Asynchronous Comprehensions --------------------------- We propose to allow using ``async for`` inside list, set and dict -comprehensions. Pending PEP 525 approval, we can also allow creation +comprehensions. Pending :pep:`525` approval, we can also allow creation of asynchronous generator expressions. Examples: @@ -131,7 +131,7 @@ The proposal is fully backwards compatible. Acceptance ========== -PEP 530 was accepted by Guido, September 6, 2016 [1]_. +:pep:`530` was accepted by Guido, September 6, 2016 [1]_. Implementation diff --git a/pep-0531.txt b/peps/pep-0531.rst similarity index 97% rename from pep-0531.txt rename to peps/pep-0531.rst index c7eef2e1a..13be86600 100644 --- a/pep-0531.txt +++ b/peps/pep-0531.rst @@ -13,7 +13,7 @@ Post-History: 28-Oct-2016 Abstract ======== -Inspired by PEP 505 and the related discussions, this PEP proposes the addition +Inspired by :pep:`505` and the related discussions, this PEP proposes the addition of two new control flow operators to Python: * Existence-checking precondition ("exists-then"): ``expr1 ?then expr2`` @@ -55,7 +55,7 @@ following characteristics: PEP Withdrawal ============== -When posting this PEP for discussion on python-ideas [4_], I asked reviewers to +When posting this PEP for discussion on python-ideas [4]_, I asked reviewers to consider 3 high level design questions before moving on to considering the specifics of this particular syntactic proposal: @@ -74,7 +74,7 @@ truth-checking "and" and "or" control flow operators were available? While the answers to the first question were generally positive, it quickly became clear that the answer to the second question is "No". -Steven D'Aprano articulated the counter-argument well in [5_], but the general +Steven D'Aprano articulated the counter-argument well in [5]_, but the general idea is that when checking for "missing data" sentinels, we're almost always looking for a *specific* sentinel value, rather than *any* sentinel value. @@ -96,15 +96,15 @@ make sense, and it is accordingly withdrawn. However, the discussion of the proposal did prompt consideration of a potential protocol based approach to make the existing ``and``, ``or`` and ``if-else`` -operators more flexible [6_] without introducing any new syntax, so I'll be -writing that up as another possible alternative to PEP 505. +operators more flexible [6]_ without introducing any new syntax, so I'll be +writing that up as another possible alternative to :pep:`505`. Relationship with other PEPs ============================ While this PEP was inspired by and builds on Mark Haase's excellent work in -putting together PEP 505, it ultimately competes with that PEP due to +putting together :pep:`505`, it ultimately competes with that PEP due to significant differences in the specifics of the proposed syntax and semantics for the feature. @@ -313,7 +313,7 @@ truth check". Taking that path would also have the advantage of aligning Python's syntax with corresponding syntax in other languages that offer similar features. -Drawing from the existing summary in PEP 505 and the Wikipedia articles on +Drawing from the existing summary in :pep:`505` and the Wikipedia articles on the "safe navigation operator [1]_ and the "null coalescing operator" [2]_, we see: @@ -572,7 +572,7 @@ protocol definition significantly more complex, both to define and to use: a singleton tuple containing a reference to the object to be used as the result of the existence check * at the C layer, ``tp_exists`` implementations would return NULL to indicate - non-existence, and otherwise return a `PyObject *` pointer as the + non-existence, and otherwise return a ``PyObject *`` pointer as the result of the existence check Given that change, the sentinel object idiom could be rewritten as:: @@ -608,7 +608,7 @@ sufficient to guide the creation of a reference implementation. Implementation ============== -As with PEP 505, actual implementation has been deferred pending in-principle +As with :pep:`505`, actual implementation has been deferred pending in-principle interest in the idea of adding these operators - the implementation isn't the hard part of these proposals, the hard part is deciding whether or not this is a change where the long term benefits for new and existing Python users @@ -645,13 +645,3 @@ Copyright This document has been placed in the public domain under the terms of the CC0 1.0 license: https://creativecommons.org/publicdomain/zero/1.0/ - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0532.txt b/peps/pep-0532.rst similarity index 95% rename from pep-0532.txt rename to peps/pep-0532.rst index fa92016d3..9b974a8d0 100644 --- a/pep-0532.txt +++ b/peps/pep-0532.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 30-Oct-2016 Python-Version: 3.8 -Post-History: 5-Nov-2016 +Post-History: 05-Nov-2016 PEP Deferral ============ @@ -20,7 +20,7 @@ earliest. Abstract ======== -Inspired by PEP 335, PEP 505, PEP 531, and the related discussions, this PEP +Inspired by :pep:`335`, :pep:`505`, :pep:`531`, and the related discussions, this PEP proposes the definition of a new circuit breaking protocol (using the method names ``__then__`` and ``__else__``) that provides a common underlying semantic foundation for: @@ -28,8 +28,8 @@ semantic foundation for: * conditional expressions: ``LHS if COND else RHS`` * logical conjunction: ``LHS and RHS`` * logical disjunction: ``LHS or RHS`` -* the None-aware operators proposed in PEP 505 -* the rich comparison chaining model proposed in PEP 535 +* the None-aware operators proposed in :pep:`505` +* the rich comparison chaining model proposed in :pep:`535` Taking advantage of the new protocol, it further proposes that the definition of conditional expressions be revised to also permit the use of ``if`` and @@ -69,7 +69,7 @@ the key proposals are discussed below. PEP 531: Existence checking protocol ------------------------------------ -This PEP is a direct successor to PEP 531, replacing the existence checking +This PEP is a direct successor to :pep:`531`, replacing the existence checking protocol and the new ``?then`` and ``?else`` syntactic operators defined there with the new circuit breaking protocol and adjustments to conditional expressions and the ``not`` operator. @@ -78,7 +78,7 @@ expressions and the ``not`` operator. PEP 505: None-aware operators ----------------------------- -This PEP complements the None-aware operator proposals in PEP 505, by offering +This PEP complements the None-aware operator proposals in :pep:`505`, by offering an underlying protocol-driven semantic framework that explains their short-circuiting behaviour as highly optimised syntactic sugar for particular uses of conditional expressions. @@ -102,24 +102,24 @@ more general protocol-driven proposal in this PEP. PEP 335: Overloadable Boolean operators --------------------------------------- -PEP 335 proposed the ability to overload the short-circuiting ``and`` and +:pep:`335` proposed the ability to overload the short-circuiting ``and`` and ``or`` operators directly, with the ability to overload the semantics of comparison chaining being one of the consequences of that change. The proposal in an earlier version of this PEP to instead handle the element-wise comparison use case by changing the semantic definition of comparison chaining -is drawn directly from Guido's rejection of PEP 335 [1_]. +is drawn directly from Guido's rejection of :pep:`335` [1]_. However, initial feedback on this PEP indicated that the number of different proposals that it covered made it difficult to read, so that part of the -proposal has been separated out as PEP 535. +proposal has been separated out as :pep:`535`. PEP 535: Rich comparison chaining --------------------------------- -As noted above, PEP 535 is a proposal to build on the circuit breaking protocol +As noted above, :pep:`535` is a proposal to build on the circuit breaking protocol defined in this PEP in order to expand the rich comparison support introduced -in PEP 207 to also handle comparison chaining operations like +in :pep:`207` to also handle comparison chaining operations like ``LEFT_BOUND < VALUE < RIGHT_BOUND``. @@ -167,7 +167,7 @@ the semantics of conditional expressions. Rather, it comes from the proposed addition of ``if`` and ``else`` as general purpose protocol driven short-circuiting operators to complement the existing ``True`` and ``False`` based short-circuiting operators (``or`` and ``and``, respectively) as well -as the ``None`` based short-circuiting operator proposed in PEP 505 (``??``). +as the ``None`` based short-circuiting operator proposed in :pep:`505` (``??``). Together, these two operators would be known as the circuit breaking operators. @@ -192,7 +192,7 @@ expressions themselves do. This grammar definition means precedence/associativity in the otherwise ambiguous case of ``expr1 if cond else expr2 else expr3`` resolves as ``(expr1 if cond else expr2) else epxr3``. However, a guideline will also be -added to PEP 8 to say "don't do that", as such a construct will be inherently +added to :pep:`8` to say "don't do that", as such a construct will be inherently confusing for readers, regardless of how the interpreter executes it. The right-associative circuit breaking operator (``LHS if RHS``) would then @@ -397,7 +397,7 @@ breaking operator instead: None-aware operators -------------------- -If both this PEP and PEP 505's None-aware operators were accepted, then the +If both this PEP and :pep:`505`'s None-aware operators were accepted, then the proposed ``is_sentinel`` and ``is_not_sentinel`` circuit breaker factories would be used to encapsulate the notion of "None checking": seeing if a value is ``None`` and either falling back to an alternative value (an operation known @@ -438,7 +438,7 @@ behaviour of the operators at runtime. Rich chained comparisons ------------------------ -Refer to PEP 535 for a detailed discussion of this possible use case. +Refer to :pep:`535` for a detailed discussion of this possible use case. Other conditional constructs @@ -471,7 +471,7 @@ rather than assigning ``CONDITION`` to the given name directly. Style guide recommendations --------------------------- -The following additions to PEP 8 are proposed in relation to the new features +The following additions to :pep:`8` are proposed in relation to the new features introduced by this PEP: * Avoid combining conditional expressions (``if-else``) and the standalone @@ -490,7 +490,7 @@ Rationale Adding new operators -------------------- -Similar to PEP 335, early drafts of this PEP focused on making the existing +Similar to :pep:`335`, early drafts of this PEP focused on making the existing ``and`` and ``or`` operators less rigid in their interpretation, rather than proposing new operators. However, this proved to be problematic for a few key reasons: @@ -554,7 +554,7 @@ operators are combined with the explicit ``if-else`` conditional expression syntax. The PEP handles that ambiguity by explicitly specifying how it should be -handled by interpreter implementers, but proposing to point out in PEP 8 +handled by interpreter implementers, but proposing to point out in :pep:`8` that even though interpreters will understand it, human readers probably won't, and hence it won't be a good idea to use both conditional expressions and the circuit breaking operators in a single expression. @@ -660,12 +660,12 @@ raised when discussing PEPs 335, 505 and 531. One consequence of this approach is that this PEP *on its own* doesn't produce much in the way of direct benefits to end users aside from making it possible -to omit some common ``None if `` prefixes and `` else None`` suffixes from +to omit some common ``None if`` prefixes and ``else None`` suffixes from particular forms of conditional expression. Instead, what it mainly provides is a common foundation that would allow the -None-aware operator proposals in PEP 505 and the rich comparison chaining -proposal in PEP 535 to be pursued atop a common underlying semantic framework +None-aware operator proposals in :pep:`505` and the rich comparison chaining +proposal in :pep:`535` to be pursued atop a common underlying semantic framework that would also be shared with conditional expressions and the existing ``and`` and ``or`` operators. @@ -681,6 +681,7 @@ breaking protocol (although it glosses over the technical detail of looking up the special methods via the type rather than the instance): .. image:: pep-0532/circuit-breaking-protocol.svg + :class: invert-in-dark-mode :alt: diagram of circuit breaking protocol applied to ternary expression We will work through the following expression:: @@ -759,13 +760,13 @@ expression as follows:: >>> maybe_value = is_not_none(data.get("key")) >>> maybe_value if maybe_value else y -If `bool(maybe_value)`` is ``True``, then the expression short-circuits and +If ``bool(maybe_value)`` is ``True``, then the expression short-circuits and the interpreter calls ``type(maybe_value).__else__(maybe_value, maybe_value)``. The implementation of ``types.CircuitBreaker.__then__`` detects that the instance method has received itself as its argument and returns the wrapped value (i.e. ``data.get("key")``) rather than the circuit breaker. -If `bool(maybe_value)`` is ``True``, the interpreter calls +If ``bool(maybe_value)`` is ``True``, the interpreter calls ``type(maybe_value).__else__(maybe_value, y)``. The implementation of ``types.CircuitBreaker.__else__`` doesn't see anything that indicates short-circuiting has taken place, and hence returns ``y``. @@ -865,7 +866,7 @@ created explicitly via API calls, and are never produced implicitly. Implementation ============== -As with PEP 505, actual implementation has been deferred pending in-principle +As with :pep:`505`, actual implementation has been deferred pending in-principle interest in the idea of making these changes. ...TBD... @@ -874,9 +875,9 @@ interest in the idea of making these changes. Acknowledgements ================ -Thanks go to Steven D'Aprano for his detailed critique [2_] of the initial +Thanks go to Steven D'Aprano for his detailed critique [2]_ of the initial draft of this PEP that inspired many of the changes in the second draft, as -well as to all of the other participants in that discussion thread [3_] +well as to all of the other participants in that discussion thread [3]_. References @@ -896,13 +897,3 @@ Copyright This document has been placed in the public domain under the terms of the CC0 1.0 license: https://creativecommons.org/publicdomain/zero/1.0/ - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0532/circuit-breaking-protocol.svg b/peps/pep-0532/circuit-breaking-protocol.svg similarity index 100% rename from pep-0532/circuit-breaking-protocol.svg rename to peps/pep-0532/circuit-breaking-protocol.svg diff --git a/pep-0533.txt b/peps/pep-0533.rst similarity index 98% rename from pep-0533.txt rename to peps/pep-0533.rst index f7c6df031..1299a03d1 100644 --- a/pep-0533.txt +++ b/peps/pep-0533.rst @@ -37,13 +37,13 @@ Background and motivation ========================= Python iterables often hold resources which require cleanup. For -example: ``file`` objects need to be closed; the `WSGI spec -`_ adds a ``close`` method +example: ``file`` objects need to be closed; the :pep:`WSGI spec +<333>` adds a ``close`` method on top of the regular iterator protocol and demands that consumers call it at the appropriate time (though forgetting to do so is a `frequent source of bugs `_); -and PEP 342 (based on PEP 325) extended generator objects to add a +and :pep:`342` (based on :pep:`325`) extended generator objects to add a ``close`` method to allow generators to clean up after themselves. Generally, objects that need to clean up after themselves also define @@ -59,12 +59,12 @@ several cases: file descriptor exhaustion, or WSGI timing middleware that collects bogus times. -- Async generators (PEP 525) can only perform cleanup under the +- Async generators (:pep:`525`) can only perform cleanup under the supervision of the appropriate coroutine runner. ``__del__`` doesn't have access to the coroutine runner; indeed, the coroutine runner might be garbage collected before the generator object. So relying on the garbage collector is effectively impossible without some kind - of language extension. (PEP 525 does provide such an extension, but + of language extension. (:pep:`525` does provide such an extension, but it has a number of limitations that this proposal fixes; see the "alternatives" section below for discussion.) @@ -180,14 +180,14 @@ Alternatives PEP 525 asyncgen hooks ---------------------- -PEP 525 `proposes a set of global thread-local hooks -`_ +:pep:`PEP 525 proposes a set of global thread-local hooks +<525#finalization>` managed by new ``sys.{get/set}_asyncgen_hooks()`` functions, which allow event loops to integrate with the garbage collector to run -cleanup for async generators. In principle, this proposal and PEP 525 +cleanup for async generators. In principle, this proposal and :pep:`525` are complementary, in the same way that ``with`` blocks and ``__del__`` are complementary: this proposal takes care of ensuring -deterministic cleanup in most cases, while PEP 525's GC hooks clean up +deterministic cleanup in most cases, while :pep:`525`'s GC hooks clean up anything that gets missed. But ``__aiterclose__`` provides a number of advantages over GC hooks alone: @@ -730,7 +730,7 @@ very soon), so the async changes do not produce any backwards incompatibilities. (There is existing code using async iterators, but using the new async for loop on an old async iterator is harmless, because old async iterators don't have ``__aiterclose__``.) In -addition, PEP 525 was accepted on a provisional basis, and async +addition, :pep:`525` was accepted on a provisional basis, and async generators are by far the biggest beneficiary of this PEP's proposed changes. Therefore, I think we should strongly consider enabling ``__aiterclose__`` for ``async for`` loops and async generators ASAP, diff --git a/pep-0534.txt b/peps/pep-0534.rst similarity index 97% rename from pep-0534.txt rename to peps/pep-0534.rst index b6e649fbe..a572180fd 100644 --- a/pep-0534.txt +++ b/peps/pep-0534.rst @@ -1,7 +1,5 @@ PEP: 534 Title: Improved Errors for Missing Standard Library Modules -Version: $Revision$ -Last-Modified: $Date$ Author: TomĂĄĆĄ Orsava , Petr Viktorin , Nick Coghlan @@ -116,7 +114,7 @@ with two additional functions: * ``sysconfig.get_optional_modules()``, which will list optional public top level standard library module names -The results of ``sysconfig.get_optional_modules()``and the existing +The results of ``sysconfig.get_optional_modules()`` and the existing ``sys.builtin_module_names`` will both be subsets of the full list provided by the new ``sysconfig.get_stdlib_modules()`` function. @@ -328,7 +326,7 @@ a proper, distro-specific way to install missing standard library modules upon encountering a `ModuleNotFoundError`_. Some downstream distributors are already using this method of patching -`sys.excepthook` to integrate with platform crash reporting mechanisms. +``sys.excepthook`` to integrate with platform crash reporting mechanisms. .. _`site.py`: https://docs.python.org/3.7/library/site.html .. _`sitecustomize.py`: `site.py`_ @@ -374,14 +372,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0535.txt b/peps/pep-0535.rst similarity index 91% rename from pep-0535.txt rename to peps/pep-0535.rst index ccb028f04..420cfac69 100644 --- a/pep-0535.txt +++ b/peps/pep-0535.rst @@ -20,8 +20,8 @@ earliest. Abstract ======== -Inspired by PEP 335, and building on the circuit breaking protocol described -in PEP 532, this PEP proposes a change to the definition of chained comparisons, +Inspired by :pep:`335`, and building on the circuit breaking protocol described +in :pep:`532`, this PEP proposes a change to the definition of chained comparisons, where the comparison chaining will be updated to use the left-associative circuit breaking operator (``else``) rather than the logical disjunction operator (``and``) if the left hand comparison returns a circuit breaker as @@ -37,13 +37,13 @@ or tautologically returning ``True`` (indicating a non-empty matrix). Relationship with other PEPs ============================ -This PEP has been extracted from earlier iterations of PEP 532, as a +This PEP has been extracted from earlier iterations of :pep:`532`, as a follow-on use case for the circuit breaking protocol, rather than an essential part of its introduction. The specific proposal in this PEP to handle the element-wise comparison use case by changing the semantic definition of comparison chaining is drawn -directly from Guido's rejection of PEP 335. +directly from Guido's rejection of :pep:`335`. Specification @@ -59,7 +59,7 @@ is currently roughly semantically equivalent to:: _lhs_result = LEFT_BOUND LEFT_OP _expr _expr_result = _lhs_result and (_expr RIGHT_OP RIGHT_BOUND) -Using the circuit breaking concepts introduced in PEP 532, this PEP proposes +Using the circuit breaking concepts introduced in :pep:`532`, this PEP proposes that comparison chaining be changed to explicitly check if the left comparison returns a circuit breaker, and if so, use ``else`` rather than ``and`` to implement the comparison chaining:: @@ -81,7 +81,7 @@ operations would be the same as the existing expansion for ``and``. Rationale ========= -In ultimately rejecting PEP 335, Guido van Rossum noted [1_]: +In ultimately rejecting :pep:`335`, Guido van Rossum noted [1]_: The NumPy folks brought up a somewhat separate issue: for them, the most common use case is chained comparisons (e.g. A < B < C). @@ -171,7 +171,7 @@ Implementation ============== Actual implementation has been deferred pending in-principle interest in the -idea of making the changes proposed in PEP 532. +idea of making the changes proposed in :pep:`532`. ...TBD... @@ -187,13 +187,3 @@ Copyright This document has been placed in the public domain under the terms of the CC0 1.0 license: https://creativecommons.org/publicdomain/zero/1.0/ - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0536.txt b/peps/pep-0536.rst similarity index 94% rename from pep-0536.txt rename to peps/pep-0536.rst index db012665d..f4a6ba3c6 100644 --- a/pep-0536.txt +++ b/peps/pep-0536.rst @@ -13,13 +13,13 @@ Post-History: 12-Dec-2016 Abstract ======== -PEP 498 introduced Literal String Interpolation (or “f-strings”). +:pep:`498` introduced Literal String Interpolation (or “f-strings”). The expression portions of those literals however are subject to certain restrictions. This PEP proposes a formal grammar lifting those restrictions, promoting “f-strings” to “f expressions” or f-literals. -This PEP expands upon the f-strings introduced by PEP 498, -so this text requires familiarity with PEP 498. +This PEP expands upon the f-strings introduced by :pep:`498`, +so this text requires familiarity with :pep:`498`. PEP Status ========== @@ -81,7 +81,7 @@ Rationale The restrictions mentioned in Motivation_ are non-obvious and counter-intuitive unless the user is familiar with the f-literals’ implementation details. -As mentioned, a previous version of PEP 498 allowed escape sequences +As mentioned, a previous version of :pep:`498` allowed escape sequences anywhere in f-strings, including as ways to encode the braces delimiting the expression portions and in their code. They would be expanded before the code is parsed, which would have had several important ramifications: @@ -124,7 +124,7 @@ expressions instead of just variable names. [2]_ Specification ============= -PEP 498 specified f-strings as the following, but places restrictions on it:: +:pep:`498` specified f-strings as the following, but places restrictions on it:: f ' { } ... ' @@ -143,7 +143,7 @@ as explained below: syntactically valid. The first ``':'`` or ``'!'`` that is not part of an expression has to be followed a valid coercion or format specifier. -A remaining restriction not explicitly mentioned by PEP 498 is line breaks +A remaining restriction not explicitly mentioned by :pep:`498` is line breaks in expression portions. Since strings delimited by single ``'`` or ``"`` characters are expected to be single line, line breaks remain illegal in expression portions of single line strings. diff --git a/pep-0537.txt b/peps/pep-0537.rst similarity index 83% rename from pep-0537.txt rename to peps/pep-0537.rst index 2a2d766a8..f20161363 100644 --- a/pep-0537.txt +++ b/peps/pep-0537.rst @@ -3,8 +3,9 @@ Title: Python 3.7 Release Schedule Version: $Revision$ Last-Modified: $Date$ Author: Ned Deily -Status: Active +Status: Final Type: Informational +Topic: Release Content-Type: text/x-rst Created: 23-Dec-2016 Python-Version: 3.7 @@ -17,10 +18,6 @@ This document describes the development and release schedule for Python 3.7. The schedule primarily concerns itself with PEP-sized items. -.. Small features may be added up to the first beta - release. Bugs may be fixed until the final release, - which is planned for 2018-06. - Release Manager and Crew ======================== @@ -42,6 +39,12 @@ After that, it is expected that (source only) will be released as needed until 5 years after the release of 3.7 final, so until approximately 2023-06. +As of 2023-06-27, 3.7 has reached the +`end-of-life phase `_ +of its release cycle. 3.7.17 was the final security release. The code base for +3.7 is now frozen and no further updates will be provided nor issues of any +kind will be accepted on the bug tracker. + Release Schedule ================ @@ -139,12 +142,30 @@ releases are planned. - 3.7.12 final: 2021-09-04 -3.7.13 and beyond schedule --------------------------- +3.7.13 schedule +--------------- -Security fixes only, as needed, until 2023-06 +- 3.7.13 final: 2022-03-16 -- TBD +3.7.14 schedule +--------------- + +- 3.7.14 final: 2022-09-06 + +3.7.15 schedule +--------------- + +- 3.7.15 final: 2022-10-11 + +3.7.16 schedule +--------------- + +- 3.7.16 final: 2022-12-06 + +3.7.17 schedule (last security-only release) +-------------------------------------------- + +- 3.7.17 final: 2023-06-06 Features for 3.7 diff --git a/pep-0538.txt b/peps/pep-0538.rst similarity index 94% rename from pep-0538.txt rename to peps/pep-0538.rst index 1fca7afc3..cf3df04b2 100644 --- a/pep-0538.txt +++ b/peps/pep-0538.rst @@ -9,10 +9,10 @@ Type: Standards Track Content-Type: text/x-rst Created: 28-Dec-2016 Python-Version: 3.7 -Post-History: 03-Jan-2017 (linux-sig), - 07-Jan-2017 (python-ideas), - 05-Mar-2017 (python-dev), - 09-May-2017 (python-dev) +Post-History: 03-Jan-2017, + 07-Jan-2017, + 05-Mar-2017, + 09-May-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-May/148035.html Abstract @@ -26,7 +26,7 @@ implies a default text encoding of ASCII, which is entirely inadequate for the development of networked services and client applications in a multilingual world. -PEP 540 proposes a change to CPython's handling of the legacy C locale such +:pep:`540` proposes a change to CPython's handling of the legacy C locale such that CPython will assume the use of UTF-8 in such environments, rather than persisting with the demonstrably problematic assumption of ASCII as an appropriate encoding for communicating with operating system interfaces. @@ -46,7 +46,7 @@ works, rather than relying primarily on existing configuration settings that are supported by Python versions prior to Python 3.7. Accordingly, this PEP proposes that independently of the UTF-8 mode proposed -in PEP 540, the way the CPython implementation handles the default C locale be +in :pep:`540`, the way the CPython implementation handles the default C locale be changed to be roughly equivalent to the following existing configuration settings (supported since Python 3.1):: @@ -70,7 +70,7 @@ With this change, any \*nix platform that does *not* offer at least one of the ``C.UTF-8``, ``C.utf8`` or ``UTF-8`` locales as part of its standard configuration would only be considered a fully supported platform for CPython 3.7+ deployments when a suitable locale other than the default ``C`` locale is -configured explicitly (e.g. ``en_AU.UTF-8``, ``zh_CN.gb18030``). If PEP 540 is +configured explicitly (e.g. ``en_AU.UTF-8``, ``zh_CN.gb18030``). If :pep:`540` is accepted in addition to this PEP, then pure Python modules would also be supported when using the proposed ``PYTHONUTF8`` mode, but expectations for full Unicode compatibility in extension modules would continue to be limited @@ -113,7 +113,7 @@ reported by ``sys.getfilesystemencoding()`` matches the encoding used during this early bootstrapping process. On Windows, the limitations of the ``mbcs`` format used by default in these -conversions proved sufficiently problematic that PEP 528 and PEP 529 were +conversions proved sufficiently problematic that :pep:`528` and :pep:`529` were implemented to bypass the operating system supplied interfaces for binary data handling and force the use of UTF-8 instead. @@ -124,7 +124,7 @@ can cause problems in some situations (for example, when using the GNU readline module [16_]). On non-Apple and non-Android \*nix systems, these operations are handled using -the C locale system in glibc, which has the following characteristics [4_]: +the C locale system in glibc, which has the following characteristics [4]_: * by default, all processes start in the ``C`` locale, which uses ``ASCII`` for these conversions. This is almost never what anyone doing multilingual @@ -136,7 +136,7 @@ the C locale system in glibc, which has the following characteristics [4_]: The specific locale category that covers the APIs that CPython depends on is ``LC_CTYPE``, which applies to "classification and conversion of characters, -and to multibyte and wide characters" [5_]. Accordingly, CPython includes the +and to multibyte and wide characters" [5]_. Accordingly, CPython includes the following key calls to ``setlocale``: * in the main ``python`` binary, CPython calls ``setlocale(LC_ALL, "")`` to @@ -149,7 +149,7 @@ following key calls to ``setlocale``: (This summary of the locale handling omits several technical details related to exactly where and when the text encoding declared as part of the locale -settings is used - see PEP 540 for further discussion, as these particular +settings is used - see :pep:`540` for further discussion, as these particular details matter more when decoupling CPython from the declared C locale than they do when overriding the locale with one based on UTF-8) @@ -183,7 +183,7 @@ Mac OS X and other \*BSD systems have taken a different approach: instead of offering a ``C.UTF-8`` locale, they offer a partial ``UTF-8`` locale that only defines the ``LC_CTYPE`` category. On such systems, the preferred environmental locale adjustment is to set ``LC_CTYPE=UTF-8`` rather than to set -``LC_ALL`` or ``LANG``. [17_] +``LC_ALL`` or ``LANG``. [17]_ In the specific case of Docker containers and similar technologies, the appropriate locale setting can be specified directly in the container image @@ -197,11 +197,11 @@ more narrowly scoped ``LC_MESSAGES=C`` or ``LANGUAGE=en``. Relationship with other PEPs ============================ -This PEP shares a common problem statement with PEP 540 (improving Python 3's +This PEP shares a common problem statement with :pep:`540` (improving Python 3's behaviour in the default C locale), but diverges markedly in the proposed solution: -* PEP 540 proposes to entirely decouple CPython's default text encoding from +* :pep:`540` proposes to entirely decouple CPython's default text encoding from the C locale system in that case, allowing text handling inconsistencies to arise between CPython and other locale-aware components running in the same process and in subprocesses. This approach aims to make CPython behave less @@ -219,7 +219,7 @@ solution: in the wider C/C++ ecosystem After reviewing both PEPs, it became clear that they didn't actually conflict -at a technical level, and the proposal in PEP 540 offered a superior option in +at a technical level, and the proposal in :pep:`540` offered a superior option in cases where no suitable locale was available, as well as offering a better reference behaviour for platforms where the notion of a "locale encoding" doesn't make sense (for example, embedded systems running MicroPython rather @@ -229,13 +229,13 @@ Meanwhile, this PEP offered improved compatibility with other locale-aware components, and an approach more amenable to being backported to Python 3.6 by downstream redistributors. -As a result, this PEP was amended to refer to PEP 540 as a complementary +As a result, this PEP was amended to refer to :pep:`540` as a complementary solution that offered improved behaviour when none of the standard UTF-8 based locales were available, as well as extending the changes in the default settings to APIs that aren't currently independently configurable (such as the default encoding and error handler for ``open()``). -The availability of PEP 540 also meant that the ``LC_CTYPE=en_US.UTF-8`` legacy +The availability of :pep:`540` also meant that the ``LC_CTYPE=en_US.UTF-8`` legacy fallback was removed from the list of UTF-8 locales tried as a coercion target, with the expectation being that CPython will instead rely solely on the proposed PYTHONUTF8 mode in such cases. @@ -247,8 +247,8 @@ Motivation While Linux container technologies like Docker, Kubernetes, and OpenShift are best known for their use in web service development, the related container formats and execution models are also being adopted for Linux command line -application development. Technologies like Gnome Flatpak [7_] and -Ubuntu Snappy [8_] further aim to bring these same techniques to Linux GUI +application development. Technologies like Gnome Flatpak [7]_ and +Ubuntu Snappy [8]_ further aim to bring these same techniques to Linux GUI application development. When using Python 3 for application development in these contexts, it isn't @@ -327,11 +327,11 @@ with this problem automatically rather than relying on redistributors or end users to handle it through system configuration changes. While the glibc developers are working towards making the C.UTF-8 locale -universally available for use by glibc based applications like CPython [6_], +universally available for use by glibc based applications like CPython [6]_, this unfortunately doesn't help on platforms that ship older versions of glibc without that feature, and also don't provide C.UTF-8 (or an equivalent) as an on-disk locale the way Debian and Fedora do. These platforms are considered -out of scope for this PEP - see PEP 540 for further discussion of possible +out of scope for this PEP - see :pep:`540` for further discussion of possible options for improving CPython's default behaviour in such environments. @@ -592,7 +592,7 @@ Android-specific details: Platform Support Changes ======================== -A new "Legacy C Locale" section will be added to PEP 11 that states: +A new "Legacy C Locale" section will be added to :pep:`11` that states: * as of CPython 3.7, \*nix platforms are expected to provide at least one of ``C.UTF-8`` (full locale), ``C.utf8`` (full locale) or ``UTF-8`` ( @@ -649,8 +649,8 @@ Defaulting to "surrogateescape" error handling on the standard IO streams By coercing the locale away from the legacy C default and its assumption of ASCII as the preferred text encoding, this PEP also disables the implicit use of the "surrogateescape" error handler on the standard IO streams that was -introduced in Python 3.5 ([15_]), as well as the automatic use of -``surrogateescape`` when operating in PEP 540's proposed UTF-8 mode. +introduced in Python 3.5 ([15]_), as well as the automatic use of +``surrogateescape`` when operating in :pep:`540`'s proposed UTF-8 mode. Rather than introducing yet another configuration option to adjust that behaviour, this PEP instead proposes to extend the "surrogateescape" default @@ -662,7 +662,7 @@ provided text values are typically able to be transparently passed through a Python 3 application even if it is incorrect in assuming that that text has been encoded as UTF-8. -In particular, GB 18030 [12_] is a Chinese national text encoding standard +In particular, GB 18030 [12]_ is a Chinese national text encoding standard that handles all Unicode code points, that is formally incompatible with both ASCII and UTF-8, but will nevertheless often tolerate processing as surrogate escaped data - the points where GB 18030 reuses ASCII byte values in an @@ -672,7 +672,7 @@ the relevant ASCII code points. Operations that don't involve splitting on or searching for particular ASCII or Unicode code point values are almost certain to work correctly. -Similarly, Shift-JIS [13_] and ISO-2022-JP [14_] remain in widespread use in +Similarly, Shift-JIS [13]_ and ISO-2022-JP [14]_ remain in widespread use in Japan, and are incompatible with both ASCII and UTF-8, but will tolerate text processing operations that don't involve splitting on or searching for particular ASCII or Unicode code point values. @@ -794,7 +794,7 @@ legacy C locale for over a decade at this point. Not only haven't we been able to get it to work, neither has anyone else - the only viable alternatives identified have been to pass the bytes along verbatim without eagerly decoding them to text (C/C++, Python 2.x, Ruby, etc), or else to largely ignore the -nominal C/C++ locale encoding and assume the use of either UTF-8 (PEP 540, +nominal C/C++ locale encoding and assume the use of either UTF-8 (:pep:`540`, Rust, Go, Node.js, etc) or UTF-16-LE (JVM, .NET CLR). While this PEP ensures that developers that genuinely need to do so can still @@ -803,7 +803,7 @@ opt-in to running their Python code in the legacy C locale (by setting ``--without-c-locale-coercion``), it also makes it clear that we *don't* expect Python 3's Unicode handling to be completely reliable in that configuration, and the recommended alternative is to use a more appropriate -locale setting (potentially in combination with PEP 540's UTF-8 mode, if that +locale setting (potentially in combination with :pep:`540`'s UTF-8 mode, if that is available). @@ -908,7 +908,7 @@ This was later removed on the grounds that setting only ``LC_CTYPE`` is sufficient to handle all of the problematic scenarios that the PEP aimed to resolve, while setting ``LANG`` as well would break cases where ``LANG`` was set correctly, and the locale problems were solely due to an incorrect -``LC_CTYPE`` setting ([22_]). +``LC_CTYPE`` setting ([22]_). For example, consider a Python application that called the Linux ``date`` utility in a subprocess rather than doing its own date formatting:: @@ -954,15 +954,15 @@ coercion notice in that case, coercion is instead skipped entirely. Considering locale coercion independently of "UTF-8 mode" --------------------------------------------------------- -With both this PEP's locale coercion and PEP 540's UTF-8 mode under +With both this PEP's locale coercion and :pep:`540`'s UTF-8 mode under consideration for Python 3.7, it makes sense to ask whether or not we can limit ourselves to only doing one or the other, rather than making both changes. -The UTF-8 mode proposed in PEP 540 has two major limitations that make it a +The UTF-8 mode proposed in :pep:`540` has two major limitations that make it a potential complement to this PEP rather than a potential replacement. -First, unlike this PEP, PEP 540's UTF-8 mode makes it possible to change default +First, unlike this PEP, :pep:`540`'s UTF-8 mode makes it possible to change default behaviours that are not currently configurable at all. While that's exactly what makes the proposal interesting, it's also what makes it an entirely unproven approach. By contrast, the approach proposed in this PEP builds @@ -975,7 +975,7 @@ Secondly, one of the things we know based on that experience is that the proposed locale coercion can resolve problems not only in CPython itself, but also in extension modules that interact with the standard streams, like GNU readline. As an example, consider the following interactive session -from a PEP 538 enabled CPython build, where each line after the first is +from a :pep:`538` enabled CPython build, where each line after the first is executed by doing "up-arrow, left-arrow x4, delete, enter":: $ LANG=C ./python @@ -1018,7 +1018,7 @@ locale settings:: That particular misbehaviour is coming from GNU readline, *not* CPython - because the command history editing wasn't UTF-8 aware, it corrupted the history buffer and fed such nonsense to stdin that even the surrogateescape error -handler was bypassed. While PEP 540's UTF-8 mode could technically be updated +handler was bypassed. While :pep:`540`'s UTF-8 mode could technically be updated to also reconfigure readline, that's just *one* extension module that might be interacting with the standard streams without going through the CPython C API, and any change made by CPython would only apply when readline is running @@ -1077,7 +1077,7 @@ be entirely redundant. However, that assumption turned out to be incorrect, as subsequent investigations showed that if you explicitly configure ``LANG=C`` on these platforms, extension modules like GNU readline will misbehave in much the -same way as they do on other \*nix systems. [21_] +same way as they do on other \*nix systems. [21]_ In addition, Mac OS X is also frequently used as a development and testing platform for Python software intended for deployment to other \*nix environments @@ -1093,12 +1093,12 @@ Implementation ============== The reference implementation is being developed in the -``pep538-coerce-c-locale`` feature branch [18_] in Nick Coghlan's fork of the -CPython repository on GitHub. A work-in-progress PR is available at [20_]. +``pep538-coerce-c-locale`` feature branch [18]_ in Nick Coghlan's fork of the +CPython repository on GitHub. A work-in-progress PR is available at [20]_. This reference implementation covers not only the enhancement request in -issue 28180 [1_], but also the Android compatibility fixes needed to resolve -issue 28997 [16_]. +issue 28180 [1]_, but also the Android compatibility fixes needed to resolve +issue 28997 [16]_. Backporting to earlier Python 3 releases @@ -1115,7 +1115,7 @@ default, or else specifically for platforms where such a locale is already consistently available. At least the Fedora project is planning to pursue this approach for the -upcoming Fedora 26 release [19_]. +upcoming Fedora 26 release [19]_. Backporting to other 3.x releases @@ -1139,7 +1139,7 @@ Acknowledgements The locale coercion approach proposed in this PEP is inspired directly by Armin Ronacher's handling of this problem in the ``click`` command line -utility development framework [2_]:: +utility development framework [2]_:: $ LANG=C python3 -c 'import click; cli = click.command()(lambda:None); cli()' Traceback (most recent call last): @@ -1157,18 +1157,18 @@ utility development framework [2_]:: export LANG=C.UTF-8 The change was originally proposed as a downstream patch for Fedora's -system Python 3.6 package [3_], and then reformulated as a PEP for Python 3.7 +system Python 3.6 package [3]_, and then reformulated as a PEP for Python 3.7 with a section allowing for backports to earlier versions by redistributors. In parallel with the development of the upstream patch, Charalampos Stratakis has been working on the Fedora 26 backport and providing feedback on the practical viability of the proposed changes. -The initial draft was posted to the Python Linux SIG for discussion [10_] and +The initial draft was posted to the Python Linux SIG for discussion [10]_ and then amended based on both that discussion and Victor Stinner's work in -PEP 540 [11_]. +:pep:`540` [11]_. The "â„™ÆŽâ˜‚â„ŒĂžáŒ€" string used in the Unicode handling examples throughout this PEP -is taken from Ned Batchelder's excellent "Pragmatic Unicode" presentation [9_]. +is taken from Ned Batchelder's excellent "Pragmatic Unicode" presentation [9]_. Stephen Turnbull has long provided valuable insight into the text encoding handling challenges he regularly encounters at the University of Tsukuba @@ -1179,16 +1179,16 @@ References ========== .. [1] CPython: sys.getfilesystemencoding() should default to utf-8 - (http://bugs.python.org/issue28180) + (https://bugs.python.org/issue28180) .. [2] Locale configuration required for click applications under Python 3 - (http://click.pocoo.org/5/python3/#python-3-surrogate-handling) + (https://click.palletsprojects.com/en/5.x/python3/#python-3-surrogate-handling) .. [3] Fedora: force C.UTF-8 when Python 3 is run under the C locale (https://bugzilla.redhat.com/show_bug.cgi?id=1404918) .. [4] GNU C: How Programs Set the Locale - ( https://www.gnu.org/software/libc/manual/html_node/Setting-the-Locale.html) + (https://www.gnu.org/software/libc/manual/html_node/Setting-the-Locale.html) .. [5] GNU C: Locale Categories (https://www.gnu.org/software/libc/manual/html_node/Locale-Categories.html) @@ -1197,13 +1197,13 @@ References (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) .. [7] GNOME Flatpak - (http://flatpak.org/) + (https://flatpak.org/) .. [8] Ubuntu Snappy (https://www.ubuntu.com/desktop/snappy) .. [9] Pragmatic Unicode - (http://nedbatchelder.com/text/unipain.html) + (https://nedbatchelder.com/text/unipain.html) .. [10] linux-sig discussion of initial PEP draft (https://mail.python.org/pipermail/linux-sig/2017-January/000014.html) @@ -1224,10 +1224,10 @@ References (https://bugs.python.org/issue19977) .. [16] test_readline.test_nonascii fails on Android - (http://bugs.python.org/issue28997) + (https://bugs.python.org/issue28997) .. [17] UTF-8 locale discussion on "locale.getdefaultlocale() fails on Mac OS X with default language set to English" - (http://bugs.python.org/issue18378#msg215215) + (https://bugs.python.org/issue18378#msg215215) .. [18] GitHub branch diff for ``ncoghlan:pep538-coerce-c-locale`` (https://github.com/python/cpython/compare/master...ncoghlan:pep538-coerce-c-locale) @@ -1250,13 +1250,3 @@ Copyright This document has been placed in the public domain under the terms of the CC0 1.0 license: https://creativecommons.org/publicdomain/zero/1.0/ - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0539.txt b/peps/pep-0539.rst similarity index 98% rename from pep-0539.txt rename to peps/pep-0539.rst index ad0b821e6..2aa74fc8a 100644 --- a/pep-0539.txt +++ b/peps/pep-0539.rst @@ -207,7 +207,7 @@ When ``Py_LIMITED_API`` is defined, a TSS key must be dynamically allocated:: Platform Support Changes ======================== -A new "Native Thread Implementation" section will be added to PEP 11 that +A new "Native Thread Implementation" section will be added to :pep:`11` that states: * As of CPython 3.7, all platforms are required to provide a native thread @@ -252,7 +252,7 @@ with Python's API because their ``pthread_key_t`` is defined in a way that cannot be safely cast to ``int``. In fact, the possibility of running into this problem was raised by MvL at the time pthreads TLS was added [2]_. -It could be argued that PEP-11 makes specific requirements for supporting a +It could be argued that :pep:`11` makes specific requirements for supporting a new, not otherwise officially-support platform (such as CloudABI), and that the status of Cygwin support is currently dubious. However, this creates a very high barrier to supporting platforms that are otherwise Linux- and/or @@ -316,7 +316,7 @@ Rejected Ideas * Do nothing: The status quo is fine because it works on Linux, and platforms wishing to be supported by CPython should follow the requirements of - PEP-11. As explained above, while this would be a fair argument if + :pep:`11`. As explained above, while this would be a fair argument if CPython were being to asked to make changes to support particular quirks or features of a specific platform, in this case it is a quirk of CPython that prevents it from being used to its full potential on otherwise @@ -382,8 +382,7 @@ References and Footnotes .. [10] https://github.com/python/cpython/compare/master...ma8ma:pep539-tss-api .. [11] https://github.com/python/cpython/pull/1362 .. [12] https://docs.python.org/3/c-api/init.html#c.Py_FinalizeEx -.. [13] It is also called as "stable ABI" - (https://www.python.org/dev/peps/pep-0384/) +.. [13] It is also called as "stable ABI" (:pep:`384`) .. [14] http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_key_create.html .. [15] http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf#page=404 .. [16] https://bugs.python.org/issue31370 diff --git a/pep-0540.txt b/peps/pep-0540.rst similarity index 98% rename from pep-0540.txt rename to peps/pep-0540.rst index 296852530..fd6f49204 100644 --- a/pep-0540.txt +++ b/peps/pep-0540.rst @@ -274,13 +274,13 @@ Links * `bpo-29240: Implementation of the PEP 540: Add a new UTF-8 Mode `_ -* `PEP 538 `_: +* :pep:`538`: "Coercing the legacy C locale to C.UTF-8" -* `PEP 529 `_: +* :pep:`529`: "Change Windows filesystem encoding to UTF-8" -* `PEP 528 `_: +* :pep:`528`: "Change Windows console encoding to UTF-8" -* `PEP 383 `_: +* :pep:`383`: "Non-decodable Bytes in System Character Interfaces" diff --git a/pep-0541.txt b/peps/pep-0541.rst similarity index 96% rename from pep-0541.txt rename to peps/pep-0541.rst index a83d34a4c..ea741ee74 100644 --- a/pep-0541.txt +++ b/peps/pep-0541.rst @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Ɓukasz Langa BDFL-Delegate: Mark Mangoba -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Final Type: Process +Topic: Packaging Content-Type: text/x-rst Created: 12-Jan-2017 Post-History: @@ -72,16 +73,16 @@ The use cases covered by this document are: * Abandoned projects: - * continued maintenance by a different set of users; or - * removal from the Index for use with a different project. + * continued maintenance by a different set of users; or + * removal from the Index for use with a different project. * Active projects: - * resolving disputes over a name. + * resolving disputes over a name. * Invalid projects: - * projects subject to a claim of intellectual property infringement. + * projects subject to a claim of intellectual property infringement. The proposed extension to the Terms of Use, as expressed in the Implementation section, will be published as a separate document on the @@ -205,12 +206,15 @@ A project published on the Package Index meeting ANY of the following is considered invalid and will be removed from the Index: * project does not conform to Terms of Use; -* project is malware (designed to exploit or harm systems or users); +* project is malware (designed to exploit or harm systems or users directly, to + facilitate command-and-control attacks, or perform data exfiltration); +* project is spam (designed to advertise or solicit goods or services); * project contains illegal content; * project violates copyright, trademarks, patents, or licenses; * project is name squatting (package has no functionality or is empty); * project name, description, or content violates the Code of Conduct; +* project uses obfuscation to hide or mask functionality; or * project is abusing the Package Index for purposes it was not intended. diff --git a/pep-0542.txt b/peps/pep-0542.rst similarity index 96% rename from pep-0542.txt rename to peps/pep-0542.rst index b7a7216a3..3359e8e45 100644 --- a/pep-0542.txt +++ b/peps/pep-0542.rst @@ -134,7 +134,7 @@ Similar to how decorators are syntastic sugar:: Implementation ============== -The `__name__` would follow the principles of a normal function:: +The ``__name__`` would follow the principles of a normal function:: class MyClass: def my_function1(self): @@ -146,7 +146,7 @@ The `__name__` would follow the principles of a normal function:: assert my_function1.__name__ == 'my_function1' assert my_function2.__name__ == 'my_function2' -The grammar would use `dotted_name` to support chaining of attributes:: +The grammar would use ``dotted_name`` to support chaining of attributes:: def Person.name.fset(self, value): self._name = value diff --git a/pep-0543.rst b/peps/pep-0543.rst similarity index 100% rename from pep-0543.rst rename to peps/pep-0543.rst diff --git a/pep-0544.txt b/peps/pep-0544.rst similarity index 94% rename from pep-0544.txt rename to peps/pep-0544.rst index 80830b72f..04df7bd7f 100644 --- a/pep-0544.txt +++ b/peps/pep-0544.rst @@ -4,9 +4,10 @@ Version: $Revision$ Last-Modified: $Date$ Author: Ivan Levkivskyi , Jukka Lehtosalo , Ɓukasz Langa BDFL-Delegate: Guido van Rossum -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 05-Mar-2017 Python-Version: 3.8 @@ -16,24 +17,24 @@ Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/ Abstract ======== -Type hints introduced in PEP 484 can be used to specify type metadata -for static type checkers and other third party tools. However, PEP 484 +Type hints introduced in :pep:`484` can be used to specify type metadata +for static type checkers and other third party tools. However, :pep:`484` only specifies the semantics of *nominal* subtyping. In this PEP we specify static and runtime semantics of protocol classes that will provide a support for *structural* subtyping (static duck typing). -.. _rationale: +.. _PEP 544 rationale: Rationale and Goals =================== -Currently, PEP 484 and the ``typing`` module [typing]_ define abstract +Currently, :pep:`484` and the ``typing`` module [typing]_ define abstract base classes for several common Python protocols such as ``Iterable`` and ``Sized``. The problem with them is that a class has to be explicitly marked to support them, which is unpythonic and unlike what one would normally do in idiomatic dynamically typed Python code. For example, -this conforms to PEP 484:: +this conforms to :pep:`484`:: from typing import Sized, Iterable, Iterator @@ -80,13 +81,13 @@ Nominal vs structural subtyping Structural subtyping is natural for Python programmers since it matches the runtime semantics of duck typing: an object that has certain properties is treated independently of its actual runtime class. -However, as discussed in PEP 483, both nominal and structural +However, as discussed in :pep:`483`, both nominal and structural subtyping have their strengths and weaknesses. Therefore, in this PEP we -*do not propose* to replace the nominal subtyping described by PEP 484 with +*do not propose* to replace the nominal subtyping described by :pep:`484` with structural subtyping completely. Instead, protocol classes as specified in this PEP complement normal classes, and users are free to choose -where to apply a particular solution. See section on `rejected`_ ideas at the -end of this PEP for additional motivation. +where to apply a particular solution. See section on `rejected +`_ ideas at the end of this PEP for additional motivation. Non-goals @@ -95,7 +96,7 @@ Non-goals At runtime, protocol classes will be simple ABCs. There is no intent to provide sophisticated runtime instance and class checks against protocol classes. This would be difficult and error-prone and will contradict the logic -of PEP 484. As well, following PEP 484 and PEP 526 we state that protocols are +of :pep:`484`. As well, following :pep:`484` and :pep:`526` we state that protocols are **completely optional**: * No runtime semantics will be imposed for variables or parameters annotated @@ -180,8 +181,9 @@ approaches related to structural subtyping in Python and other languages: assert issubclass(tuple, MyTuple) assert isinstance((), MyTuple) - As mentioned in the `rationale`_, we want to avoid such necessity, especially - in static context. However, in a runtime context, ABCs are good candidates for + As mentioned in the `rationale `_, + we want to avoid such necessity, especially in static context. + However, in a runtime context, ABCs are good candidates for protocol classes and they are already used extensively in the ``typing`` module. @@ -199,8 +201,9 @@ approaches related to structural subtyping in Python and other languages: assert isinstance(MyIterable(), Iterable) Such behavior seems to be a perfect fit for both runtime and static behavior - of protocols. As discussed in `rationale`_, we propose to add static support - for such behavior. In addition, to allow users to achieve such runtime + of protocols. As discussed in `rationale `_, + we propose to add static support for such behavior. + In addition, to allow users to achieve such runtime behavior for *user-defined* protocols a special ``@runtime_checkable`` decorator will be provided, see detailed `discussion`_ below. @@ -224,9 +227,9 @@ approaches related to structural subtyping in Python and other languages: prohibits redundant members in implementations. While the idea of optional members looks interesting, it would complicate this proposal and it is not clear how useful it will be. Therefore, it is proposed to postpone - this; see `rejected`_ ideas. In general, the idea of static protocol - checking without runtime implications looks reasonable, and basically - this proposal follows the same line. + this; see `rejected `_ ideas. In general, the idea of + static protocol checking without runtime implications looks reasonable, + and basically this proposal follows the same line. * Go [golang]_ uses a more radical approach and makes interfaces the primary way to provide type information. Also, assignments are used to explicitly @@ -241,11 +244,9 @@ approaches related to structural subtyping in Python and other languages: } Both these ideas are questionable in the context of this proposal. See - the section on `rejected`_ ideas. + the section on `rejected `_ ideas. -.. _specification: - Specification ============= @@ -330,7 +331,7 @@ Protocol members All methods defined in the protocol class body are protocol members, both normal and decorated with ``@abstractmethod``. If any parameters of a protocol method are not annotated, then their types are assumed to be ``Any`` -(see PEP 484). Bodies of protocol methods are type checked. +(see :pep:`484`). Bodies of protocol methods are type checked. An abstract method that should not be called via ``super()`` ought to raise ``NotImplementedError``. Example:: @@ -348,7 +349,7 @@ An abstract method that should not be called via ``super()`` ought to raise Static methods, class methods, and properties are equally allowed in protocols. -To define a protocol variable, one can use PEP 526 variable +To define a protocol variable, one can use :pep:`526` variable annotations in the class body. Additional attributes *only* defined in the body of a method by assignment via ``self`` are not allowed. The rationale for this is that the protocol class implementation is often not shared by @@ -376,7 +377,7 @@ Examples:: To distinguish between protocol class variables and protocol instance variables, the special ``ClassVar`` annotation should be used as specified -by PEP 526. By default, protocol variables as defined above are considered +by :pep:`526`. By default, protocol variables as defined above are considered readable and writable. To define a read-only protocol variable, one can use an (abstract) property. @@ -495,8 +496,9 @@ considering subtyping, since structural compatibility is the criterion, not the MRO. If ``Protocol`` is included in the base class list, all the other base classes -must be protocols. A protocol can't extend a regular class, see `rejected`_ -ideas for rationale. Note that rules around explicit subclassing are different +must be protocols. A protocol can't extend a regular class, see `rejected +`_ ideas for rationale. +Note that rules around explicit subclassing are different from regular ABCs, where abstractness is simply defined by having at least one abstract method being unimplemented. Protocol classes must be marked *explicitly*. @@ -550,7 +552,7 @@ the declared variance. Examples:: Note that unlike nominal classes, de facto covariant protocols cannot be declared as invariant, since this can break transitivity of subtyping -(see `rejected`_ ideas for details). For example:: +(see `rejected `_ ideas for details). For example:: T = TypeVar('T') @@ -563,7 +565,7 @@ Recursive protocols ------------------- Recursive protocols are also supported. Forward references to the protocol -class names can be given as strings as specified by PEP 484. Recursive +class names can be given as strings as specified by :pep:`484`. Recursive protocols are useful for representing self-referential data structures like trees in an abstract fashion:: @@ -594,8 +596,9 @@ Continuing the previous example:: Self-types in protocols ----------------------- -The self-types in protocols follow the corresponding specification -[self-types]_ of PEP 484. For example:: +The self-types in protocols follow the +:pep:`corresponding specification <484#annotating-instance-and-class-methods>` +of :pep:`484`. For example:: C = TypeVar('C', bound='Copyable') class Copyable(Protocol): @@ -620,7 +623,7 @@ Callback protocols Protocols can be used to define flexible callback types that are hard (or even impossible) to express using the ``Callable[...]`` syntax -specified by PEP 484, such as variadic, overloaded, and complex generic +specified by :pep:`484`, such as variadic, overloaded, and complex generic callbacks. They can be defined as protocols with a ``__call__`` member:: from typing import Optional, List, Protocol @@ -729,8 +732,8 @@ Example:: cached_func((1, 2, 3)) # OK, tuple is both hashable and iterable If this will prove to be a widely used scenario, then a special -intersection type construct could be added in future as specified by PEP 483, -see `rejected`_ ideas for more details. +intersection type construct could be added in future as specified by :pep:`483`, +see `rejected `_ ideas for more details. ``Type[]`` and class objects vs protocols @@ -886,7 +889,8 @@ that provides the same semantics for class and instance checks as for assert isinstance(open('some/file'), SupportsClose) Note that instance checks are not 100% reliable statically, this is why -this behavior is opt-in, see section on `rejected`_ ideas for examples. +this behavior is opt-in, see section on `rejected `_ +ideas for examples. The most type checkers can do is to treat ``isinstance(obj, Iterator)`` roughly as a simpler way to write ``hasattr(x, '__iter__') and hasattr(x, '__next__')``. To minimize @@ -943,7 +947,7 @@ Properties can be settable and/or abstract if needed:: def d(self) -> int: # ... or it can be abstract return 0 -Also function type comments can be used as per PEP 484 (for example +Also function type comments can be used as per :pep:`484` (for example to provide compatibility with Python 2). The ``typing`` module changes proposed in this PEP will also be backported to earlier versions via the backport currently available on PyPI. @@ -1025,7 +1029,7 @@ at runtime in this case). But together with other introspection tools this give a reasonable perspective for runtime type checking tools. -.. _rejected: +.. _PEP 544 rejected: Rejected/Postponed Ideas ======================== @@ -1116,7 +1120,7 @@ variables. However, using getters and setters in cases where only a simple variable is needed would be quite unpythonic. Moreover, the widespread use of properties (that often act as type validators) in large code bases is partially due to previous absence of static type checkers for Python, -the problem that PEP 484 and this PEP are aiming to solve. For example:: +the problem that :pep:`484` and this PEP are aiming to solve. For example:: # without static types @@ -1178,7 +1182,7 @@ complicate both the concept and the implementation. On the other hand, Zope interfaces are conceptually a superset of protocols defined here, but using an incompatible syntax to define them, -because before PEP 526 there was no straightforward way to annotate attributes. +because before :pep:`526` there was no straightforward way to annotate attributes. In the 3.6+ world, ``zope.interface`` might potentially adopt the ``Protocol`` syntax. In this case, type checkers could be taught to recognize interfaces as protocols and make simple structural checks with respect to them. @@ -1332,7 +1336,7 @@ Overriding inferred variance of protocol classes ------------------------------------------------ It was proposed to allow declaring protocols as invariant if they are actually -covariant or contravariant (as it is possible for nominal classes, see PEP 484). +covariant or contravariant (as it is possible for nominal classes, see :pep:`484`). However, it was decided not to do this because of several downsides: * Declared protocol invariance breaks transitivity of sub-typing. Consider @@ -1375,9 +1379,10 @@ However, it was decided not to do this because of several downsides: Support adapters and adaptation ------------------------------- -Adaptation was proposed by PEP 246 (rejected) and is supported by -``zope.interface``, see https://docs.zope.org/zope.interface/adapter.html. -Adapters is quite an advanced concept, and PEP 484 supports unions and +Adaptation was proposed by :pep:`246` (rejected) and is supported by +``zope.interface``, see `the Zope documentation on adapter registries +`_. +Adapters is quite an advanced concept, and :pep:`484` supports unions and generic aliases that can be used instead of adapters. This can be illustrated with an example of ``Iterable`` protocol, there is another way of supporting iteration by providing ``__getitem__`` and ``__len__``. If a function @@ -1490,22 +1495,8 @@ References .. [elsewhere] https://github.com/python/peps/pull/224 -.. [self-types] - https://www.python.org/dev/peps/pep-0484/#annotating-instance-and-class-methods - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0545.txt b/peps/pep-0545.rst similarity index 89% rename from pep-0545.txt rename to peps/pep-0545.rst index 9e4ff3a03..d5b475436 100644 --- a/pep-0545.txt +++ b/peps/pep-0545.rst @@ -1,7 +1,5 @@ PEP: 545 Title: Python Documentation Translations -Version: $Revision$ -Last-Modified: $Date$ Author: Julien Palard , Inada Naoki , Victor Stinner @@ -40,14 +38,14 @@ meet people who don't speak English and so are unable to read the Python official documentation. Python wants to be widely available to all users in any language: this is also why Python 3 supports any non-ASCII identifiers: -https://www.python.org/dev/peps/pep-3131/#rationale +:pep:`3131#rationale` There are at least 4 groups of people who are translating the Python documentation to their native language (French [16]_ [17]_ [18]_, -Japanese [19]_ [20]_, Spanish [21]_, Hungarian [27]_ [28]_) even +Japanese [19]_ [20]_, Spanish [21]_, Hungarian [26]_ [27]_) even though their translations are not visible on d.p.o. Other, less visible and less organized groups, are also translating the -documentation, we've heard of Russian [26]_, Chinese and +documentation, we've heard of Russian [25]_, Chinese and Korean. Others we haven't found yet might also exist. This PEP defines rules describing how to move translations on docs.python.org so they can easily be found by developers, newcomers and potential @@ -115,7 +113,7 @@ The three currently stable branches that will be translated are [12]_: branches needs to be modified to support translation [12]_, whereas these branches now only accept security-only fixes. -The development branch (master) should have a lower translation priority +The development branch (main) should have a lower translation priority than stable branches. But docsbuild-scripts should build it anyway so it is possible for a team to work on it to be ready for the next release. @@ -136,11 +134,11 @@ sometimes almost impossible when already registered, this solution should be avoided. Using subdomains like "es.docs.python.org" or "docs.es.python.org" is -possible but confusing ("is it `es.docs.python.org` or -`docs.es.python.org`?"). Hyphens in subdomains like -`pt-br.doc.python.org` is uncommon and SEOMoz [23]_ correlated the +possible but confusing ("is it ``es.docs.python.org`` or +``docs.es.python.org``?"). Hyphens in subdomains like +``pt-br.doc.python.org`` is uncommon and SEOMoz [23]_ correlated the presence of hyphens as a negative factor. Usage of underscores in -subdomain is prohibited by the RFC1123 [24]_, section 2.1. Finally, +subdomain is prohibited by the :rfc:`1123`, section 2.1. Finally, using subdomains means creating TLS certificates for each language. This not only requires more maintenance but will also cause issues in language switcher if, as for version switcher, we want a @@ -153,7 +151,7 @@ request and ``Vary: Accept-Language``) leads to a bad user experience where they can't easily change the language. According to Mozilla: "This header is a hint to be used when the server has no way of determining the language via another way, like a specific URL, that is -controlled by an explicit user decision." [25]_. As we want to be +controlled by an explicit user decision." [24]_. As we want to be able to easily change the language, we should not use the content negotiation as a main language determination, so we need something else. @@ -188,7 +186,7 @@ The current documentation is not moved to "/en/", instead Language Tag '''''''''''' -A common notation for language tags is the IETF Language Tag [3]_ +A common notation for language tags is the :rfc:`IETF Language Tag <5646>` [4]_ based on ISO 639, although gettext uses ISO 639 tags with underscores (ex: ``pt_BR``) instead of dashes to join tags [5]_ (ex: ``pt-BR``). Examples of IETF Language Tags: ``fr`` (French), @@ -202,7 +200,8 @@ internally: URLs are not meant to leak the underlying implementation. It's uncommon to see capitalized letters in URLs, and docs.python.org doesn't use any, so it may hurt readability by attracting the eye on it, like in: "https://docs.python.org/pt-BR/3.6/library/stdtypes.html". -RFC 5646 (Tags for Identifying Languages (IETF)) section-2.1 [7]_ +:rfc:`5646#section-2.1.1` +(Tags for Identifying Languages (IETF)) section-2.1 states that tags are not case sensitive. As the RFC allows lower case, and it enhances readability, we should use lowercased tags like ``pt-br``. @@ -329,7 +328,7 @@ involves creativity in the expression of the ideas. There's multiple solutions, quoting Van Lindberg from the PSF asked about the subject: - 1. Docs should either have the copyright​ assigned or be under CCO. A + 1. Docs should either have the copyright assigned or be under CCO. A permissive software license (like Apache or MIT) would also get the job done, although it is not quite fit for task. @@ -416,7 +415,7 @@ Setup a GitHub bot for Documentation Contribution Agreement To help ensuring contributors from GitHub have signed the Documentation Contribution Agreement, We can setup the "The Knights Who Say Ni" GitHub bot customized for this agreement on the migrated -repositories [29]_. +repositories [28]_. Patch docsbuild-scripts to Compile Translations @@ -547,19 +546,14 @@ the translation can be added to the language switcher. Previous Discussions ==================== -- `[Python-ideas] Cross link documentation translations (January, 2016)`_ -- `[Python-ideas] Cross link documentation translations (January, 2016)`_ -- `[Python-ideas] https://docs.python.org/fr/ ? (March 2016)`_ +`[Python-ideas] Cross link documentation translations (January, 2016) +`__ +`[Python-Dev] Translated Python documentation (February 2016) +`__ -.. _[Python-ideas] Cross link documentation translations (January, 2016): - https://mail.python.org/pipermail/python-ideas/2016-January/038010.html - -.. _[Python-Dev] Translated Python documentation (February 2016): - https://mail.python.org/pipermail/python-dev/2017-February/147416.html - -.. _[Python-ideas] https://docs.python.org/fr/ ? (March 2016): - https://mail.python.org/pipermail/python-ideas/2016-March/038879.html +`[Python-ideas] https://docs.python.org/fr/ ? (March 2016) +`__ References @@ -569,11 +563,8 @@ References Python documents? (https://mail.python.org/pipermail/i18n-sig/2013-September/002130.html) -.. [2] [Doc-SIG] Localization of Python docs - (https://mail.python.org/pipermail/doc-sig/2013-September/003948.html) - -.. [3] Tags for Identifying Languages - (http://tools.ietf.org/html/rfc5646) +[2] [Doc-SIG] Localization of Python docs +\ (https://mail.python.org/pipermail/doc-sig/2013-September/003948.html) .. [4] IETF language tag (https://en.wikipedia.org/wiki/IETF_language_tag) @@ -582,10 +573,7 @@ References (https://www.gnu.org/software/gettext/manual/html_node/Locale-Names.html) .. [6] Semantic URL: Slug - (https://en.wikipedia.org/wiki/Semantic_URL#Slug) - -.. [7] Tags for Identifying Languages: Formatting of Language Tags - (https://tools.ietf.org/html/rfc5646#section-2.1.1) + (https://en.wikipedia.org/wiki/Clean_URL#Slug) .. [8] Docsbuild-scripts GitHub repository (https://github.com/python/docsbuild-scripts/) @@ -614,8 +602,8 @@ References .. [16] French translation (https://www.afpy.org/doc/python/) -.. [17] French translation on GitHub - (https://github.com/AFPy/python_doc_fr) +.. [17] French translation on Gitea + (https://git.afpy.org/AFPy/python-docs-fr) .. [18] French mailing list (http://lists.afpy.org/mailman/listinfo/traductions) @@ -627,7 +615,7 @@ References (https://github.com/python-doc-ja/python-doc-ja) .. [21] Spanish translation - (http://docs.python.org.ar/tutorial/3/index.html) + (https://docs.python.org/es/3/tutorial/index.html) .. [22] [Python-Dev] Translated Python documentation: doc vs docs (https://mail.python.org/pipermail/python-dev/2017-February/147472.html) @@ -635,22 +623,19 @@ References .. [23] Domains - SEO Best Practices | Moz (https://moz.com/learn/seo/domain) -.. [24] Requirements for Internet Hosts -- Application and Support - (https://www.ietf.org/rfc/rfc1123.txt) - -.. [25] Accept-Language +.. [24] Accept-Language (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) -.. [26] Đ”ĐŸĐșŃƒĐŒĐ”ĐœŃ‚Đ°Ń†ĐžŃ Python 2.7! +.. [25] Đ”ĐŸĐșŃƒĐŒĐ”ĐœŃ‚Đ°Ń†ĐžŃ Python 2.7! (http://python-lab.ru/documentation/index.html) -.. [27] Python-oktatĂł - (http://harp.pythonanywhere.com/python_doc/tutorial/index.html) +.. [26] Python-oktatĂł + (http://web.archive.org/web/20170526080729/http://harp.pythonanywhere.com/python_doc/tutorial/index.html) -.. [28] The Python-hu Archives +.. [27] The Python-hu Archives (https://mail.python.org/pipermail/python-hu/) -.. [29] [Python-Dev] PEP 545: Python Documentation Translations +.. [28] [Python-Dev] PEP 545: Python Documentation Translations (https://mail.python.org/pipermail/python-dev/2017-April/147752.html) @@ -658,15 +643,3 @@ Copyright ========= This document has been placed in the public domain. - - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0546.txt b/peps/pep-0546.rst similarity index 99% rename from pep-0546.txt rename to peps/pep-0546.rst index c25654f70..867d4f175 100644 --- a/pep-0546.txt +++ b/peps/pep-0546.rst @@ -109,7 +109,7 @@ bundling PyOpenSSL. Only backporting ``ssl.MemoryBIO`` and ``ssl.SSLObject`` would avoid the need to embed pyOpenSSL, and would fix the bootstrap issue (python -> ensurepip -> pip -> requests -> MemoryBIO). -This situation is less problematic than the barrier to adoption of PEP 543, as +This situation is less problematic than the barrier to adoption of :pep:`543`, as naturally Requests does not have to move to an event loop model before it drops support for Python 2.7. However, it does make it painful for Requests (and pip) to embrace both asyncio and the ``async`` and ``await`` keywords for as long as diff --git a/pep-0547.rst b/peps/pep-0547.rst similarity index 95% rename from pep-0547.rst rename to peps/pep-0547.rst index 4dff27c26..51d524a1f 100644 --- a/pep-0547.rst +++ b/peps/pep-0547.rst @@ -28,7 +28,7 @@ Abstract This PEP proposes implementation that allows built-in and extension modules to be executed in the ``__main__`` namespace using -the PEP 489 multi-phase initialization. +the :pep:`489` multi-phase initialization. With this, a multi-phase initialization enabled module can be run using following command:: @@ -45,7 +45,7 @@ Python source modules. Specifically, it is not possible to run extension modules as scripts using Python's ``-m`` option. -The technical groundwork to make this possible has been done for PEP 489, +The technical groundwork to make this possible has been done for :pep:`489`, and enabling the ``-m`` option is listed in that PEP's “Possible Future Extensions” section. Technically, the additional changes proposed here are relatively small. @@ -91,7 +91,7 @@ This is not the case for extension modules, whose ``PyInit_*`` entry point traditionally both created a new module object (using ``PyModule_Create``), and initialized it. -Since Python 3.5, extension modules can use PEP 489 multi-phase initialization. +Since Python 3.5, extension modules can use :pep:`489` multi-phase initialization. In this scenario, the ``PyInit_*`` entry point returns a ``PyModuleDef`` structure: a description of how the module should be created and initialized. The extension can choose to customize creation of the module object using @@ -128,7 +128,7 @@ its ``md_state`` is normally ``NULL``. Before initializing an extension module in ``__main__``'s context, its module state will be allocated according to the ``PyModuleDef`` of that module. -While PEP 489 was designed to make these changes generally possible, +While :pep:`489` was designed to make these changes generally possible, it's necessary to decouple module discovery, creation, and initialization steps for extension modules, so that another module can be used instead of a newly initialized one, and the functionality needs to be added to @@ -161,7 +161,7 @@ importlib's ``ExtensionFileLoader`` will get an implementation of extension module's ``PyInit_*`` function. The ``PyInit_*`` function can return either a fully initialized module -(single-phase initialization) or a ``PyModuleDef`` (for PEP 489 multi-phase +(single-phase initialization) or a ``PyModuleDef`` (for :pep:`489` multi-phase initialization). In the single-phase initialization case, ``_imp.exec_in_module`` will raise @@ -196,7 +196,6 @@ References .. _GitHub: https://github.com/python/cpython/pull/1761 .. _Cython issue 1715: https://github.com/cython/cython/issues/1715 -.. _Possible Future Extensions section: https://www.python.org/dev/peps/pep-0489/#possible-future-extensions .. _Cython issue 1923: https://github.com/cython/cython/pull/1923 diff --git a/pep-0548.rst b/peps/pep-0548.rst similarity index 100% rename from pep-0548.rst rename to peps/pep-0548.rst diff --git a/pep-0549.rst b/peps/pep-0549.rst similarity index 97% rename from pep-0549.rst rename to peps/pep-0549.rst index ca58074c3..69040a9aa 100644 --- a/pep-0549.rst +++ b/peps/pep-0549.rst @@ -2,14 +2,14 @@ PEP: 549 Title: Instance Descriptors Version: $Revision$ Last-Modified: $Date$ -Author: larry@hastings.org (Larry Hastings) -Discussions-To: Python-Dev +Author: Larry Hastings +Discussions-To: python-dev@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 04-Sep-2017 Python-Version: 3.7 -Post-History: 4-Sep-2017 +Post-History: 04-Sep-2017 Rejection Notice diff --git a/pep-0550-hamt_vs_dict-v2.png b/peps/pep-0550-hamt_vs_dict-v2.png similarity index 100% rename from pep-0550-hamt_vs_dict-v2.png rename to peps/pep-0550-hamt_vs_dict-v2.png diff --git a/pep-0550-hamt_vs_dict.png b/peps/pep-0550-hamt_vs_dict.png similarity index 100% rename from pep-0550-hamt_vs_dict.png rename to peps/pep-0550-hamt_vs_dict.png diff --git a/pep-0550-lookup_hamt.png b/peps/pep-0550-lookup_hamt.png similarity index 100% rename from pep-0550-lookup_hamt.png rename to peps/pep-0550-lookup_hamt.png diff --git a/pep-0550.rst b/peps/pep-0550.rst similarity index 98% rename from pep-0550.rst rename to peps/pep-0550.rst index 4cb7bba1d..32d0e1496 100644 --- a/pep-0550.rst +++ b/peps/pep-0550.rst @@ -818,7 +818,9 @@ Implementation Execution context is implemented as an immutable linked list of logical contexts, where each logical context is an immutable weak key mapping. A pointer to the currently active execution context is -stored in the OS thread state:: +stored in the OS thread state: + +.. code-block:: text +-----------------+ | | ec @@ -1309,7 +1311,7 @@ execution state:: def __exit__(self, *err): self.var.set(self.old_x) -An equivalent implementation with PEP 521:: +An equivalent implementation with :pep:`521`:: local = threading.local() @@ -1568,6 +1570,7 @@ Appendix: HAMT Performance Analysis .. figure:: pep-0550-hamt_vs_dict-v2.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 1. Benchmark code can be found here: [9]_. @@ -1581,6 +1584,7 @@ The above chart demonstrates that: .. figure:: pep-0550-lookup_hamt.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 2. Benchmark code can be found here: [10]_. @@ -1693,17 +1697,15 @@ Version History References ========== -.. [1] https://blog.golang.org/context +.. [1] https://go.dev/blog/context -.. [2] https://msdn.microsoft.com/en-us/library/system.threading.executioncontext.aspx +.. [2] https://docs.microsoft.com/en-us/dotnet/api/system.threading.executioncontext .. [3] https://github.com/numpy/numpy/issues/9444 -.. [4] http://bugs.python.org/issue31179 - .. [5] https://en.wikipedia.org/wiki/Hash_array_mapped_trie -.. [6] http://blog.higher-order.net/2010/08/16/assoc-and-clojures-persistenthashmap-part-ii.html +.. [6] https://blog.higher-order.net/2010/08/16/assoc-and-clojures-persistenthashmap-part-ii.html .. [7] https://github.com/1st1/cpython/tree/hamt @@ -1713,18 +1715,6 @@ References .. [10] https://gist.github.com/1st1/dbe27f2e14c30cce6f0b5fddfc8c437e -.. [11] https://github.com/1st1/cpython/tree/pep550 - -.. [12] https://www.python.org/dev/peps/pep-0492/#async-await - -.. [13] https://github.com/MagicStack/uvloop/blob/master/examples/bench/echoserver.py - -.. [14] https://github.com/MagicStack/pgbench - -.. [15] https://github.com/python/performance - -.. [16] https://gist.github.com/1st1/6b7a614643f91ead3edf37c4451a6b4c - .. [17] https://mail.python.org/pipermail/python-ideas/2017-August/046752.html .. [18] https://mail.python.org/pipermail/python-ideas/2017-August/046772.html @@ -1749,7 +1739,7 @@ References .. [28] https://docs.python.org/3/library/decimal.html#decimal.Context.abs -.. [29] https://curio.readthedocs.io/en/latest/reference.html#task-local-storage +.. [29] https://web.archive.org/web/20170706074739/https://curio.readthedocs.io/en/latest/reference.html#task-local-storage .. [30] https://docs.atlassian.com/aiolocals/latest/usage.html @@ -1764,13 +1754,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0551.rst b/peps/pep-0551.rst similarity index 98% rename from pep-0551.rst rename to peps/pep-0551.rst index 3ac8d5bf0..1f800beb6 100644 --- a/pep-0551.rst +++ b/peps/pep-0551.rst @@ -8,7 +8,7 @@ Type: Informational Content-Type: text/x-rst Created: 23-Aug-2017 Python-Version: 3.7 -Post-History: 24-Aug-2017 (security-sig), 28-Aug-2017 (python-dev) +Post-History: 24-Aug-2017, 28-Aug-2017 .. note:: This PEP has been withdrawn. For information about integrated @@ -20,7 +20,7 @@ Relationship to PEP 578 This PEP has been split into two since its original posting. -See `PEP 578 `_ for the +See :pep:`578` for the auditing APIs proposed for addition to the next version of Python. This is now an informational PEP, providing guidance to those planning @@ -34,7 +34,7 @@ applies to the Python runtime. Visibility into actions taken by the runtime is invaluable in integrating Python into an otherwise secure and/or monitored environment. -The audit hooks described in PEP-578 are an essential component in +The audit hooks described in :pep:`578` are an essential component in detecting, identifying and analyzing misuse of Python. While the hooks themselves are neutral (in that not every reported event is inherently misuse), they provide essential context to those who are responsible @@ -140,14 +140,14 @@ tools, most network access and DNS resolution, and attempts to create and hide files or configuration settings on the local machine. To summarize, defenders have a need to audit specific uses of Python in -order to detect abnormal or malicious usage. With PEP 578, the Python +order to detect abnormal or malicious usage. With :pep:`578`, the Python runtime gains the ability to provide this. The aim of this PEP is to assist system administrators with deploying a security transparent version of Python that can integrate with their existing auditing and protection systems. On Windows, some specific features that may be integrated through the -hooks added by PEP 578 include: +hooks added by :pep:`578` include: * Script Block Logging [3]_ * DeviceGuard [4]_ diff --git a/pep-0552.rst b/peps/pep-0552.rst similarity index 99% rename from pep-0552.rst rename to peps/pep-0552.rst index c2a7b3e61..3a1a8a532 100644 --- a/pep-0552.rst +++ b/peps/pep-0552.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 04-Sep-2017 Python-Version: 3.7 -Post-History: 2017-09-07 +Post-History: 07-Sep-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-September/149649.html diff --git a/pep-0553.rst b/peps/pep-0553.rst similarity index 97% rename from pep-0553.rst rename to peps/pep-0553.rst index 1ca7eefe9..dea715092 100644 --- a/pep-0553.rst +++ b/peps/pep-0553.rst @@ -6,7 +6,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 05-Sep-2017 Python-Version: 3.7 -Post-History: 2017-09-05, 2017-09-07, 2017-09-13 +Post-History: 05-Sep-2017, 07-Sep-2017, 13-Sep-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-October/149705.html @@ -217,7 +217,7 @@ existing keyword such as ``break here``. This is rejected on several fronts. * An extended keyword such as ``break here``, while more readable and not requiring a ``__future__`` would tie the keyword extension to this new feature, preventing more useful extensions such as those proposed in - PEP 548. + :pep:`548`. * A new keyword would require a modified grammar and likely a new bytecode. Each of these makes the implementation more complex. A new built-in breaks @@ -280,22 +280,8 @@ References .. [impl] https://github.com/python/cpython/pull/3355 -.. [envar] - https://mail.python.org/pipermail/python-dev/2017-September/149447.html - Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/peps/pep-0554.rst b/peps/pep-0554.rst new file mode 100644 index 000000000..b8573ddca --- /dev/null +++ b/peps/pep-0554.rst @@ -0,0 +1,1390 @@ +PEP: 554 +Title: Multiple Interpreters in the Stdlib +Author: Eric Snow +Discussions-To: https://discuss.python.org/t/pep-554-multiple-interpreters-in-the-stdlib/24855 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 05-Sep-2017 +Python-Version: 3.13 +Post-History: `07-Sep-2017 `__, + `08-Sep-2017 `__, + `13-Sep-2017 `__, + `05-Dec-2017 `__, + `04-May-2020 `__, + `14-Mar-2023 `__, + + +Abstract +======== + +CPython has supported multiple interpreters in the same process (AKA +"subinterpreters") since version 1.5 (1997). The feature has been +available via the C-API. [c-api]_ Multiple interpreters operate in +`relative isolation from one another `_, which +facilitates novel alternative approaches to +`concurrency `_. + +This proposal introduces the stdlib ``interpreters`` module. It exposes +the basic functionality of multiple interpreters already provided by the +C-API, along with describing a *very* basic way to communicate +(i.e. pass data between interpreters). + + +A Disclaimer about the GIL +========================== + +To avoid any confusion up front: This PEP is meant to be independent +of any efforts to stop sharing the GIL between interpreters (:pep:`684`). +At most this proposal will allow users to take advantage of any +GIL-related work. + +The author's position here is that exposing multiple interpreters +to Python code is worth doing, even if they still share the GIL. +Conversations with past steering councils indicates they do not +necessarily agree. + + +Proposal +======== + +Summary: + +* add a new stdlib module: "interpreters" +* help for extension module maintainers + + +The "interpreters" Module +------------------------- + +The ``interpreters`` module will provide a high-level interface +to the multiple interpreter functionality, and wrap a new low-level +``_interpreters`` (in the same way as the ``threading`` module). +See the `Examples`_ section for concrete usage and use cases. + +Along with exposing the existing (in CPython) multiple interpreter +support, the module will also support a very basic mechanism for +passing data between interpreters. That involves setting simple objects +in the ``__main__`` module of a target subinterpreter. If one end of +an ``os.pipe()`` is passed this way then that pipe can be used to send +bytes between the two interpreters. + +Note that *objects* are not shared between interpreters since they are +tied to the interpreter in which they were created. Instead, the +objects' *data* is passed between interpreters. See the `Shared Data`_ +and `API For Sharing Data`_ sections for more details about +sharing/communicating between interpreters. + +API summary for interpreters module +----------------------------------- + +Here is a summary of the API for the ``interpreters`` module. For a +more in-depth explanation of the proposed classes and functions, see +the `"interpreters" Module API`_ section below. + +For creating and using interpreters: + ++----------------------------------+----------------------------------------------+ +| signature | description | ++==================================+==============================================+ +| ``list_all() -> [Interpreter]`` | Get all existing interpreters. | ++----------------------------------+----------------------------------------------+ +| ``get_current() -> Interpreter`` | Get the currently running interpreter. | ++----------------------------------+----------------------------------------------+ +| ``get_main() -> Interpreter`` | Get the main interpreter. | ++----------------------------------+----------------------------------------------+ +| ``create() -> Interpreter`` | Initialize a new (idle) Python interpreter. | ++----------------------------------+----------------------------------------------+ + +| + ++----------------------------------+---------------------------------------------------+ +| signature | description | ++==================================+===================================================+ +| ``class Interpreter`` | A single interpreter. | ++----------------------------------+---------------------------------------------------+ +| ``.id`` | The interpreter's ID (read-only). | ++----------------------------------+---------------------------------------------------+ +| ``.is_running() -> bool`` | Is the interpreter currently executing code? | ++----------------------------------+---------------------------------------------------+ +| ``.close()`` | Finalize and destroy the interpreter. | ++----------------------------------+---------------------------------------------------+ +| ``.run(src_str, /)`` | | Run the given source code in the interpreter | +| | | (in the current thread). | ++----------------------------------+---------------------------------------------------+ + +.. XXX Support blocking interp.run() until the interpreter + finishes its current work. + +| + ++--------------------+------------------+------------------------------------------------------+ +| exception | base | description | ++====================+==================+======================================================+ +| ``RunFailedError`` | ``RuntimeError`` | Interpreter.run() resulted in an uncaught exception. | ++--------------------+------------------+------------------------------------------------------+ + +.. XXX Add "InterpreterAlreadyRunningError"? + +Help for Extension Module Maintainers +------------------------------------- + +In practice, an extension that implements multi-phase init (:pep:`489`) +is considered isolated and thus compatible with multiple interpreters. +Otherwise it is "incompatible". + +Many extension modules are still incompatible. The maintainers and +users of such extension modules will both benefit when they are updated +to support multiple interpreters. In the meantime, users may become +confused by failures when using multiple interpreters, which could +negatively impact extension maintainers. See `Concerns`_ below. + +To mitigate that impact and accelerate compatibility, we will do the +following: + +* be clear that extension modules are *not* required to support use in + multiple interpreters +* raise ``ImportError`` when an incompatible module is imported + in a subinterpreter +* provide resources (e.g. docs) to help maintainers reach compatibility +* reach out to the maintainers of Cython and of the most used extension + modules (on PyPI) to get feedback and possibly provide assistance + + +Examples +======== + +Run isolated code +----------------- + +:: + + interp = interpreters.create() + print('before') + interp.run('print("during")') + print('after') + +Run in a thread +--------------- + +:: + + interp = interpreters.create() + def run(): + interp.run('print("during")') + t = threading.Thread(target=run) + print('before') + t.start() + t.join() + print('after') + +Pre-populate an interpreter +--------------------------- + +:: + + interp = interpreters.create() + interp.run(tw.dedent(""" + import some_lib + import an_expensive_module + some_lib.set_up() + """)) + wait_for_request() + interp.run(tw.dedent(""" + some_lib.handle_request() + """)) + +Handling an exception +--------------------- + +:: + + interp = interpreters.create() + try: + interp.run(tw.dedent(""" + raise KeyError + """)) + except interpreters.RunFailedError as exc: + print(f"got the error from the subinterpreter: {exc}") + +Re-raising an exception +----------------------- + +:: + + interp = interpreters.create() + try: + try: + interp.run(tw.dedent(""" + raise KeyError + """)) + except interpreters.RunFailedError as exc: + raise exc.__cause__ + except KeyError: + print("got a KeyError from the subinterpreter") + +Note that this pattern is a candidate for later improvement. + +Synchronize using an OS pipe +---------------------------- + +:: + + interp = interpreters.create() + r, s = os.pipe() + print('before') + interp.run(tw.dedent(f""" + import os + os.read({r}, 1) + print("during") + """)) + print('after') + os.write(s, '') + +Sharing a file descriptor +------------------------- + +:: + + interp = interpreters.create() + r1, s1 = os.pipe() + r2, s2 = os.pipe() + interp.run(tw.dedent(f""" + import os + fd = int.from_bytes( + os.read({r1}, 10), 'big') + for line in os.fdopen(fd): + print(line) + os.write({s2}, b'') + """)) + with open('spamspamspam') as infile: + fd = infile.fileno().to_bytes(1, 'big') + os.write(s1, fd) + os.read(r2, 1) + +Passing objects via pickle +-------------------------- + +:: + + interp = interpreters.create() + r, s = os.pipe() + interp.run(tw.dedent(f""" + import os + import pickle + reader = {r} + """)) + interp.run(tw.dedent(""" + data = b'' + c = os.read(reader, 1) + while c != b'\x00': + while c != b'\x00': + data += c + c = os.read(reader, 1) + obj = pickle.loads(data) + do_something(obj) + c = os.read(reader, 1) + """)) + for obj in input: + data = pickle.dumps(obj) + os.write(s, data) + os.write(s, b'\x00') + os.write(s, b'\x00') + +Capturing an interpreter's stdout +--------------------------------- + +:: + + interp = interpreters.create() + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + interp.run(tw.dedent(""" + print('spam!') + """)) + assert(stdout.getvalue() == 'spam!') + +A pipe (``os.pipe()``) could be used similarly. + +Running a module +---------------- + +:: + + interp = interpreters.create() + main_module = mod_name + interp.run(f'import runpy; runpy.run_module({main_module!r})') + +Running as script (including zip archives & directories) +-------------------------------------------------------- + +:: + + interp = interpreters.create() + main_script = path_name + interp.run(f"import runpy; runpy.run_path({main_script!r})") + + +Rationale +========= + +Running code in multiple interpreters provides a useful level of +isolation within the same process. This can be leveraged in a number +of ways. Furthermore, subinterpreters provide a well-defined framework +in which such isolation may extended. (See :pep:`684`.) + +Nick Coghlan explained some of the benefits through a comparison with +multi-processing [benefits]_:: + + [I] expect that communicating between subinterpreters is going + to end up looking an awful lot like communicating between + subprocesses via shared memory. + + The trade-off between the two models will then be that one still + just looks like a single process from the point of view of the + outside world, and hence doesn't place any extra demands on the + underlying OS beyond those required to run CPython with a single + interpreter, while the other gives much stricter isolation + (including isolating C globals in extension modules), but also + demands much more from the OS when it comes to its IPC + capabilities. + + The security risk profiles of the two approaches will also be quite + different, since using subinterpreters won't require deliberately + poking holes in the process isolation that operating systems give + you by default. + +CPython has supported multiple interpreters, with increasing levels +of support, since version 1.5. While the feature has the potential +to be a powerful tool, it has suffered from neglect +because the multiple interpreter capabilities are not readily available +directly from Python. Exposing the existing functionality +in the stdlib will help reverse the situation. + +This proposal is focused on enabling the fundamental capability of +multiple interpreters, isolated from each other, +in the same Python process. This is a +new area for Python so there is relative uncertainly about the best +tools to provide as companions to interpreters. Thus we minimize +the functionality we add in the proposal as much as possible. + +Concerns +-------- + +* "subinterpreters are not worth the trouble" + +Some have argued that subinterpreters do not add sufficient benefit +to justify making them an official part of Python. Adding features +to the language (or stdlib) has a cost in increasing the size of +the language. So an addition must pay for itself. + +In this case, multiple interpreter support provide a novel concurrency +model focused on isolated threads of execution. Furthermore, they +provide an opportunity for changes in CPython that will allow +simultaneous use of multiple CPU cores (currently prevented +by the GIL--see :pep:`684`). + +Alternatives to subinterpreters include threading, async, and +multiprocessing. Threading is limited by the GIL and async isn't +the right solution for every problem (nor for every person). +Multiprocessing is likewise valuable in some but not all situations. +Direct IPC (rather than via the multiprocessing module) provides +similar benefits but with the same caveat. + +Notably, subinterpreters are not intended as a replacement for any of +the above. Certainly they overlap in some areas, but the benefits of +subinterpreters include isolation and (potentially) performance. In +particular, subinterpreters provide a direct route to an alternate +concurrency model (e.g. CSP) which has found success elsewhere and +will appeal to some Python users. That is the core value that the +``interpreters`` module will provide. + +* "stdlib support for multiple interpreters adds extra burden + on C extension authors" + +In the `Interpreter Isolation`_ section below we identify ways in +which isolation in CPython's subinterpreters is incomplete. Most +notable is extension modules that use C globals to store internal +state. (:pep:`3121` and :pep:`489` provide a solution to that problem, +followed by some extra APIs that improve efficiency, e.g. :pep:`573`). + +Consequently, projects that publish extension modules may face an +increased maintenance burden as their users start using subinterpreters, +where their modules may break. This situation is limited to modules +that use C globals (or use libraries that use C globals) to store +internal state. For numpy, the reported-bug rate is one every 6 +months. [bug-rate]_ + +Ultimately this comes down to a question of how often it will be a +problem in practice: how many projects would be affected, how often +their users will be affected, what the additional maintenance burden +will be for projects, and what the overall benefit of subinterpreters +is to offset those costs. The position of this PEP is that the actual +extra maintenance burden will be small and well below the threshold at +which subinterpreters are worth it. + +* "creating a new concurrency API deserves much more thought and + experimentation, so the new module shouldn't go into the stdlib + right away, if ever" + +Introducing an API for a new concurrency model, like happened with +asyncio, is an extremely large project that requires a lot of careful +consideration. It is not something that can be done as simply as this +PEP proposes and likely deserves significant time on PyPI to mature. +(See `Nathaniel's post `_ on python-dev.) + +However, this PEP does not propose any new concurrency API. +At most it exposes minimal tools (e.g. subinterpreters) +which may be used to write code that follows patterns associated with +(relatively) new-to-Python `concurrency models `_. +Those tools could also be used as the basis for APIs for such +concurrency models. Again, this PEP does not propose any such API. + +* "there is no point to exposing subinterpreters if they still share + the GIL" +* "the effort to make the GIL per-interpreter is disruptive and risky" + +A common misconception is that this PEP also includes a promise that +interpreters will no longer share the GIL. When that is clarified, +the next question is "what is the point?". This is already answered +at length in this PEP. Just to be clear, the value lies in:: + + * increase exposure of the existing feature, which helps improve + the code health of the entire CPython runtime + * expose the (mostly) isolated execution of interpreters + * preparation for per-interpreter GIL + * encourage experimentation + +* "data sharing can have a negative impact on cache performance + in multi-core scenarios" + +(See [cache-line-ping-pong]_.) + +This shouldn't be a problem for now as we have no immediate plans +to actually share data between interpreters, instead focusing +on copying. + + +About Subinterpreters +===================== + +Concurrency +----------- + +Concurrency is a challenging area of software development. Decades of +research and practice have led to a wide variety of concurrency models, +each with different goals. Most center on correctness and usability. + +One class of concurrency models focuses on isolated threads of +execution that interoperate through some message passing scheme. A +notable example is Communicating Sequential Processes [CSP]_ (upon +which Go's concurrency is roughly based). The intended isolation +inherent to CPython's interpreters makes them well-suited +to this approach. + +Shared Data +----------- + +CPython's interpreters are inherently isolated (with caveats +explained below), in contrast to threads. So the same +communicate-via-shared-memory approach doesn't work. Without an +alternative, effective use of concurrency via multiple interpreters +is significantly limited. + +The key challenge here is that sharing objects between interpreters +faces complexity due to various constraints on object ownership, +visibility, and mutability. At a conceptual level it's easier to +reason about concurrency when objects only exist in one interpreter +at a time. At a technical level, CPython's current memory model +limits how Python *objects* may be shared safely between interpreters; +effectively, objects are bound to the interpreter in which they were +created. Furthermore, the complexity of *object* sharing increases as +interpreters become more isolated, e.g. after GIL removal (though this +is mitigated somewhat for some "immortal" objects (see :pep:`683`). + +Consequently, the mechanism for sharing needs to be carefully considered. +There are a number of valid solutions, several of which may be +appropriate to support in Python. Earlier versions of this proposal +included a basic capability ("channels"), though most of the options +were quite similar. + +Note that the implementation of ``Interpreter.run()`` will be done +in a way that allows for may of these solutions to be implemented +independently and to coexist, but doing so is not technically +a part of the proposal here. + +The fundamental enabling feature for communication is that most objects +can be converted to some encoding of underlying raw data, which is safe +to be passed between interpreters. For example, an ``int`` object can +be turned into a C ``long`` value, send to another interpreter, and +turned back into an ``int`` object there. + +Regardless, the effort to determine the best way forward here is outside +the scope of this PEP. In the meantime, this proposal provides a basic +interim solution, described in `API For Sharing Data`_ below. + +Interpreter Isolation +--------------------- + +CPython's interpreters are intended to be strictly isolated from each +other. Each interpreter has its own copy of all modules, classes, +functions, and variables. The same applies to state in C, including in +extension modules. The CPython C-API docs explain more. [caveats]_ + +However, there are ways in which interpreters share some state. First +of all, some process-global state remains shared: + +* file descriptors +* builtin types (e.g. dict, bytes) +* singletons (e.g. None) +* underlying static module data (e.g. functions) for + builtin/extension/frozen modules + +There are no plans to change this. + +Second, some isolation is faulty due to bugs or implementations that did +not take subinterpreters into account. This includes things like +extension modules that rely on C globals. [cryptography]_ In these +cases bugs should be opened (some are already): + +* readline module hook functions (http://bugs.python.org/issue4202) +* memory leaks on re-init (http://bugs.python.org/issue21387) + +Finally, some potential isolation is missing due to the current design +of CPython. Improvements are currently going on to address gaps in this +area: + +* GC is not run per-interpreter [global-gc]_ +* at-exit handlers are not run per-interpreter [global-atexit]_ +* extensions using the ``PyGILState_*`` API are incompatible [gilstate]_ +* interpreters share memory management (e.g. allocators, gc) +* interpreters share the GIL + +Existing Usage +-------------- + +Multiple interpreter support is not a widely used feature. In fact, +the only documented cases of widespread usage are +`mod_wsgi `_, +`OpenStack Ceph `_, and +`JEP `_. On the one hand, these cases +provide confidence that existing multiple interpreter support is +relatively stable. On the other hand, there isn't much of a sample +size from which to judge the utility of the feature. + + +Alternate Python Implementations +================================ + +I've solicited feedback from various Python implementors about support +for subinterpreters. Each has indicated that they would be able to +support multiple interpreters in the same process (if they choose to) +without a lot of trouble. Here are the projects I contacted: + +* jython ([jython]_) +* ironpython (personal correspondence) +* pypy (personal correspondence) +* micropython (personal correspondence) + + +.. _interpreters-list-all: +.. _interpreters-get-current: +.. _interpreters-create: +.. _interpreters-Interpreter: + +"interpreters" Module API +========================= + +The module provides the following functions:: + + list_all() -> [Interpreter] + + Return a list of all existing interpreters. + + get_current() => Interpreter + + Return the currently running interpreter. + + get_main() => Interpreter + + Return the main interpreter. If the Python implementation + has no concept of a main interpreter then return None. + + create() -> Interpreter + + Initialize a new Python interpreter and return it. + It will remain idle until something is run in it and always + run in its own thread. + + +The module also provides the following classes:: + + class Interpreter(id): + + id -> int: + + The interpreter's ID. (read-only) + + is_running() -> bool: + + Return whether or not the interpreter's "run()" is currently + executing code. Code running in subthreads is ignored. + Calling this on the current interpreter will always return True. + + close(): + + Finalize and destroy the interpreter. + + This may not be called on an already running interpreter. + Doing so results in a RuntimeError. + + run(source_str, /): + + Run the provided Python source code in the interpreter, + in its __main__ module. + + This may not be called on an already running interpreter. + Doing so results in a RuntimeError. + + A "run()" call is similar to an exec() call (or calling + a function that returns None). Once "run()" completes, + the code that called "run()" continues executing (in the + original interpreter). Likewise, if there is any uncaught + exception then it effectively (see below) propagates into + the code where ``run()`` was called. Like exec() (and threads), + but unlike function calls, there is no return value. If any + "return" value from the code is needed, send the data out + via a pipe (os.pipe()). + + The big difference from exec() or functions is that "run()" + executes the code in an entirely different interpreter, + with entirely separate state. The interpreters are completely + isolated from each other, so the state of the original interpreter + (including the code it was executing in the current OS thread) + does not affect the state of the target interpreter + (the one that will execute the code). Likewise, the target + does not affect the original, nor any of its other threads. + + Instead, the state of the original interpreter (for this thread) + is frozen, and the code it's executing code completely blocks. + At that point, the target interpreter is given control of the + OS thread. Then, when it finishes executing, the original + interpreter gets control back and continues executing. + + So calling "run()" will effectively cause the current Python + thread to completely pause. Sometimes you won't want that pause, + in which case you should make the "run()" call in another thread. + To do so, add a function that calls "run()" and then run that + function in a normal "threading.Thread". + + Note that the interpreter's state is never reset, neither + before "run()" executes the code nor after. Thus the + interpreter state is preserved between calls to "run()". + This includes "sys.modules", the "builtins" module, and the + internal state of C extension modules. + + Also note that "run()" executes in the namespace of the + "__main__" module, just like scripts, the REPL, "-m", and + "-c". Just as the interpreter's state is not ever reset, the + "__main__" module is never reset. You can imagine + concatenating the code from each "run()" call into one long + script. This is the same as how the REPL operates. + + Supported code: source text. + +Uncaught Exceptions +------------------- + +Regarding uncaught exceptions in ``Interpreter.run()``, we noted that +they are "effectively" propagated into the code where ``run()`` was +called. To prevent leaking exceptions (and tracebacks) between +interpreters, we create a surrogate of the exception and its traceback +(see :class:`traceback.TracebackException`), set it to ``__cause__`` +on a new ``RunFailedError``, and raise that. + +Directly raising (a proxy of) the exception is problematic since it's +harder to distinguish between an error in the ``run()`` call and an +uncaught exception from the subinterpreter. + +API For Sharing Data +-------------------- + +As discussed in `Shared Data`_ above, multiple interpreter support +is less useful without a mechanism for sharing data (communicating) +between them. Sharing actual Python objects between interpreters, +however, has enough potential problems that we are avoiding support +for that in this proposal. Nor, as mentioned earlier, are we adding +anything more than the most minimal mechanism for communication. + +That very basic mechanism, using pipes (see ``os.pipe()``), will allow +users to send data (bytes) from one interpreter to another. We'll +take a closer look in a moment. Fundamentally, it's a simple +application of the underlying sharing capability proposed here. + +The various aspects of the approach, including keeping the API minimal, +helps us avoid further exposing any underlying complexity +to Python users. + +Communicating Through OS Pipes +'''''''''''''''''''''''''''''' + +As noted, this proposal enables a very basic mechanism for +communicating between interpreters, which makes use of +``Interpreter.run()``: + +1. interpreter A calls ``os.pipe()`` to get a read/write pair + of file descriptors (both ``int`` objects) +2. interpreter A calls ``run()`` on interpreter B, including + the read FD via string formatting +3. interpreter A writes some bytes to the write FD +4. interpreter B reads those bytes + +Several of the earlier examples demonstrate this, such as +`Synchronize using an OS pipe`_. + + +Interpreter Restrictions +======================== + +Every new interpreter created by ``interpreters.create()`` +now has specific restrictions on any code it runs. This includes the +following: + +* importing an extension module fails if it does not implement + multi-phase init +* daemon threads may not be created +* ``os.fork()`` is not allowed (so no ``multiprocessing``) +* ``os.exec*()`` is not allowed + (but "fork+exec", a la ``subprocess`` is okay) + +Note that interpreters created with the existing C-API do not have these +restrictions. The same is true for the "main" interpreter, so +existing use of Python will not change. + +.. Mention the similar restrictions in PEP 684? + +We may choose to later loosen some of the above restrictions or provide +a way to enable/disable granular restrictions individually. Regardless, +requiring multi-phase init from extension modules will always be a +default restriction. + + +Documentation +============= + +The new stdlib docs page for the ``interpreters`` module will include +the following: + +* (at the top) a clear note that support for multiple interpreters + is not required from extension modules +* some explanation about what subinterpreters are +* brief examples of how to use multiple interpreters + (and communicating between them) +* a summary of the limitations of using multiple interpreters +* (for extension maintainers) a link to the resources for ensuring + multiple interpreters compatibility +* much of the API information in this PEP + +Docs about resources for extension maintainers already exist on the +`Isolating Extension Modules `_ howto page. Any +extra help will be added there. For example, it may prove helpful +to discuss strategies for dealing with linked libraries that keep +their own subinterpreter-incompatible global state. + +.. _isolation-howto: + https://docs.python.org/3/howto/isolating-extensions.html + +Note that the documentation will play a large part in mitigating any +negative impact that the new ``interpreters`` module might have on +extension module maintainers. + +Also, the ``ImportError`` for incompatible extension modules will have +a message that clearly says it is due to missing multiple interpreters +compatibility and that extensions are not required to provide it. This +will help set user expectations properly. + +Alternative Solutions +===================== + +One possible alternative to a new module is to add support for interpreters +to ``concurrent.futures``. There are several reasons why that wouldn't work: + +* the obvious place to look for multiple interpreters support + is an "interpreters" module, much as with "threading", etc. +* ``concurrent.futures`` is all about executing functions + but currently we don't have a good way to run a function + from one interpreter in another + +Similar reasoning applies for support in the ``multiprocessing`` module. + + +Deferred Functionality +====================== + +In the interest of keeping this proposal minimal, the following +functionality has been left out for future consideration. Note that +this is not a judgement against any of said capability, but rather a +deferment. That said, each is arguably valid. + +Shareable Objects +----------------- + +Earlier versions of this proposal included a mechanism by which the +data underlying a given object could be passed to another interpreter +or even shared, even if the object can't be. Without channels there +isn't enough benefit to keep the concept of shareable objects around. + +Interpreter.call() +------------------ + +It would be convenient to run existing functions in subinterpreters +directly. ``Interpreter.run()`` could be adjusted to support this or +a ``call()`` method could be added:: + + Interpreter.call(f, *args, **kwargs) + +This suffers from the same problem as sharing objects between +interpreters via queues. The minimal solution (running a source string) +is sufficient for us to get the feature out where it can be explored. + +Interpreter.run_in_thread() +--------------------------- + +This method would make a ``run()`` call for you in a thread. Doing this +using only ``threading.Thread`` and ``run()`` is relatively trivial so +we've left it out. + +Synchronization Primitives +-------------------------- + +The ``threading`` module provides a number of synchronization primitives +for coordinating concurrent operations. This is especially necessary +due to the shared-state nature of threading. In contrast, +interpreters do not share state. Data sharing is restricted to the +runtime's shareable objects capability, which does away with the need +for explicit synchronization. If any sort of opt-in shared state +support is added to CPython's interpreters in the future, that same +effort can introduce synchronization primitives to meet that need. + +CSP Library +----------- + +A ``csp`` module would not be a large step away from the functionality +provided by this PEP. However, adding such a module is outside the +minimalist goals of this proposal. + +Syntactic Support +----------------- + +The ``Go`` language provides a concurrency model based on CSP, +so it's similar to the concurrency model that multiple interpreters +support. However, ``Go`` also provides syntactic support, as well as +several builtin concurrency primitives, to make concurrency a +first-class feature. Conceivably, similar syntactic (and builtin) +support could be added to Python using interpreters. However, +that is *way* outside the scope of this PEP! + +Multiprocessing +--------------- + +The ``multiprocessing`` module could support interpreters in the same +way it supports threads and processes. In fact, the module's +maintainer, Davin Potts, has indicated this is a reasonable feature +request. However, it is outside the narrow scope of this PEP. + +C-extension opt-in/opt-out +-------------------------- + +By using the ``PyModuleDef_Slot`` introduced by :pep:`489`, we could +easily add a mechanism by which C-extension modules could opt out of +multiple interpreter support. Then the import machinery, when operating +in a subinterpreter, would need to check the module for support. +It would raise an ImportError if unsupported. + +Alternately we could support opting in to multiple interpreters support. +However, that would probably exclude many more modules (unnecessarily) +than the opt-out approach. Also, note that :pep:`489` defined that an +extension's use of the PEP's machinery implies multiple interpreters +support. + +The scope of adding the ModuleDef slot and fixing up the import +machinery is non-trivial, but could be worth it. It all depends on +how many extension modules break under subinterpreters. Given that +there are relatively few cases we know of through mod_wsgi, we can +leave this for later. + +Resetting __main__ +------------------ + +As proposed, every call to ``Interpreter.run()`` will execute in the +namespace of the interpreter's existing ``__main__`` module. This means +that data persists there between ``run()`` calls. Sometimes this isn't +desirable and you want to execute in a fresh ``__main__``. Also, +you don't necessarily want to leak objects there that you aren't using +any more. + +Note that the following won't work right because it will clear too much +(e.g. ``__name__`` and the other "__dunder__" attributes:: + + interp.run('globals().clear()') + +Possible solutions include: + +* a ``create()`` arg to indicate resetting ``__main__`` after each + ``run`` call +* an ``Interpreter.reset_main`` flag to support opting in or out + after the fact +* an ``Interpreter.reset_main()`` method to opt in when desired +* ``importlib.util.reset_globals()`` [reset_globals]_ + +Also note that resetting ``__main__`` does nothing about state stored +in other modules. So any solution would have to be clear about the +scope of what is being reset. Conceivably we could invent a mechanism +by which any (or every) module could be reset, unlike ``reload()`` +which does not clear the module before loading into it. + +Regardless, since ``__main__`` is the execution namespace of the +interpreter, resetting it has a much more direct correlation to +interpreters and their dynamic state than does resetting other modules. +So a more generic module reset mechanism may prove unnecessary. + +This isn't a critical feature initially. It can wait until later +if desirable. + +Resetting an interpreter's state +-------------------------------- + +It may be nice to re-use an existing subinterpreter instead of +spinning up a new one. Since an interpreter has substantially more +state than just the ``__main__`` module, it isn't so easy to put an +interpreter back into a pristine/fresh state. In fact, there *may* +be parts of the state that cannot be reset from Python code. + +A possible solution is to add an ``Interpreter.reset()`` method. This +would put the interpreter back into the state it was in when newly +created. If called on a running interpreter it would fail (hence the +main interpreter could never be reset). This would likely be more +efficient than creating a new interpreter, though that depends on +what optimizations will be made later to interpreter creation. + +While this would potentially provide functionality that is not +otherwise available from Python code, it isn't a fundamental +functionality. So in the spirit of minimalism here, this can wait. +Regardless, I doubt it would be controversial to add it post-PEP. + +Copy an existing interpreter's state +------------------------------------ + +Relatedly, it may be useful to support creating a new interpreter +based on an existing one, e.g. ``Interpreter.copy()``. This ties +into the idea that a snapshot could be made of an interpreter's memory, +which would make starting up CPython, or creating new interpreters, +faster in general. The same mechanism could be used for a +hypothetical ``Interpreter.reset()``, as described previously. + +Shareable file descriptors and sockets +-------------------------------------- + +Given that file descriptors and sockets are process-global resources, +making them shareable is a reasonable idea. They would be a good +candidate for the first effort at expanding the supported shareable +types. They aren't strictly necessary for the initial API. + +Integration with async +---------------------- + +Per Antoine Pitrou [async]_:: + + Has any thought been given to how FIFOs could integrate with async + code driven by an event loop (e.g. asyncio)? I think the model of + executing several asyncio (or Tornado) applications each in their + own subinterpreter may prove quite interesting to reconcile multi- + core concurrency with ease of programming. That would require the + FIFOs to be able to synchronize on something an event loop can wait + on (probably a file descriptor?). + +The basic functionality of multiple interpreters support does not depend +on async and can be added later. + +channels +-------- + +We could introduce some relatively efficient, native data types for +passing data between interpreters, to use instead of OS pipes. Earlier +versions of this PEP introduced one such mechanism, called "channels". +This can be pursued later. + +Pipes and Queues +---------------- + +With the proposed object passing mechanism of "os.pipe()", other similar +basic types aren't strictly required to achieve the minimal useful +functionality of multiple interpreters. Such types include pipes +(like unbuffered channels, but one-to-one) and queues (like channels, +but more generic). See below in `Rejected Ideas`_ for more information. + +Even though these types aren't part of this proposal, they may still +be useful in the context of concurrency. Adding them later is entirely +reasonable. The could be trivially implemented as wrappers around +channels. Alternatively they could be implemented for efficiency at the +same low level as channels. + +Support inheriting settings (and more?) +--------------------------------------- + +Folks might find it useful, when creating a new interpreter, to be +able to indicate that they would like some things "inherited" by the +new interpreter. The mechanism could be a strict copy or it could be +copy-on-write. The motivating example is with the warnings module +(e.g. copy the filters). + +The feature isn't critical, nor would it be widely useful, so it +can wait until there's interest. Notably, both suggested solutions +will require significant work, especially when it comes to complex +objects and most especially for mutable containers of mutable +complex objects. + +Make exceptions shareable +------------------------- + +Exceptions are propagated out of ``run()`` calls, so it isn't a big +leap to make them shareable. However, as noted elsewhere, +it isn't essential or (particularly common) so we can wait on doing +that. + +Make RunFailedError.__cause__ lazy +---------------------------------- + +An uncaught exception in a subinterpreter (from ``run()``) is copied +to the calling interpreter and set as ``__cause__`` on a +``RunFailedError`` which is then raised. That copying part involves +some sort of deserialization in the calling interpreter, which can be +expensive (e.g. due to imports) yet is not always necessary. + +So it may be useful to use an ``ExceptionProxy`` type to wrap the +serialized exception and only deserialize it when needed. That could +be via ``ExceptionProxy__getattribute__()`` or perhaps through +``RunFailedError.resolve()`` (which would raise the deserialized +exception and set ``RunFailedError.__cause__`` to the exception. + +It may also make sense to have ``RunFailedError.__cause__`` be a +descriptor that does the lazy deserialization (and set ``__cause__``) +on the ``RunFailedError`` instance. + +Make everything shareable through serialization +----------------------------------------------- + +We could use pickle (or marshal) to serialize everything and thus +make them shareable. Doing this is potentially inefficient, +but it may be a matter of convenience in the end. +We can add it later, but trying to remove it later +would be significantly more painful. + +Return a value from ``run()`` +----------------------------- + +Currently ``run()`` always returns None. One idea is to return the +return value from whatever the subinterpreter ran. However, for now +it doesn't make sense. The only thing folks can run is a string of +code (i.e. a script). This is equivalent to ``PyRun_StringFlags()``, +``exec()``, or a module body. None of those "return" anything. We can +revisit this once ``run()`` supports functions, etc. + +Add a shareable synchronization primitive +----------------------------------------- + +This would be ``_threading.Lock`` (or something like it) where +interpreters would actually share the underlying mutex. The main +concern is that locks and isolated interpreters may not mix well +(as learned in Go). + +We can add this later if it proves desirable without much trouble. + +Propagate SystemExit and KeyboardInterrupt Differently +------------------------------------------------------ + +The exception types that inherit from ``BaseException`` (aside from +``Exception``) are usually treated specially. These types are: +``KeyboardInterrupt``, ``SystemExit``, and ``GeneratorExit``. It may +make sense to treat them specially when it comes to propagation from +``run()``. Here are some options:: + + * propagate like normal via RunFailedError + * do not propagate (handle them somehow in the subinterpreter) + * propagate them directly (avoid RunFailedError) + * propagate them directly (set RunFailedError as __cause__) + +We aren't going to worry about handling them differently. Threads +already ignore ``SystemExit``, so for now we will follow that pattern. + + +Rejected Ideas +============== + +Add an API based on pipes +------------------------- + +(Earlier versions of this PEP proposed "channels" for communicating +between interpreters. This idea is written relative to that.) + +A pipe would be a simplex FIFO between exactly two interpreters. For +most use cases this would be sufficient. It could potentially simplify +the implementation as well. However, it isn't a big step to supporting +a many-to-many simplex FIFO via channels. Also, with pipes the API +ends up being slightly more complicated, requiring naming the pipes. + +Add an API based on queues +-------------------------- + +(Earlier versions of this PEP proposed "channels" for communicating +between interpreters. This idea is written relative to that.) + +Queues and buffered channels are almost the same thing. The main +difference is that channels have a stronger relationship with context +(i.e. the associated interpreter). + +The name "Channel" was used instead of "Queue" to avoid confusion with +the stdlib ``queue.Queue``. + +"enumerate" +----------- + +The ``list_all()`` function provides the list of all interpreters. +In the threading module, which partly inspired the proposed API, the +function is called ``enumerate()``. The name is different here to +avoid confusing Python users that are not already familiar with the +threading API. For them "enumerate" is rather unclear, whereas +"list_all" is clear. + +Alternate solutions to prevent leaking exceptions across interpreters +--------------------------------------------------------------------- + +In function calls, uncaught exceptions propagate to the calling frame. +The same approach could be taken with ``run()``. However, this would +mean that exception objects would leak across the inter-interpreter +boundary. Likewise, the frames in the traceback would potentially leak. + +While that might not be a problem currently, it would be a problem once +interpreters get better isolation relative to memory management (which +is necessary to stop sharing the GIL between interpreters). We've +resolved the semantics of how the exceptions propagate by raising a +``RunFailedError`` instead, for which ``__cause__`` wraps a safe proxy +for the original exception and traceback. + +Rejected possible solutions: + +* reproduce the exception and traceback in the original interpreter + and raise that. +* raise a subclass of RunFailedError that proxies the original + exception and traceback. +* raise RuntimeError instead of RunFailedError +* convert at the boundary (a la ``subprocess.CalledProcessError``) + (requires a cross-interpreter representation) +* support customization via ``Interpreter.excepthook`` + (requires a cross-interpreter representation) +* wrap in a proxy at the boundary (including with support for + something like ``err.raise()`` to propagate the traceback). +* return the exception (or its proxy) from ``run()`` instead of + raising it +* return a result object (like ``subprocess`` does) [result-object]_ + (unnecessary complexity?) +* throw the exception away and expect users to deal with unhandled + exceptions explicitly in the script they pass to ``run()`` + (they can pass error info out via ``os.pipe()``); + with threads you have to do something similar + +Always associate each new interpreter with its own thread +--------------------------------------------------------- + +As implemented in the C-API, an interpreter is not inherently tied to +any thread. Furthermore, it will run in any existing thread, whether +created by Python or not. You only have to activate one of its thread +states (``PyThreadState``) in the thread first. This means that the +same thread may run more than one interpreter (though obviously +not at the same time). + +The proposed module maintains this behavior. Interpreters are not +tied to threads. Only calls to ``Interpreter.run()`` are. However, +one of the key objectives of this PEP is to provide a more +human-centric concurrency model. With that in mind, from a conceptual +standpoint the module *might* be easier to understand if each +interpreter were associated with its own thread. + +That would mean ``interpreters.create()`` would create a new thread +and ``Interpreter.run()`` would only execute in that thread (and +nothing else would). The benefit is that users would not have to +wrap ``Interpreter.run()`` calls in a new ``threading.Thread``. Nor +would they be in a position to accidentally pause the current +interpreter (in the current thread) while their interpreter +executes. + +The idea is rejected because the benefit is small and the cost is high. +The difference from the capability in the C-API would be potentially +confusing. The implicit creation of threads is magical. The early +creation of threads is potentially wasteful. The inability to run +arbitrary interpreters in an existing thread would prevent some valid +use cases, frustrating users. Tying interpreters to threads would +require extra runtime modifications. It would also make the module's +implementation overly complicated. Finally, it might not even make +the module easier to understand. + +Allow multiple simultaneous calls to Interpreter.run() +------------------------------------------------------ + +This would make sense especially if ``Interpreter.run()`` were to +manage new threads for you (which we've rejected). Essentially, +each call would run independently, which would be mostly fine +from a narrow technical standpoint, since each interpreter +can have multiple threads. + +The problem is that the interpreter has only one ``__main__`` module +and simultaneous ``Interpreter.run()`` calls would have to sort out +sharing ``__main__`` or we'd have to invent a new mechanism. Neither +would be simple enough to be worth doing. + +Add a "reraise" method to RunFailedError +---------------------------------------- + +While having ``__cause__`` set on ``RunFailedError`` helps produce a +more useful traceback, it's less helpful when handling the original +error. To help facilitate this, we could add +``RunFailedError.reraise()``. This method would enable the following +pattern:: + + try: + try: + interp.run(script) + except RunFailedError as exc: + exc.reraise() + except MyException: + ... + +This would be made even simpler if there existed a ``__reraise__`` +protocol. + +All that said, this is completely unnecessary. Using ``__cause__`` +is good enough:: + + try: + try: + interp.run(script) + except RunFailedError as exc: + raise exc.__cause__ + except MyException: + ... + +Note that in extreme cases it may require a little extra boilerplate:: + + try: + try: + interp.run(script) + except RunFailedError as exc: + if exc.__cause__ is not None: + raise exc.__cause__ + raise # re-raise + except MyException: + ... + + +Implementation +============== + +The implementation of the PEP has 4 parts: + +* the high-level module described in this PEP (mostly a light wrapper + around a low-level C extension +* the low-level C extension module +* additions to the ("private") C=API needed by the low-level module +* secondary fixes/changes in the CPython runtime that facilitate + the low-level module (among other benefits) + +These are at various levels of completion, with more done the lower +you go: + +* the high-level module has been, at best, roughly implemented. + However, fully implementing it will be almost trivial. +* the low-level module is mostly complete. The bulk of the + implementation was merged into master in December 2018 as the + "_xxsubinterpreters" module (for the sake of testing multiple + interpreters functionality). Only 3 parts of the implementation + remain: "send_wait()", "send_buffer()", and exception propagation. + All three have been mostly finished, but were blocked by work + related to ceval. That blocker is basically resolved now and + finishing the low-level will not require extensive work. +* all necessary C-API work has been finished +* all anticipated work in the runtime has been finished + +The implementation effort for :pep:`554` is being tracked as part of +a larger project aimed at improving multi-core support in CPython. +[multi-core-project]_ + + +References +========== + +.. [c-api] + https://docs.python.org/3/c-api/init.html#sub-interpreter-support + +.. [CSP] + https://en.wikipedia.org/wiki/Communicating_sequential_processes + https://github.com/futurecore/python-csp + +.. [caveats] + https://docs.python.org/3/c-api/init.html#bugs-and-caveats + +.. [cryptography] + https://github.com/pyca/cryptography/issues/2299 + +.. [global-gc] + http://bugs.python.org/issue24554 + +.. [gilstate] + https://bugs.python.org/issue10915 + http://bugs.python.org/issue15751 + +.. [global-atexit] + https://bugs.python.org/issue6531 + +.. [bug-rate] + https://mail.python.org/pipermail/python-ideas/2017-September/047094.html + +.. [benefits] + https://mail.python.org/pipermail/python-ideas/2017-September/047122.html + +.. [reset_globals] + https://mail.python.org/pipermail/python-dev/2017-September/149545.html + +.. [async] + https://mail.python.org/pipermail/python-dev/2017-September/149420.html + https://mail.python.org/pipermail/python-dev/2017-September/149585.html + +.. [result-object] + https://mail.python.org/pipermail/python-dev/2017-September/149562.html + +.. [jython] + https://mail.python.org/pipermail/python-ideas/2017-May/045771.html + +.. [multi-core-project] + https://github.com/ericsnowcurrently/multi-core-python + +.. [cache-line-ping-pong] + https://mail.python.org/archives/list/python-dev@python.org/message/3HVRFWHDMWPNR367GXBILZ4JJAUQ2STZ/ + +.. _nathaniel-asyncio: + https://mail.python.org/archives/list/python-dev@python.org/message/TUEAZNZHVJGGLL4OFD32OW6JJDKM6FAS/ + +* mp-conn + https://docs.python.org/3/library/multiprocessing.html#connection-objects + +* main-thread + https://mail.python.org/pipermail/python-ideas/2017-September/047144.html + https://mail.python.org/pipermail/python-dev/2017-September/149566.html + +* petr-c-ext + https://mail.python.org/pipermail/import-sig/2016-June/001062.html + https://mail.python.org/pipermail/python-ideas/2016-April/039748.html + +Copyright +========= + +This document has been placed in the public domain. diff --git a/pep-0555.rst b/peps/pep-0555.rst similarity index 97% rename from pep-0555.rst rename to peps/pep-0555.rst index f001601aa..23e7b7893 100644 --- a/pep-0555.rst +++ b/peps/pep-0555.rst @@ -359,7 +359,7 @@ For code that does not use ``contextvars``, the additions are O(1) and essential More on implementation '''''''''''''''''''''' -The rest of the functionality, including ``contextvars.leaking_yields``, contextvars.capture()``, ``contextvars.get_local_state()`` and ``contextvars.clean_context()`` are in fact quite straightforward to implement, but their implementation will be discussed further in later versions of this proposal. Caching of assigned values is somewhat more complicated, and will be discussed later, but it seems that most cases should achieve O(1) complexity. +The rest of the functionality, including ``contextvars.leaking_yields``, ``contextvars.capture()``, ``contextvars.get_local_state()`` and ``contextvars.clean_context()`` are in fact quite straightforward to implement, but their implementation will be discussed further in later versions of this proposal. Caching of assigned values is somewhat more complicated, and will be discussed later, but it seems that most cases should achieve O(1) complexity. Backwards compatibility ======================= @@ -403,13 +403,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0556.rst b/peps/pep-0556.rst similarity index 99% rename from pep-0556.rst rename to peps/pep-0556.rst index d8b060902..2f70120fb 100644 --- a/pep-0556.rst +++ b/peps/pep-0556.rst @@ -6,7 +6,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 08-Sep-2017 Python-Version: 3.7 -Post-History: 2017-09-08 +Post-History: 08-Sep-2017 Deferral Notice diff --git a/pep-0557.rst b/peps/pep-0557.rst similarity index 97% rename from pep-0557.rst rename to peps/pep-0557.rst index 9520272d8..44d17286e 100644 --- a/pep-0557.rst +++ b/peps/pep-0557.rst @@ -14,7 +14,7 @@ Notice for Reviewers This PEP and the initial implementation were drafted in a separate repo: https://github.com/ericvsmith/dataclasses. Before commenting in -a public forum please at least read the `discussion`_ listed at the +a public forum please at least read the discussion_ listed at the end of this PEP. Abstract @@ -28,7 +28,7 @@ inheritance, metaclasses, docstrings, user-defined methods, class factories, and other Python class features. A class decorator is provided which inspects a class definition for -variables with type annotations as defined in PEP 526, "Syntax for +variables with type annotations as defined in :pep:`526`, "Syntax for Variable Annotations". In this document, such variables are called fields. Using these fields, the decorator adds generated method definitions to the class to support instance initialization, a repr, @@ -108,7 +108,7 @@ Some examples include: So, why is this PEP needed? -With the addition of PEP 526, Python has a concise way to specify the +With the addition of :pep:`526`, Python has a concise way to specify the type of class members. This PEP leverages that syntax to provide a simple, unobtrusive way to describe Data Classes. With two exceptions, the specified attribute type annotation is completely ignored by Data @@ -121,7 +121,7 @@ interference from Data Classes. The decorated classes are truly interfere with any usage of the class. One main design goal of Data Classes is to support static type -checkers. The use of PEP 526 syntax is one example of this, but so is +checkers. The use of :pep:`526` syntax is one example of this, but so is the design of the ``fields()`` function and the ``@dataclass`` decorator. Due to their very dynamic nature, some of the libraries mentioned above are difficult to use with static type checkers. @@ -139,8 +139,6 @@ Where is it not appropriate to use Data Classes? - Type validation beyond that provided by PEPs 484 and 526 is required, or value validation or conversion is required. -.. _Specification: - Specification ============= @@ -400,7 +398,7 @@ Class variables --------------- One place where ``dataclass`` actually inspects the type of a field is -to determine if a field is a class variable as defined in PEP 526. It +to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is ``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded from consideration as a field and is ignored by the Data Class mechanisms. For more @@ -666,8 +664,6 @@ Module level helper functions def is_dataclass_instance(obj): return is_dataclass(obj) and not isinstance(obj, type) -.. _discussion: - Discussion ========== @@ -676,7 +672,7 @@ python-ideas discussion This discussion started on python-ideas [#]_ and was moved to a GitHub repo [#]_ for further discussion. As part of this discussion, we made -the decision to use PEP 526 syntax to drive the discovery of fields. +the decision to use :pep:`526` syntax to drive the discovery of fields. Support for automatically setting ``__slots__``? ------------------------------------------------ @@ -810,7 +806,7 @@ asdict and astuple function names --------------------------------- The names of the module-level helper functions ``asdict()`` and -``astuple()`` are arguably not PEP 8 compliant, and should be +``astuple()`` are arguably not :pep:`8` compliant, and should be ``as_dict()`` and ``as_tuple()``, respectively. However, after discussion [#]_ it was decided to keep consistency with ``namedtuple._asdict()`` and ``attr.asdict()``. @@ -949,8 +945,7 @@ References .. [#] Python documentation for __hash__ (https://docs.python.org/3/reference/datamodel.html#object.__hash__) -.. [#] ClassVar discussion in PEP 526 - (https://www.python.org/dev/peps/pep-0526/#class-and-instance-variable-annotations) +.. [#] :pep:`ClassVar discussion in PEP 526 <526#class-and-instance-variable-annotations>` .. [#] Start of python-ideas discussion (https://mail.python.org/pipermail/python-ideas/2017-May/045618.html) @@ -964,7 +959,7 @@ References .. [#] why not just attrs? (https://github.com/ericvsmith/dataclasses/issues/19) -.. [#] PEP 8 names for asdict and astuple +.. [#] :pep:`8` names for asdict and astuple (https://github.com/ericvsmith/dataclasses/issues/110) .. [#] Copying mutable defaults @@ -975,13 +970,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0558.rst b/peps/pep-0558.rst similarity index 69% rename from pep-0558.rst rename to peps/pep-0558.rst index c1c3dc741..2ec688ba9 100644 --- a/pep-0558.rst +++ b/peps/pep-0558.rst @@ -2,14 +2,41 @@ PEP: 558 Title: Defined semantics for locals() Author: Nick Coghlan BDFL-Delegate: Nathaniel J. Smith -Discussions-To: -Status: Draft +Discussions-To: python-dev@python.org +Status: Deferred Type: Standards Track Content-Type: text/x-rst Created: 08-Sep-2017 -Python-Version: 3.11 -Post-History: 2017-09-08, 2019-05-22, 2019-05-30, 2019-12-30, 2021-07-18, 2021-08-26 +Python-Version: 3.13 +Post-History: 08-Sep-2017, 22-May-2019, 30-May-2019, 30-Dec-2019, 18-Jul-2021, + 26-Aug-2021 +PEP Deferral +============ + +While the original reference implementation for this PEP served a useful purpose +in determining that it *is* feasible to solve the known problems with the +semantics of the ``locals()`` namespace, that implementation itself is no longer +viable as an implementation of the PEP as written. + +This situation came about for two main reasons: + +* first, several key parts of the original implementation were invalidated when + the semantic improvements arising from the development of :pep:`667` resulted in + the proposal in the PEP diverging from what had previously been implemented +* second, several remaining practical aspects of the original implementation were + invalidated by the frame management changes that formed part of the significant + performance improvements published in the CPython 3.11 release, so the development + branch doesn't even offer a useful starting point for an updated implementation + +As a result, further progress on this PEP requires the development of a new reference +implementation, either for this PEP or for :pep:`667` (the semantics of the two PEPs +are so close together now that an implementation for either would require only a few +relatively small changes to instead implement the other). + +Since the PEP author has no current plans to produce that new reference implementation +themselves, but the semantics proposed by the PEPs seem generally acceptable to the +Python core development team, the most appropriate state for the PEP is "Deferred". Abstract ======== @@ -23,7 +50,9 @@ behaviour at function scope to make it more predictable and independent of the presence or absence of tracing functions. In addition, it proposes that the following functions be added to the stable -Python C API/ABI:: +Python C API/ABI: + +.. code-block:: c typedef enum { PyLocals_UNDEFINED = -1, @@ -35,7 +64,6 @@ Python C API/ABI:: PyLocals_Kind PyLocals_GetKind(); PyObject * PyLocals_Get(); PyObject * PyLocals_GetCopy(); - PyObject * PyLocals_GetView(); It also proposes the addition of several supporting functions and type definitions to the CPython C API. @@ -145,7 +173,7 @@ builtin to read as follows: dictionaries. -There would also be a versionchanged note for the release making this change: +There would also be a ``versionchanged`` note for the release making this change: In prior versions, the semantics of mutating the mapping object returned from ``locals()`` were formally undefined. In CPython specifically, @@ -273,18 +301,20 @@ Summary of proposed implementation-specific changes * Changes are made as necessary to provide the updated Python level semantics * Two new functions are added to the stable ABI to replicate the updated - behaviour of the Python ``locals()`` builtin:: + behaviour of the Python ``locals()`` builtin: + +.. code-block:: c PyObject * PyLocals_Get(); PyLocals_Kind PyLocals_GetKind(); + * One new function is added to the stable ABI to efficiently get a snapshot of - the local namespace in the running frame:: + the local namespace in the running frame: + +.. code-block:: c PyObject * PyLocals_GetCopy(); -* One new function is added to the stable ABI to get a read-only view of the - local namespace in the running frame:: - PyObject * PyLocals_GetView(); * Corresponding frame accessor functions for these new public APIs are added to the CPython frame C API * On optimised frames, the Python level ``f_locals`` API will return dynamically @@ -309,7 +339,7 @@ Summary of proposed implementation-specific changes mutable read/write mapping for the local variables. * The trace hook implementation will no longer call ``PyFrame_FastToLocals()`` implicitly. The version porting guide will recommend migrating to - ``PyFrame_GetLocalsView()`` for read-only access and + ``PyFrame_GetLocals()`` for read-only access and ``PyObject_GetAttrString(frame, "f_locals")`` for read/write access. @@ -379,6 +409,7 @@ retained for two key purposes: fast locals array (e.g. the ``__return__`` and ``__exception__`` keys set by ``pdb`` when tracing code execution for debugging purposes) + With the changes in this PEP, this internal frame value cache is no longer directly accessible from Python code (whereas historically it was both returned by the ``locals()`` builtin and available as the ``frame.f_locals`` @@ -397,50 +428,46 @@ Fast locals proxy objects and the internal frame value cache returned by to the frame itself, and will only be reliably visible via fast locals proxies for the same frame if the change relates to extra variables that don't have slots in the frame's fast locals array -* changes made by executing code in the frame will be visible to newly created - fast locals proxy objects, when directly accessing specific keys on existing - fast locals proxy objects, and when performing intrinsically O(n) operations - on existing fast locals proxy objects. Visibility in the internal frame value - cache (and in fast locals proxy operations that rely on the frame) cache is - subject to the cache update guidelines discussed in the next section +* changes made by executing code in the frame will be immediately visible to all + fast locals proxy objects for that frame (both existing proxies and newly + created ones). Visibility in the internal frame value cache cache returned + by ``PyEval_GetLocals()`` is subject to the cache update guidelines discussed + in the next section -Due to the last point, the frame API documentation will recommend that a new -``frame.f_locals`` reference be retrieved whenever an optimised frame (or -a related frame) might have been running code that binds or unbinds local -variable or cell references, and the code iterates over the proxy, checks -its length, or calls ``popitem()``. This will be the most natural style of use -in tracing function implementations, as those are passed references to frames -rather than directly to ``frames.f_locals``. +As a result of these points, only code using ``PyEval_GetLocals()``, +``PyLocals_Get()``, or ``PyLocals_GetCopy()`` will need to be concerned about +the frame value cache potentially becoming stale. Code using the new frame fast +locals proxy API (whether from Python or from C) will always see the live state +of the frame. Fast locals proxy implementation details ---------------------------------------- -Each fast locals proxy instance has two internal attributes that are not +Each fast locals proxy instance has a single internal attribute that is not exposed as part of the Python runtime API: * *frame*: the underlying optimised frame that the proxy provides access to -* *frame_cache_updated*: whether this proxy has already updated the frame's - internal value cache at least once In addition, proxy instances use and update the following attributes stored on the -underlying frame: +underlying frame or code object: -* *fast_refs*: a hidden mapping from variable names to either fast local storage - offsets (for local variables) or to closure cells (for closure variables). - This mapping is lazily initialized on the first frame read or write access - through a fast locals proxy, rather than being eagerly populated as soon as - the first fast locals proxy is created. +* *_name_to_offset_mapping*: a hidden mapping from variable names to fast local + storage offsets. This mapping is lazily initialized on the first frame read or + write access through a fast locals proxy, rather than being eagerly populated + as soon as the first fast locals proxy is created. Since the mapping is + identical for all frames running a given code object, a single copy is stored + on the code object, rather than each frame object populating its own mapping * *locals*: the internal frame value cache returned by the ``PyEval_GetLocals()`` C API and updated by the ``PyFrame_FastToLocals()`` C API. This is the mapping that the ``locals()`` builtin returns in Python 3.10 and earlier. -``__getitem__`` operations on the proxy will populate the ``fast_refs`` mapping -(if it is not already populated), and then either return the relevant value -(if the key is found in either the ``fast_refs`` mapping or the internal frame -value cache), or else raise ``KeyError``. Variables that are defined on the -frame but not currently bound raise ``KeyError`` (just as they're omitted from -the result of ``locals()``). +``__getitem__`` operations on the proxy will populate the ``_name_to_offset_mapping`` +on the code object (if it is not already populated), and then either return the +relevant value (if the key is found in either the ``_name_to_offset_mapping`` +mapping or the internal frame value cache), or else raise ``KeyError``. Variables +that are defined on the frame but not currently bound also raise ``KeyError`` +(just as they're omitted from the result of ``locals()``). As the frame storage is always accessed directly, the proxy will automatically pick up name binding and unbinding operations that take place as the function @@ -453,8 +480,7 @@ directly affect the corresponding fast local or cell reference on the underlying frame, ensuring that changes are immediately visible to the running Python code, rather than needing to be written back to the runtime storage at some later time. Such changes are also immediately written to the internal frame value cache to -reduce the opportunities for the cache to get out of sync with the frame state -and to make them visible to users of the ``PyEval_GetLocals()`` C API. +make them visible to users of the ``PyEval_GetLocals()`` C API. Keys that are not defined as local or closure variables on the underlying frame are still written to the internal value cache on optimised frames. This allows @@ -462,40 +488,11 @@ utilities like ``pdb`` (which writes ``__return__`` and ``__exception__`` values into the frame's ``f_locals`` mapping) to continue working as they always have. These additional keys that do not correspond to a local or closure variable on the frame will be left alone by future cache sync operations. - -Fast locals proxy objects offer a proxy-specific method that explicitly syncs -the internal frame cache with the current state of the fast locals array: -``proxy.sync_frame_cache()``. This method runs ``PyFrame_FastToLocalsWithError()`` -to ensure the cache is consistent with the current frame state. - -Using a particular proxy instance to sync the frame cache sets the internal -``frame_cache_updated`` flag on that instance. - -For most use cases, explicitly syncing the frame cache shouldn't be necessary, -as the following intrinsically O(n) operations implicitly sync the frame cache -whenever they're called on a proxy instance: - -* ``__str__`` -* ``__or__`` (dict union) -* ``copy()`` - -While the following operations will implicitly sync the frame cache if -``frame_cache_updated`` has not yet been set on that instance: - - - * ``__len__`` - * ``__iter__`` - * ``__reversed__`` - * ``keys()`` - * ``values()`` - * ``items()`` - * ``popitem()`` - * value comparison operations - - -Other ``Mapping`` and ``MutableMapping`` methods on the proxy will behave as -expected for a mapping with these essential method semantics regardless of -whether the internal frame value cache is up to date or not. +Using the frame value cache to store these extra keys (rather than defining a +new mapping that holds only the extra keys) provides full interoperability +with the existing ``PyEval_GetLocals()`` API (since users of either API will +see extra keys added by users of either API, rather than users of the new fast +locals proxy API only seeing keys added via that API). An additional benefit of storing only the variable value cache on the frame (rather than storing an instance of the proxy type), is that it avoids @@ -514,7 +511,7 @@ variables, it doesn't clear the cells themselves. This PEP could be a potential opportunity to narrow the scope of attempts to clear the frame variables directly by leaving cells belonging to outer frames alone, and only clearing local variables and cells belonging directly to the frame underlying the proxy -(this issue affects PEP 667 as well, as the question relates to the handling of +(this issue affects :pep:`667` as well, as the question relates to the handling of cell variables, and is entirely independent of the internal frame value cache). @@ -531,7 +528,9 @@ independent, behaviour. However, it is also desirable to allow C code to exactly mimic the behaviour of Python code at the same scope. To enable mimicking the behaviour of Python code, the stable C ABI would gain -the following new functions:: +the following new functions: + +.. code-block:: c PyObject * PyLocals_Get(); PyLocals_Kind PyLocals_GetKind(); @@ -558,25 +557,29 @@ ensure that it is safe to cast arbitrary signed 32-bit signed integers to This query API allows extension module code to determine the potential impact of mutating the mapping returned by ``PyLocals_Get()`` without needing access -to the details of the running frame object. +to the details of the running frame object. Python code gets equivalent +information visually through lexical scoping (as covered in the new ``locals()`` +builtin documentation). To allow extension module code to behave consistently regardless of the active -Python scope, the stable C ABI would gain the following new functions:: +Python scope, the stable C ABI would gain the following new function: + +.. code-block:: c PyObject * PyLocals_GetCopy(); - PyObject * PyLocals_GetView(); ``PyLocals_GetCopy()`` returns a new dict instance populated from the current locals namespace. Roughly equivalent to ``dict(locals())`` in Python code, but avoids the double-copy in the case where ``locals()`` already returns a shallow -copy. +copy. Akin to the following code, but doesn't assume there will only ever be +two kinds of locals result: -``PyLocals_GetView()`` returns a new read-only mapping proxy instance for the -current locals namespace. This view immediately reflects all local variable -changes, independently of whether the running frame is optimised or not. -However, some operations (e.g. length checking, iteration, mapping equality -comparisons) may be subject to frame cache consistency issues on optimised -frames (as noted above when describing the behaviour of the fast locals proxy). +.. code-block:: c + + locals = PyLocals_Get(); + if (PyLocals_GetKind() == PyLocals_DIRECT_REFERENCE) { + locals = PyDict_Copy(locals); + } The existing ``PyEval_GetLocals()`` API will retain its existing behaviour in CPython (mutable locals at class and module scope, shared dynamic snapshot @@ -587,8 +590,9 @@ The ``PyEval_GetLocals()`` documentation will also be updated to recommend replacing usage of this API with whichever of the new APIs is most appropriate for the use case: -* Use ``PyLocals_GetView()`` for read-only access to the current locals - namespace. +* Use ``PyLocals_Get()`` (optionally combined with ``PyDictProxy_New()``) for + read-only access to the current locals namespace. This form of usage will + need to be aware that the copy may go stale in optimised frames. * Use ``PyLocals_GetCopy()`` for a regular mutable dict that contains a copy of the current locals namespace, but has no ongoing connection to the active frame. @@ -619,16 +623,13 @@ will be updated only in the following circumstance: * any call to ``PyFrame_GetLocals()``, ``PyFrame_GetLocalsCopy()``, ``_PyFrame_BorrowLocals()``, ``PyFrame_FastToLocals()``, or ``PyFrame_FastToLocalsWithError()`` for the frame -* retrieving the ``f_locals`` attribute from a Python level frame object -* any call to the ``sync_frame_cache()`` method on a fast locals proxy - referencing that frame -* any operation on a fast locals proxy object that requires the shared - mapping to be up to date on the underlying frame. In the initial reference +* any operation on a fast locals proxy object that updates the shared + mapping as part of its implementation. In the initial reference implementation, those operations are those that are intrinsically ``O(n)`` - operations (``flp.copy()`` and rendering as a string), as well as those that - refresh the cache entries for individual keys. + operations (``len(flp)``, mapping comparison, ``flp.copy()`` and rendering as + a string), as well as those that refresh the cache entries for individual keys. -Accessing the frame "view" APIs will *not* implicitly update the shared dynamic +Requesting a fast locals proxy will *not* implicitly update the shared dynamic snapshot, and the CPython trace hook handling will no longer implicitly update it either. @@ -637,12 +638,13 @@ specifics of when the namespace it returns gets refreshed are still an interpreter implementation detail) The additions to the public CPython C API are the frame level enhancements -needed to support the stable C API/ABI updates:: +needed to support the stable C API/ABI updates: + +.. code-block:: c PyLocals_Kind PyFrame_GetLocalsKind(frame); PyObject * PyFrame_GetLocals(frame); PyObject * PyFrame_GetLocalsCopy(frame); - PyObject * PyFrame_GetLocalsView(frame); PyObject * _PyFrame_BorrowLocals(frame); @@ -654,8 +656,6 @@ needed to support the stable C API/ABI updates:: ``PyFrame_GetLocalsCopy(frame)`` is the underlying API for ``PyLocals_GetCopy()``. -``PyFrame_GetLocalsView(frame)`` is the underlying API for ``PyLocals_GetView()``. - ``_PyFrame_BorrowLocals(frame)`` is the underlying API for ``PyEval_GetLocals()``. The underscore prefix is intended to discourage use and to indicate that code using it is unlikely to be portable across @@ -670,7 +670,9 @@ affected code should be updated to use instead. In addition to the above documented interfaces, the draft reference -implementation also exposes the following undocumented interfaces:: +implementation also exposes the following undocumented interfaces: + +.. code-block:: c PyTypeObject _PyFastLocalsProxy_Type; #define _PyFastLocalsProxy_CheckExact(self) Py_IS_TYPE(op, &_PyFastLocalsProxy_Type) @@ -693,7 +695,7 @@ this PEP incorporates Victor Stinner's proposal to no longer implicitly call ``PyFrame_FastToLocalsWithError()`` before calling trace hooks implemented in Python. -Code using the new frame view APIs will have the dynamic locals snapshot +Code using the new fast locals proxy objects will have the dynamic locals snapshot implicitly refreshed when accessing methods that need it, while code using the ``PyEval_GetLocals()`` API will implicitly refresh it when making that call. @@ -785,7 +787,7 @@ function, but finish running after the rebinding operation has taken place (in which case the rebinding will be reverted, which is the bug reported in [1]_). In addition to preserving the de facto semantics which have been in place since -PEP 227 introduced nested scopes in Python 2.1, the other benefit of restricting +:pep:`227` introduced nested scopes in Python 2.1, the other benefit of restricting the write-through proxy support to the implementation-defined frame object API is that it means that only interpreter implementations which emulate the full frame API need to offer the write-through capability at all, and that @@ -818,14 +820,6 @@ With the frame value cache being kept around anyway, it then further made sense to rely on it to simplify the fast locals proxy mapping implementation. -Delaying implicit frame value cache updates -------------------------------------------- - -Earlier iterations of this PEP proposed updating the internal frame value cache -whenever a new fast locals proxy instance was created for that frame. They also -proposed storing a separate copy of the ``fast_refs`` lookup mapping on each - - What happens with the default args for ``eval()`` and ``exec()``? ----------------------------------------------------------------- @@ -903,11 +897,9 @@ arbitrary frames, so the standard library test suite fails if that functionality no longer works. Accordingly, the ability to store arbitrary keys was retained, at the expense -of certain operations on proxy objects currently either being slower than desired -(as they need to update the dynamic snapshot in order to provide correct -behaviour), or else assuming that the cache is currently up to date (and hence -potentially giving an incorrect answer if the frame state has changed in a -way that doesn't automatically update the cache contents). +of certain operations on proxy objects being slower than could otherwise be +(since they can't assume that only names defined on the code object will be +accessible through the proxy). It is expected that the exact details of the interaction between the fast locals proxy and the ``f_locals`` value cache on the underlying frame will evolve over @@ -978,8 +970,9 @@ into the following cases: current Python ``locals()`` namespace, but *not* wanting any changes to be visible to Python code. This is the ``PyLocals_GetCopy()`` API. * always wanting a read-only view of the current locals namespace, without - incurring the runtime overhead of making a full copy each time. This is the - ``PyLocals_GetView()`` API. + incurring the runtime overhead of making a full copy each time. This isn't + readily offered for optimised frames due to the need to check whether names + are currently bound or not, so no specific API is being added to cover it. Historically, these kinds of checks and operations would only have been possible if a Python implementation emulated the full CPython frame API. With @@ -991,22 +984,22 @@ flexibility in how they provide those capabilities. Comparison with PEP 667 ----------------------- -PEP 667 offers a partially competing proposal for this PEP that suggests it +:pep:`667` offers a partially competing proposal for this PEP that suggests it would be reasonable to eliminate the internal frame value cache on optimised frames entirely. -These changes were originally offered as amendments to PEP 558, and the PEP +These changes were originally offered as amendments to :pep:`558`, and the PEP author rejected them for three main reasons: -* the claim that ``PyEval_GetLocals()`` is unfixable because it returns a - borrowed reference is simply false, as it is still working in the PEP 558 +* the initial claim that ``PyEval_GetLocals()`` was unfixable because it returns + a borrowed reference was simply false, as it is still working in the :pep:`558` reference implementation. All that is required to keep it working is to retain the internal frame value cache and design the fast locals proxy in such a way that it is reasonably straightforward to keep the cache up to date with changes in the frame state without incurring significant runtime overhead when the cache isn't needed. Given that this claim is false, the proposal to require that all code using the ``PyEval_GetLocals()`` API be rewritten to use - a new API with different refcounting semantics fails PEP 387's requirement + a new API with different refcounting semantics fails :pep:`387`'s requirement that API compatibility breaks should have a large benefit to breakage ratio (since there's no significant benefit gained from dropping the cache, no code breakage can be justified). The only genuinely unfixable public API is @@ -1016,16 +1009,16 @@ author rejected them for three main reasons: example, becomes consistently O(n) in the number of variables defined on the frame, as the proxy has to iterate over the entire fast locals array to see which names are currently bound to values before it can determine the answer. - By contrast, maintaining an internal frame value cache allows proxies to - largely be treated as normal dictionaries from an algorithmic complexity point - of view, with allowances only needing to be made for the initial implicit O(n) - cache refresh that runs the first time an operation that relies on the cache - being up to date is executed. + By contrast, maintaining an internal frame value cache potentially allows + proxies to largely be treated as normal dictionaries from an algorithmic + complexity point of view, with allowances only needing to be made for the + initial implicit O(n) cache refresh that runs the first time an operation + that relies on the cache being up to date is executed. * the claim that a cache-free implementation would be simpler is highly suspect, - as PEP 667 includes only a pure Python sketch of a subset of a mutable mapping + as :pep:`667` includes only a pure Python sketch of a subset of a mutable mapping implementation, rather than a full-fledged C implementation of a new mapping type integrated with the underlying data storage for optimised frames. - PEP 558's fast locals proxy implementation delegates heavily to the + :pep:`558`'s fast locals proxy implementation delegates heavily to the frame value cache for the operations needed to fully implement the mutable mapping API, allowing it to re-use the existing dict implementations of the following operations: @@ -1045,119 +1038,269 @@ author rejected them for three main reasons: Of the three reasons, the first is the most important (since we need compelling reasons to break API backwards compatibility, and we don't have them). -The other two points relate to why the author of this PEP doesn't believe PEP -667's proposal would actually offer any significant benefits to either API -consumers (while the author of this PEP concedes that PEP 558's internal frame -cache sync management is more complex to deal with than PEP 667's API -algorithmic complexity quirks, it's still markedly less complex than the -tracing mode semantics in current Python versions) or to CPython core developers -(the author of this PEP certainly didn't want to write C implementations of five -new fast locals proxy specific mutable mapping helper types when he could -instead just write a single cache refresh helper method and then reuse the -existing builtin dict method implementations). +However, after reviewing :pep:`667`'s proposed Python level semantics, the author +of this PEP eventually agreed that they *would* be simpler for users of the +Python ``locals()`` API, so this distinction between the two PEPs has been +eliminated: regardless of which PEP and implementation is accepted, the fast +locals proxy object *always* provides a consistent view of the current state +of the local variables, even if this results in some operations becoming O(n) +that would be O(1) on a regular dictionary (specifically, ``len(proxy)`` +becomes O(n), since it needs to check which names are currently bound, and proxy +mapping comparisons avoid relying on the length check optimisation that allows +differences in the number of stored keys to be detected quickly for regular +mappings). -Taking the specific frame access example cited in PEP 667:: +Due to the adoption of these non-standard performance characteristics in the +proxy implementation, the ``PyLocals_GetView()`` and ``PyFrame_GetLocalsView()`` +C APIs were also removed from the proposal in this PEP. - def foo(): - x = sys._getframe().f_locals - y = locals() - print(tuple(x)) - print(tuple(y)) +This leaves the only remaining points of distinction between the two PEPs as +specifically related to the C API: -Following the implementation improvements prompted by the suggestions in PEP 667, -PEP 558 prints the same result as PEP 667 does:: +* :pep:`667` still proposes completely unnecessary C API breakage (the programmatic + deprecation and eventual removal of ``PyEval_GetLocals()``, + ``PyFrame_FastToLocalsWithError()``, and ``PyFrame_FastToLocals()``) without + justification, when it is entirely possible to keep these working indefinitely + (and interoperably) given a suitably designed fast locals proxy implementation +* the fast locals proxy handling of additional variables is defined in this PEP + in a way that is fully interoperable with the existing ``PyEval_GetLocals()`` + API. In the proxy implementation proposed in :pep:`667`, users of the new frame + API will not see changes made to additional variables by users of the old API, + and changes made to additional variables via the old API will be overwritten + on subsequent calls to ``PyEval_GetLocals()``. +* the ``PyLocals_Get()`` API in this PEP is called ``PyEval_Locals()`` in :pep:`667`. + This function name is a bit strange as it lacks a verb, making it look more + like a type name than a data access API. +* this PEP adds ``PyLocals_GetCopy()`` and ``PyFrame_GetLocalsCopy()`` APIs to + allow extension modules to easily avoid incurring a double copy operation in + frames where ``PyLocals_Get()`` already makes a copy +* this PEP adds ``PyLocals_Kind``, ``PyLocals_GetKind()``, and + ``PyFrame_GetLocalsKind()`` to allow extension modules to identify when code + is running at function scope without having to inspect non-portable frame and + code object APIs (without the proposed query API, the existing equivalent to + the new ``PyLocals_GetKind() == PyLocals_SHALLOW_COPY`` check is to include + the CPython internal frame API headers and check if + ``_PyFrame_GetCode(PyEval_GetFrame())->co_flags & CO_OPTIMIZED`` is set) - ('x', 'y') - ('x',) +The Python pseudo-code below is based on the implementation sketch presented +in :pep:`667` as of the time of writing (2021-10-24). The differences that +provide the improved interoperability between the new fast locals proxy API +and the existing ``PyEval_GetLocals()`` API are noted in comments. -That said, it's certainly possible to desynchronise the cache quite easily when -keeping proxy references around while letting code run in the frame. -This isn't a new problem, as it's similar to the way that -``sys._getframe().f_locals`` behaves in existing versions when no trace hooks -are installed. The following example:: +As in :pep:`667`, all attributes that start with an underscore are invisible and +cannot be accessed directly. They serve only to illustrate the proposed design. - def foo(): - x = sys._getframe().f_locals - print(tuple(x)) - y = locals() - print(tuple(x)) - print(tuple(y)) +For simplicity (and as in :pep:`667`), the handling of module and class level +frames is omitted (they're much simpler, as ``_locals`` *is* the execution +namespace, so no translation is required). -will print the following under PEP 558, as the first ``tuple(x)`` call consumes -the single implicit cache update performed by the proxy instance, and ``y`` -hasn't been bound yet when the ``locals()`` call refreshes it again:: +:: - ('x',) - ('x',) - ('x',) + NULL: Object # NULL is a singleton representing the absence of a value. -However, this is the origin of the coding style guideline in the body of the -PEP: don't keep fast locals proxy references around if code might have been -executed in that frame since the proxy instance was created. With the code -updated to follow that guideline:: + class CodeType: - def foo(): - x = sys._getframe().f_locals - print(tuple(x)) - y = locals() - x = sys._getframe().f_locals - print(tuple(x)) - print(tuple(y)) + _name_to_offset_mapping_impl: dict | NULL + ... + def __init__(self, ...): + self._name_to_offset_mapping_impl = NULL + self._variable_names = deduplicate( + self.co_varnames + self.co_cellvars + self.co_freevars + ) + ... -The output once again becomes the same as it would be under PEP 667:: + def _is_cell(self, offset): + ... # How the interpreter identifies cells is an implementation detail - ('x',) - ('x', 'y',) - ('x',) + @property + def _name_to_offset_mapping(self): + "Mapping of names to offsets in local variable array." + if self._name_to_offset_mapping_impl is NULL: -Tracing function implementations, which are expected to be the main consumer of -the fast locals proxy API, generally won't run into the above problem, since -they get passed a reference to the frame object (and retrieve a fresh fast -locals proxy instance from that), while the frame itself isn't running code -while the trace function is running. If the trace function *does* allow code to -be run on the frame (e.g. it's a debugger), then it should also follow the -coding guideline and retrieve a new proxy instance each time it allows code -to run in the frame. + self._name_to_offset_mapping_impl = { + name: index for (index, name) in enumerate(self._variable_names) + } + return self._name_to_offset_mapping_impl -Most trace functions are going to be reading or writing individual keys, or -running intrinsically O(n) operations like iterating over all currently bound -variables, so they also shouldn't be impacted *too* badly by the performance -quirks in the PEP 667 proposal. The most likely source of annoyance would be -the O(n) ``len(proxy)`` implementation. + class FrameType: -Note: the simplest way to convert the PEP 558 reference implementation into a -PEP 667 implementation that doesn't break ``PyEval_GetLocals()`` would be to -remove the ``frame_cache_updated`` checks in affected operations, and instead -always sync the frame cache in those methods. Adopting that approach would -change the algorithmic complexity of the following operations as shown -(where ``n`` is the number of local and cell variables defined on the frame): + _fast_locals : array[Object] # The values of the local variables, items may be NULL. + _locals: dict | NULL # Dictionary returned by PyEval_GetLocals() + + def __init__(self, ...): + self._locals = NULL + ... + + @property + def f_locals(self): + return FastLocalsProxy(self) + + class FastLocalsProxy: + + __slots__ "_frame" + + def __init__(self, frame:FrameType): + self._frame = frame + + def _set_locals_entry(self, name, val): + f = self._frame + if f._locals is NULL: + f._locals = {} + f._locals[name] = val + + def __getitem__(self, name): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + val = f._fast_locals[index] + if val is NULL: + raise KeyError(name) + if co._is_cell(offset) + val = val.cell_contents + if val is NULL: + raise KeyError(name) + # PyEval_GetLocals() interop: implicit frame cache refresh + self._set_locals_entry(name, val) + return val + # PyEval_GetLocals() interop: frame cache may contain additional names + if f._locals is NULL: + raise KeyError(name) + return f._locals[name] + + def __setitem__(self, name, value): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + kind = co._local_kinds[index] + if co._is_cell(offset) + cell = f._locals[index] + cell.cell_contents = val + else: + f._fast_locals[index] = val + # PyEval_GetLocals() interop: implicit frame cache update + # even for names that are part of the fast locals array + self._set_locals_entry(name, val) + + def __delitem__(self, name): + f = self._frame + co = f.f_code + if name in co._name_to_offset_mapping: + index = co._name_to_offset_mapping[name] + kind = co._local_kinds[index] + if co._is_cell(offset) + cell = f._locals[index] + cell.cell_contents = NULL + else: + f._fast_locals[index] = NULL + # PyEval_GetLocals() interop: implicit frame cache update + # even for names that are part of the fast locals array + if f._locals is not NULL: + del f._locals[name] + + def __iter__(self): + f = self._frame + co = f.f_code + for index, name in enumerate(co._variable_names): + val = f._fast_locals[index] + if val is NULL: + continue + if co._is_cell(offset): + val = val.cell_contents + if val is NULL: + continue + yield name + for name in f._locals: + # Yield any extra names not defined on the frame + if name in co._name_to_offset_mapping: + continue + yield name + + def popitem(self): + f = self._frame + co = f.f_code + for name in self: + val = self[name] + # PyEval_GetLocals() interop: implicit frame cache update + # even for names that are part of the fast locals array + del name + return name, val + + def _sync_frame_cache(self): + # This method underpins PyEval_GetLocals, PyFrame_FastToLocals + # PyFrame_GetLocals, PyLocals_Get, mapping comparison, etc + f = self._frame + co = f.f_code + res = 0 + if f._locals is NULL: + f._locals = {} + for index, name in enumerate(co._variable_names): + val = f._fast_locals[index] + if val is NULL: + f._locals.pop(name, None) + continue + if co._is_cell(offset): + if val.cell_contents is NULL: + f._locals.pop(name, None) + continue + f._locals[name] = val + + def __len__(self): + self._sync_frame_cache() + return len(self._locals) + +Note: the simplest way to convert the earlier iterations of the :pep:`558` +reference implementation into a preliminary implementation of the now proposed +semantics is to remove the ``frame_cache_updated`` checks in affected operations, +and instead always sync the frame cache in those methods. Adopting that approach +changes the algorithmic complexity of the following operations as shown (where +``n`` is the number of local and cell variables defined on the frame): * ``__len__``: O(1) -> O(n) + * value comparison operations: no longer benefit from O(1) length check shortcut * ``__iter__``: O(1) -> O(n) * ``__reversed__``: O(1) -> O(n) * ``keys()``: O(1) -> O(n) * ``values()``: O(1) -> O(n) * ``items()``: O(1) -> O(n) * ``popitem()``: O(1) -> O(n) - * value comparison operations: no longer benefit from O(1) length check shortcut -Keeping the iterator/iterable retrieval methods as ``O(1)`` would involve -writing custom replacements for the corresponding builtin dict helper types. -``popitem()`` could be improved from "always O(n)" to "O(n) worst case" by -creating a custom implementation that iterates over the fast locals array -directly. The length check and value comparison operations have very limited -opportunities for improvement: without a cache, the only way to know how many -variables are currently bound is to iterate over all of them and check, and if -the implementation is going to be spending that much time on an operation -anyway, it may as well spend it updating the frame value cache and then -consuming the result. +The length check and value comparison operations have relatively limited +opportunities for improvement: without allowing usage of a potentially stale +cache, the only way to know how many variables are currently bound is to iterate +over all of them and check, and if the implementation is going to be spending +that many cycles on an operation anyway, it may as well spend it updating the +frame value cache and then consuming the result. These operations are O(n) in +both this PEP and in :pep:`667`. Customised implementations could be provided that +*are* faster than updating the frame cache, but it's far from clear that the +extra code complexity needed to speed these operations up would be worthwhile +when it only offers a linear performance improvement rather than an algorithmic +complexity improvement. -This feels worse than PEP 558 as written, where folks that don't want to think -too hard about the cache management details, and don't care about potential -performance issues with large frames, are free to add as many -``proxy.sync_frame_cache()`` (or other internal frame cache updating) calls to -their code as they like. +The O(1) nature of the other operations can be restored by adding implementation +code that doesn't rely on the value cache being up to date. + +Keeping the iterator/iterable retrieval methods as O(1) will involve +writing custom replacements for the corresponding builtin dict helper types, +just as proposed in :pep:`667`. As illustrated above, the implementations would +be similar to the pseudo-code presented in :pep:`667`, but not identical (due to +the improved ``PyEval_GetLocals()`` interoperability offered by this PEP +affecting the way it stores extra variables). + +``popitem()`` can be improved from "always O(n)" to "O(n) worst case" by +creating a custom implementation that relies on the improved iteration APIs. + +To ensure stale frame information is never presented in the Python fast locals +proxy API, these changes in the reference implementation will need to be +implemented before merging. + +The current implementation at time of writing (2021-10-24) also still stores a +copy of the fast refs mapping on each frame rather than storing a single +instance on the underlying code object (as it still stores cell references +directly, rather than check for cells on each fast locals array access). Fixing +this would also be required before merging. Implementation @@ -1175,7 +1318,7 @@ Thanks to Nathaniel J. Smith for proposing the write-through proxy idea in PEP that attempted to avoid introducing such a proxy. Thanks to Steve Dower and Petr Viktorin for asking that more attention be paid -to the developer experience of the proposed C API additions [8,13]_. +to the developer experience of the proposed C API additions [8]_ [13]_. Thanks to Larry Hastings for the suggestion on how to use enums in the stable ABI while ensuring that they safely support typecasting from arbitrary @@ -1184,67 +1327,54 @@ integers. Thanks to Mark Shannon for pushing for further simplification of the C level API and semantics, as well as significant clarification of the PEP text (and for restarting discussion on the PEP in early 2021 after a further year of -inactivity) [10,11,12]_. Mark's comments that were ultimately published as -PEP 667 also directly resulted in several implementation efficiency improvements +inactivity) [10]_ [11]_ [12]_. Mark's comments that were ultimately published as +:pep:`667` also directly resulted in several implementation efficiency improvements that avoid incurring the cost of redundant O(n) mapping refresh operations -when the relevant mappings aren't used. +when the relevant mappings aren't used, as well as the change to ensure that +the state reported through the Python level ``f_locals`` API is never stale. References ========== -.. [1] Broken local variable assignment given threads + trace hook + closure - (https://bugs.python.org/issue30744) +.. [1] `Broken local variable assignment given threads + trace hook + closure + `_ -.. [2] Clarify the required behaviour of ``locals()`` - (https://bugs.python.org/issue17960) +.. [3] `Updating function local variables from pdb is unreliable + `_ -.. [3] Updating function local variables from pdb is unreliable - (https://bugs.python.org/issue9633) +.. [4] `CPython's Python API for installing trace hooks + `_ -.. [4] CPython's Python API for installing trace hooks - (https://docs.python.org/dev/library/sys.html#sys.settrace) +.. [5] `CPython's C API for installing trace hooks + `_ -.. [5] CPython's C API for installing trace hooks - (https://docs.python.org/3/c-api/init.html#c.PyEval_SetTrace) +.. [6] `PEP 558 reference implementation + `_ -.. [6] PEP 558 reference implementation - (https://github.com/python/cpython/pull/3640/files) +.. [7] `Nathaniel's review of possible function level semantics for locals() + `_ -.. [7] Nathaniel's review of possible function level semantics for locals() - (https://mail.python.org/pipermail/python-dev/2019-May/157738.html) +.. [8] `Discussion of more intentionally designed C API enhancements + `_ -.. [8] Discussion of more intentionally designed C API enhancements - (https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936/3) +.. [9] `Disable automatic update of frame locals during tracing + `_ -.. [9] Disable automatic update of frame locals during tracing - (https://bugs.python.org/issue42197) +.. [10] `python-dev thread: Resurrecting PEP 558 (Defined semantics for locals()) + `_ -.. [10] python-dev thread: Resurrecting PEP 558 (Defined semantics for locals()) - (https://mail.python.org/archives/list/python-dev@python.org/thread/TUQOEWQSCQZPUDV2UFFKQ3C3I4WGFPAJ/) +.. [11] `python-dev thread: Comments on PEP 558 + `_ -.. [11] python-dev thread: Comments on PEP 558 - (https://mail.python.org/archives/list/python-dev@python.org/thread/A3UN4DGBCOB45STE6AQBITJFW6UZE43O/) +.. [12] `python-dev thread: More comments on PEP 558 + `_ -.. [12] python-dev thread: More comments on PEP 558 - (https://mail.python.org/archives/list/python-dev@python.org/thread/7TKPMD5LHCBXGFUIMKDAUZELRH6EX76S/) - -.. [13] Petr Viktorin's suggestion to use an enum for ``PyLocals_Get``'s behaviour - (https://mail.python.org/archives/list/python-dev@python.org/message/BTQUBHIVE766RPIWLORC5ZYRCRC4CEBL/) +.. [13] `Petr Viktorin's suggestion to use an enum for PyLocals_Get's behaviour + `_ Copyright ========= This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0559.rst b/peps/pep-0559.rst similarity index 97% rename from pep-0559.rst rename to peps/pep-0559.rst index 4c99f2fe3..63574e256 100644 --- a/pep-0559.rst +++ b/peps/pep-0559.rst @@ -24,7 +24,7 @@ It is trivial to implement a no-op function in Python. It's so easy in fact that many people do it many times over and over again. It would be useful in many cases to have a common built-in function that does nothing. -One use case would be for PEP 553, where you could set the breakpoint +One use case would be for :pep:`553`, where you could set the breakpoint environment variable to the following in order to effectively disable it:: $ setenv PYTHONBREAKPOINT=noop diff --git a/pep-0560.rst b/peps/pep-0560.rst similarity index 97% rename from pep-0560.rst rename to peps/pep-0560.rst index d3bcd4811..154145d0c 100644 --- a/pep-0560.rst +++ b/peps/pep-0560.rst @@ -13,10 +13,10 @@ Resolution: https://mail.python.org/pipermail/python-dev/2017-December/151038.ht Abstract ======== -Initially PEP 484 was designed in such way that it would not introduce +Initially :pep:`484` was designed in such way that it would not introduce *any* changes to the core CPython interpreter. Now type hints and -the ``typing`` module are extensively used by the community, e.g. PEP 526 -and PEP 557 extend the usage of type hints, and the backport of ``typing`` +the ``typing`` module are extensively used by the community, e.g. :pep:`526` +and :pep:`557` extend the usage of type hints, and the backport of ``typing`` on PyPI has 1M downloads/month. Therefore, this restriction can be removed. It is proposed to add two special methods ``__class_getitem__`` and ``__mro_entries__`` to the core CPython for better support of @@ -38,7 +38,7 @@ Performance The ``typing`` module is one of the heaviest and slowest modules in the standard library even with all the optimizations made. Mainly this is -because subscripted generic types (see PEP 484 for definition of terms used +because subscripted generic types (see :pep:`484` for definition of terms used in this PEP) are class objects (see also [1]_). There are three main ways how the performance can be improved with the help of the proposed special methods: @@ -244,7 +244,7 @@ and deferring all check to static type checkers only:: Such class can be used as a normal generic in Python type annotations (a corresponding stub file should be provided for static type checkers, -see PEP 484 for details):: +see :pep:`484` for details):: from simple_extension import SimpleGeneric from typing import TypeVar @@ -280,8 +280,8 @@ With the reference implementation I measured negligible performance effects time performance of generics is significantly improved: * ``importlib.reload(typing)`` is up to 7x faster -* Creation of user defined generic classes is up to 4x faster (on a micro- - benchmark with an empty body) +* Creation of user defined generic classes is up to 4x faster (on a + micro-benchmark with an empty body) * Instantiation of generic classes is up to 5x faster (on a micro-benchmark with an empty ``__init__``) * Other operations with generic types and instances (like method lookup and diff --git a/pep-0561.rst b/peps/pep-0561.rst similarity index 90% rename from pep-0561.rst rename to peps/pep-0561.rst index 31aaae236..c8c424860 100644 --- a/pep-0561.rst +++ b/peps/pep-0561.rst @@ -1,8 +1,9 @@ PEP: 561 Title: Distributing and Packaging Type Information Author: Ethan Smith -Status: Accepted +Status: Final Type: Standards Track +Topic: Packaging, Typing Content-Type: text/x-rst Created: 09-Sep-2017 Python-Version: 3.7 @@ -12,7 +13,7 @@ Post-History: 10-Sep-2017, 12-Sep-2017, 06-Oct-2017, 26-Oct-2017, 12-Apr-2018 Abstract ======== -PEP 484 introduced type hinting to Python, with goals of making typing +:pep:`484` introduced type hinting to Python, with goals of making typing gradual and easy to adopt. Currently, typing information must be distributed manually. This PEP provides a standardized means to leverage existing tooling to package and distribute type information with minimal work and an ordering @@ -34,8 +35,9 @@ typeshed [1]_. However, this does not scale and becomes a burden on the maintainers of typeshed. In addition, it ties bug fixes in stubs to releases of the tool using typeshed. -PEP 484 has a brief section on distributing typing information. In this -section [2]_ the PEP recommends using ``shared/typehints/pythonX.Y/`` for +:pep:`484` has a brief section on distributing typing information. In this +:pep:`section <484#storing-and-distributing-stub-files>` +the PEP recommends using ``shared/typehints/pythonX.Y/`` for shipping stub files. However, manually adding a path to stub files for each third party library does not scale. The simplest approach people have taken is to add ``site-packages`` to their ``MYPYPATH``, but this causes type @@ -47,16 +49,16 @@ Definition of Terms =================== The definition of "MAY", "MUST", and "SHOULD", and "SHOULD NOT" are -to be interpreted as described in RFC 2119. +to be interpreted as described in :rfc:`2119`. -"inline" - the types are part of the runtime code using PEP 526 and -PEP 3107 syntax (the filename ends in ``.py``). +"inline" - the types are part of the runtime code using :pep:`526` and +:pep:`3107` syntax (the filename ends in ``.py``). "stubs" - files containing only type information, empty of runtime code (the filename ends in ``.pyi``). "Distributions" are the packaged files which are used to publish and distribute -a release. [3]_ +a release. (:pep:`426`) "Module" a file containing Python runtime code or stubbed type information. @@ -86,8 +88,8 @@ packaging and deployment. The two major parts of this specification are the packaging specifications and the resolution order for resolving module type information. The type -checking spec is meant to replace the ``shared/typehints/pythonX.Y/`` spec -of PEP 484 [2]_. +checking spec is meant to replace the ``shared/typehints/pythonX.Y/`` +:pep:`spec of PEP 484 <484#storing-and-distributing-stub-files>`. New third party stub libraries SHOULD distribute stubs via the third party packaging methods proposed in this PEP in place of being added to typeshed. @@ -119,7 +121,7 @@ Distutils option example:: ..., ) -For namespace packages (see PEP 420), the ``py.typed`` file should be in the +For namespace packages (see :pep:`420`), the ``py.typed`` file should be in the submodules of the namespace, to avoid conflicts and for clarity. This PEP does not support distributing typing information as part of @@ -158,7 +160,7 @@ in pip 9.0, if you update ``flyingcircus-stubs``, it will update ``--upgrade-strategy=only-if-needed`` flag. In pip 10.0 this is the default behavior. -For namespace packages (see PEP 420), stub-only packages should +For namespace packages (see :pep:`420`), stub-only packages should use the ``-stubs`` suffix on only the root namespace package. All stub-only namespace packages should omit ``__init__.pyi`` files. ``py.typed`` marker files are not necessary for stub-only packages, but similarly @@ -180,6 +182,7 @@ laid out as follows:: └── hexagon    └── __init__.pyi +.. _mro: Type Checker Module Resolution Order ------------------------------------ @@ -198,8 +201,10 @@ resolve modules containing type information: 3. Stub packages - these packages SHOULD supersede any installed inline package. They can be found at ``foopkg-stubs`` for package ``foopkg``. -4. Inline packages - if there is nothing overriding the installed - package, *and* it opts into type checking, inline types SHOULD be used. +4. Packages with a ``py.typed`` marker file - if there is nothing overriding + the installed package, *and* it opts into type checking, the types + bundled with the package SHOULD be used (be they in ``.pyi`` type + stub files or inline in ``.py`` files). 5. Typeshed (if used) - Provides the stdlib types and several third party libraries. @@ -235,7 +240,7 @@ checkers MUST maintain the normal resolution order of checking ``*.pyi`` before If a stub package distribution is partial it MUST include ``partial\n`` in a ``py.typed`` file. For stub-packages distributing within a namespace -package (see PEP 420), the ``py.typed`` file should be in the +package (see :pep:`420`), the ``py.typed`` file should be in the submodules of the namespace. Type checkers should treat namespace packages within stub-packages as @@ -254,7 +259,7 @@ sample package checker [pkg_checker]_ which reads the metadata of installed packages and reports on their status as either not typed, inline typed, or a stub package. -The mypy type checker has an implementation of PEP 561 searching which can be +The mypy type checker has an implementation of :pep:`561` searching which can be read about in the mypy docs [4]_. [numpy-stubs]_ is an example of a real stub-only package for the numpy @@ -272,6 +277,12 @@ Vlasovskikh, Nathaniel Smith, and Guido van Rossum. Version History =============== +* 2023-01-13 + + * Clarify that the 4th step of the :ref:`Module Resolution Order ` applies + to any package with a ``py.typed`` marker file (and not just + inline packages). + * 2021-09-20 * Clarify expectations and typechecker behavior for stub-only namespace packages @@ -334,12 +345,6 @@ References ========== .. [1] Typeshed (https://github.com/python/typeshed) -.. [2] PEP 484, Storing and Distributing Stub Files - (https://www.python.org/dev/peps/pep-0484/#storing-and-distributing-stub-files) - -.. [3] PEP 426 definitions - (https://www.python.org/dev/peps/pep-0426/) - .. [4] Example implementation in a type checker (https://mypy.readthedocs.io/en/latest/installed_packages.html) diff --git a/pep-0562.rst b/peps/pep-0562.rst similarity index 94% rename from pep-0562.rst rename to peps/pep-0562.rst index c16712955..2c446c67f 100644 --- a/pep-0562.rst +++ b/peps/pep-0562.rst @@ -43,7 +43,7 @@ on module *instances*. For example:: if name in deprecated_names: warn(f"{name} is deprecated", DeprecationWarning) return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__} has no attribute {name}") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") # main.py @@ -74,12 +74,12 @@ imports. Consider a simple example:: import lib lib.submod.HeavyClass # prints "Submodule loaded" -There is a related proposal PEP 549 that proposes to support instance +There is a related proposal :pep:`549` that proposes to support instance properties for a similar functionality. The difference is this PEP proposes a faster and simpler mechanism, but provides more basic customization. -An additional motivation for this proposal is that PEP 484 already defines +An additional motivation for this proposal is that :pep:`484` already defines the use of module ``__getattr__`` for this purpose in Python stub files, -see [1]_. +see :pep:`484#stub-files`. In addition, to allow modifying result of a ``dir()`` call on a module to show deprecated and other dynamically generated attributes, it is @@ -184,9 +184,6 @@ called only once. References ========== -.. [1] PEP 484 section about ``__getattr__`` in stub files - (https://www.python.org/dev/peps/pep-0484/#stub-files) - .. [2] The reference implementation (https://github.com/ilevkivskyi/cpython/pull/3/files) diff --git a/pep-0563.rst b/peps/pep-0563.rst similarity index 94% rename from pep-0563.rst rename to peps/pep-0563.rst index 1af1cfd36..585734c1e 100644 --- a/pep-0563.rst +++ b/peps/pep-0563.rst @@ -3,22 +3,23 @@ Title: Postponed Evaluation of Annotations Version: $Revision$ Last-Modified: $Date$ Author: Ɓukasz Langa -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 08-Sep-2017 Python-Version: 3.7 -Post-History: 1-Nov-2017, 21-Nov-2017 +Post-History: 01-Nov-2017, 21-Nov-2017 Resolution: https://mail.python.org/pipermail/python-dev/2017-December/151042.html Abstract ======== -PEP 3107 introduced syntax for function annotations, but the semantics -were deliberately left undefined. PEP 484 introduced a standard meaning -to annotations: type hints. PEP 526 defined variable annotations, +:pep:`3107` introduced syntax for function annotations, but the semantics +were deliberately left undefined. :pep:`484` introduced a standard meaning +to annotations: type hints. :pep:`526` defined variable annotations, explicitly tying them with the type hinting use case. This PEP proposes changing function annotations and variable annotations @@ -32,7 +33,7 @@ This change is being introduced gradually, starting with a Rationale and Goals =================== -PEP 3107 added support for arbitrary annotations on parts of a function +:pep:`3107` added support for arbitrary annotations on parts of a function definition. Just like default values, annotations are evaluated at function definition time. This creates a number of issues for the type hinting use case: @@ -45,13 +46,13 @@ hinting use case: computationally free. Postponing the evaluation of annotations solves both problems. -NOTE: PEP 649 proposes an alternative solution to the above issues, -putting this PEP in danger of being superceded. +NOTE: :pep:`649` proposes an alternative solution to the above issues, +putting this PEP in danger of being superseded. Non-goals --------- -Just like in PEP 484 and PEP 526, it should be emphasized that **Python +Just like in :pep:`484` and :pep:`526`, it should be emphasized that **Python will remain a dynamically typed language, and the authors have no desire to ever make type hints mandatory, even by convention.** @@ -69,14 +70,14 @@ Non-typing usage of annotations While annotations are still available for arbitrary use besides type checking, it is worth mentioning that the design of this PEP, as well -as its precursors (PEP 484 and PEP 526), is predominantly motivated by +as its precursors (:pep:`484` and :pep:`526`), is predominantly motivated by the type hinting use case. -In Python 3.8 PEP 484 will graduate from provisional status. Other -enhancements to the Python programming language like PEP 544, PEP 557, -or PEP 560, are already being built on this basis as they depend on -type annotations and the ``typing`` module as defined by PEP 484. -In fact, the reason PEP 484 is staying provisional in Python 3.7 is to +In Python 3.8 :pep:`484` will graduate from provisional status. Other +enhancements to the Python programming language like :pep:`544`, :pep:`557`, +or :pep:`560`, are already being built on this basis as they depend on +type annotations and the ``typing`` module as defined by :pep:`484`. +In fact, the reason :pep:`484` is staying provisional in Python 3.7 is to enable rapid evolution for another release cycle that some of the aforementioned enhancements require. @@ -104,7 +105,7 @@ Annotations can only use names present in the module scope as postponed evaluation using local names is not reliable (with the sole exception of class-level names resolved by ``typing.get_type_hints()``). -Note that as per PEP 526, local variable annotations are not evaluated +Note that as per :pep:`526`, local variable annotations are not evaluated at all since they are not accessible outside of the function's closure. Enabling the future behavior in Python 3.7 @@ -286,7 +287,7 @@ Starting with Python 3.7, a ``__future__`` import is required to use the described functionality. No warnings are raised. NOTE: Whether this will eventually become the default behavior is currently unclear -pending decision on PEP 649. In any case, use of annotations that depend upon +pending decision on :pep:`649`. In any case, use of annotations that depend upon their eager evaluation is incompatible with both proposals and is no longer supported. @@ -424,7 +425,7 @@ never be able to use forward references since that would force the library users to use this new hypothetical -O switch. Second, this throws the baby out with the bath water. Now *no* runtime -annotation use can be performed. PEP 557 is one example of a recent +annotation use can be performed. :pep:`557` is one example of a recent development where evaluating type annotations at runtime is useful. All that being said, a granular -O option to drop annotations is @@ -475,7 +476,7 @@ Prior discussion In PEP 484 ---------- -The forward reference problem was discussed when PEP 484 was originally +The forward reference problem was discussed when :pep:`484` was originally drafted, leading to the following statement in the document: A compromise is possible where a ``__future__`` import could enable @@ -527,7 +528,7 @@ beginners: While typing usability is an interesting problem, it is out of scope of this PEP. Specifically, any extensions of the typing syntax -standardized in PEP 484 will require their own respective PEPs and +standardized in :pep:`484` will require their own respective PEPs and approval. Issue 400 ultimately suggests postponing evaluation of annotations and @@ -641,7 +642,7 @@ introspectable as strings. Most importantly, Guido van Rossum explicitly stated interest in gradually restricting the use of annotations to static typing (with an optional runtime component). -Nick Coghlan got convinced to PEP 563, too, promptly beginning +Nick Coghlan got convinced to :pep:`563`, too, promptly beginning the mandatory bike shedding session on the name of the ``__future__`` import. Many debaters agreed that ``annotations`` seems like an overly broad name for the feature name. Guido van Rossum briefly diff --git a/pep-0564.rst b/peps/pep-0564.rst similarity index 97% rename from pep-0564.rst rename to peps/pep-0564.rst index 2be6b8c68..447bd4e63 100644 --- a/pep-0564.rst +++ b/peps/pep-0564.rst @@ -66,7 +66,7 @@ precision:: Previous rejected PEP --------------------- -Five years ago, the PEP 410 proposed a large and complex change in all +Five years ago, the :pep:`410` proposed a large and complex change in all Python functions returning time to support nanosecond resolution using the ``decimal.Decimal`` type. @@ -124,13 +124,13 @@ with nanosecond resolution. CPython enhancements of the last 5 years ---------------------------------------- -Since the PEP 410 was rejected: +Since the :pep:`410` was rejected: * The ``os.stat_result`` structure got 3 new fields for timestamps as nanoseconds (Python ``int``): ``st_atime_ns``, ``st_ctime_ns`` and ``st_mtime_ns``. -* The PEP 418 was accepted, Python 3.3 got 3 new clocks: +* The :pep:`418` was accepted, Python 3.3 got 3 new clocks: ``time.monotonic()``, ``time.perf_counter()`` and ``time.process_time()``. @@ -244,7 +244,7 @@ Modifying time.time() result type It was proposed to modify ``time.time()`` to return a different number type with better precision. -The PEP 410 proposed to return ``decimal.Decimal`` which already exists and +The :pep:`410` proposed to return ``decimal.Decimal`` which already exists and supports arbitrary precision, but it was rejected. Apart from ``decimal.Decimal``, no portable real number type with better precision is currently available in Python. @@ -264,7 +264,7 @@ Many ideas of new types were proposed to support larger or arbitrary precision: fractions, structures or 2-tuple using integers, fixed-point number, etc. -See also the PEP 410 for a previous long discussion on other types. +See also the :pep:`410` for a previous long discussion on other types. Adding a new type requires more effort to support it, than reusing the existing ``int`` type. The standard library, third party code and diff --git a/pep-0565.rst b/peps/pep-0565.rst similarity index 91% rename from pep-0565.rst rename to peps/pep-0565.rst index ee7a0b0e4..9082e2d3d 100644 --- a/pep-0565.rst +++ b/peps/pep-0565.rst @@ -14,14 +14,14 @@ Abstract ======== In Python 2.7 and Python 3.2, the default warning filters were updated to hide -DeprecationWarning by default, such that deprecation warnings in development +``DeprecationWarning`` by default, such that deprecation warnings in development tools that were themselves written in Python (e.g. linters, static analysers, test runners, code generators), as well as any other applications that merely happened to be written in Python, wouldn't be visible to their users unless those users explicitly opted in to seeing them. However, this change has had the unfortunate side effect of making -DeprecationWarning markedly less effective at its primary intended purpose: +``DeprecationWarning`` markedly less effective at its primary intended purpose: providing advance notice of breaking changes in APIs (whether in CPython, the standard library, or in third party libraries) to users of those APIs. @@ -74,10 +74,10 @@ to be:: This means that in cases where the nominal location of the warning (as determined by the ``stacklevel`` parameter to ``warnings.warn``) is in the -``__main__`` module, the first occurrence of each DeprecationWarning will once +``__main__`` module, the first occurrence of each ``DeprecationWarning`` will once again be reported. -This change will lead to DeprecationWarning being displayed by default for: +This change will lead to ``DeprecationWarning`` being displayed by default for: * code executed directly at the interactive prompt * code executed directly as part of a single-file script @@ -164,7 +164,7 @@ no changes are needed beyond those in this PEP. Interactive shell implementations which use a namespace other than ``__main__`` will need to add their own filter. For example, IPython uses the -following command ([8_]) to set up a suitable filter:: +following command ([6]_) to set up a suitable filter:: warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__")) @@ -215,8 +215,8 @@ of the related documentation. Reference Implementation ======================== -A reference implementation is available in the PR [4_] linked from the -related tracker issue for this PEP [5_]. +A reference implementation is available in the PR [4]_ linked from the +related tracker issue for this PEP [5]_. As a side-effect of implementing this PEP, the internal warnings filter list will start allowing the use of plain strings as part of filter definitions (in @@ -229,7 +229,7 @@ early access to the ``re`` module. Motivation ========== -As discussed in [1_] and mentioned in [2_], Python 2.7 and Python 3.2 changed +As discussed in [1]_ and mentioned in [2]_, Python 2.7 and Python 3.2 changed the default handling of ``DeprecationWarning`` such that: * the warning was hidden by default during normal code execution @@ -274,7 +274,7 @@ Limitations on PEP Scope This PEP exists specifically to explain both the proposed addition to the default warnings filter for 3.7, *and* to more clearly articulate the rationale -for the original change to the handling of DeprecationWarning back in Python 2.7 +for the original change to the handling of ``DeprecationWarning`` back in Python 2.7 and 3.2. This PEP does not solve all known problems with the current approach to handling @@ -289,7 +289,7 @@ deprecation warnings. Most notably: variable. * The standard library doesn't provide a straightforward way to opt-in to seeing all warnings emitted *by* a particular dependency prior to upgrading it - (the third-party ``warn`` module [3_] does provide this, but enabling it + (the third-party ``warn`` module [3]_ does provide this, but enabling it involves monkeypatching the standard library's ``warnings`` module). * When software has been factored out into support modules, but those modules have little or no automated test coverage, re-enabling deprecation warnings @@ -321,12 +321,12 @@ changes in 3.7: * a new ``-X dev`` command line option that combines several developer centric settings (including ``-Wd``) into one command line flag: - https://bugs.python.org/issue32043 + https://github.com/python/cpython/issues/76224 * changing the behaviour in debug builds to show more of the warnings that are - off by default in regular interpreter builds: https://bugs.python.org/issue32088 + off by default in regular interpreter builds: https://github.com/python/cpython/issues/76269 Independently of the proposed changes to the default filters in this PEP, -issue 32229 [9_] is a proposal to add a ``warnings.hide_warnings`` API to +issue 32229 [7]_ is a proposal to add a ``warnings.hide_warnings`` API to make it simpler for application developers to hide warnings during normal operation, while easily making them visible when testing. @@ -347,32 +347,21 @@ References (https://github.com/python/cpython/pull/4458) .. [5] Tracker issue for PEP 565 implementation - (https://bugs.python.org/issue31975) + (https://github.com/python/cpython/issues/76156) -.. [6] First python-dev discussion thread - (https://mail.python.org/pipermail/python-dev/2017-November/150477.html) - -.. [7] Second python-dev discussion thread - (https://mail.python.org/pipermail/python-dev/2017-November/150819.html) - -.. [8] IPython's DeprecationWarning auto-configuration +.. [6] IPython's ``DeprecationWarning`` auto-configuration (https://github.com/ipython/ipython/blob/6.2.x/IPython/core/interactiveshell.py#L619) -.. [9] ``warnings.hide_warnings`` API proposal - (https://bugs.python.org/issue32229) +.. [7] ``warnings.hide_warnings`` API proposal + (https://github.com/python/cpython/issues/76410) + +* `First python-dev discussion thread + `__ + +* `Second python-dev discussion thread + `__ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0566.rst b/peps/pep-0566.rst similarity index 90% rename from pep-0566.rst rename to peps/pep-0566.rst index fee303543..7c0391175 100644 --- a/pep-0566.rst +++ b/peps/pep-0566.rst @@ -1,12 +1,11 @@ PEP: 566 Title: Metadata for Python Software Packages 2.1 -Version: $Revision$ -Last-Modified: $Date$ Author: Dustin Ingram BDFL-Delegate: Daniel Holth -Discussions-To: distutils-sig +Discussions-To: distutils-sig@python.org Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 01-Dec-2017 Python-Version: 3.x @@ -14,12 +13,16 @@ Post-History: Replaces: 345 Resolution: https://mail.python.org/pipermail/distutils-sig/2018-February/032014.html + +.. canonical-pypa-spec:: :ref:`packaging:core-metadata` + + Abstract ======== This PEP describes the changes between versions 1.2 and 2.1 of the core -metadata specification for Python packages. Version 1.2 is specified in PEP -345. +metadata specification for Python packages. Version 1.2 is specified in +:pep:`345`. It also changes to the canonical source for field specifications to the `Core Metadata Specification`_ reference document, which includes specifics of the @@ -79,7 +82,7 @@ Name :::: The specification for the format of this field is now identical to the -distribution name specification defined in PEP 508. +distribution name specification defined in :pep:`508`. Description ::::::::::: @@ -104,7 +107,7 @@ also be able to handle version specifiers in parentheses. Further, public index servers MAY prohibit strict version matching clauses or direct references in these fields. -Usage of version specifiers is otherwise unchanged from PEP 345. +Usage of version specifiers is otherwise unchanged from :pep:`345`. Environment markers =================== @@ -114,9 +117,9 @@ field after a semi-colon (";"), to add a condition about the execution environment. The environment marker format used to declare such a condition is defined in -the environment markers section of PEP 508. +the environment markers section of :pep:`508`. -Usage of environment markers is otherwise unchanged from PEP 345. +Usage of environment markers is otherwise unchanged from :pep:`345`. JSON-compatible Metadata ======================== @@ -134,7 +137,7 @@ as follows: #. The transformed value for any field marked with "(Multiple-use") should be a single list containing all the original values for the given key; #. The ``Keywords`` field should be converted to a list by splitting the - original value on whitespace characters; + original value on commas; #. The message body, if present, should be set to the value of the ``description`` key. #. The result should be stored as a string-keyed dictionary. @@ -148,7 +151,7 @@ Summary of Differences From PEP 345 * Added two new fields: ``Description-Content-Type`` and ``Provides-Extra`` -* Acceptable values for the ``Name`` field are now specified as per PEP 508. +* Acceptable values for the ``Name`` field are now specified as per :pep:`508`. * Added canonical method of transformation into JSON-compatible data structure. @@ -156,10 +159,10 @@ References ========== This document specifies version 2.1 of the metadata format. -Version 1.0 is specified in PEP 241. -Version 1.1 is specified in PEP 314. -Version 1.2 is specified in PEP 345. -Version 2.0, while not formally accepted, was specified in PEP 426. +Version 1.0 is specified in :pep:`241`. +Version 1.1 is specified in :pep:`314`. +Version 1.2 is specified in :pep:`345`. +Version 2.0, while not formally accepted, was specified in :pep:`426`. .. _`Core Metadata Specification`: https://packaging.python.org/specifications/core-metadata/ diff --git a/pep-0567.rst b/peps/pep-0567.rst similarity index 99% rename from pep-0567.rst rename to peps/pep-0567.rst index 3dec79b2e..4c6514293 100644 --- a/pep-0567.rst +++ b/peps/pep-0567.rst @@ -859,16 +859,16 @@ See also issue 32436 [4]_. Acceptance ========== -PEP 567 was accepted by Guido on Monday, January 22, 2018 [5]_. +:pep:`567` was accepted by Guido on Monday, January 22, 2018 [5]_. The reference implementation was merged on the same day. References ========== -.. [1] https://www.python.org/dev/peps/pep-0550/#appendix-hamt-performance-analysis +.. [1] :pep:`550#appendix-hamt-performance-analysis` -.. [2] https://www.python.org/dev/peps/pep-0550/#replication-of-threading-local-interface +.. [2] :pep:`550#replication-of-threading-local-interface` .. [3] https://github.com/python/cpython/pull/5027 diff --git a/pep-0568.rst b/peps/pep-0568.rst similarity index 96% rename from pep-0568.rst rename to peps/pep-0568.rst index d7dc3b951..7f288ae7d 100644 --- a/pep-0568.rst +++ b/peps/pep-0568.rst @@ -15,10 +15,10 @@ Abstract Context variables provide a generic mechanism for tracking dynamic, context-local state, similar to thread-local storage but generalized to cope work with other kinds of thread-like contexts, such as asyncio -Tasks. PEP 550 proposed a mechanism for context-local state that was +Tasks. :pep:`550` proposed a mechanism for context-local state that was also sensitive to generator context, but this was pretty complicated, -so the BDFL requested it be simplified. The result was PEP 567, which -is targeted for inclusion in 3.7. This PEP then extends PEP 567's +so the BDFL requested it be simplified. The result was :pep:`567`, which +is targeted for inclusion in 3.7. This PEP then extends :pep:`567`'s machinery to add generator context sensitivity. This PEP is starting out in the "deferred" status, because there isn't @@ -57,10 +57,10 @@ Specification Review of PEP 567 ----------------- -Let's start by reviewing how PEP 567 works, and then in the next +Let's start by reviewing how :pep:`567` works, and then in the next section we'll describe the differences. -In PEP 567, a ``Context`` is a ``Mapping`` from ``ContextVar`` objects +In :pep:`567`, a ``Context`` is a ``Mapping`` from ``ContextVar`` objects to arbitrary values. In our pseudo-code here we'll pretend that it uses a ``dict`` for backing storage. (The real implementation uses a HAMT, which is semantically equivalent to a ``dict`` but with @@ -301,7 +301,7 @@ That's all. Comparison to PEP 550 ===================== -The main difference from PEP 550 is that it reified what we're calling +The main difference from :pep:`550` is that it reified what we're calling "contexts" and "context stacks" as two different concrete types (``LocalContext`` and ``ExecutionContext`` respectively). This led to lots of confusion about what the differences were, and which object @@ -331,7 +331,7 @@ else an intrusive linked list (``PyThreadState`` → ``Context`` → ``Context`` → ...), with the "top" of the stack at the beginning of the list to allow efficient push/pop. -A critical optimization in PEP 567 is the caching of values inside +A critical optimization in :pep:`567` is the caching of values inside ``ContextVar``. Switching from a single context to a context stack makes this a little bit more complicated, but not too much. Currently, we invalidate the cache whenever the threadstate's current ``Context`` diff --git a/pep-0569.rst b/peps/pep-0569.rst similarity index 84% rename from pep-0569.rst rename to peps/pep-0569.rst index b0cf48914..caa9593e2 100644 --- a/pep-0569.rst +++ b/peps/pep-0569.rst @@ -1,10 +1,9 @@ PEP: 569 Title: Python 3.8 Release Schedule -Version: $Revision$ -Last-Modified: $Date$ Author: Ɓukasz Langa Status: Active Type: Informational +Topic: Release Content-Type: text/x-rst Created: 27-Jan-2018 Python-Version: 3.8 @@ -56,7 +55,7 @@ Release Schedule - 3.8.0 beta 1: Tuesday, 2019-06-04 (No new features beyond this point.) -- 3.8.0 beta 2: Monday, 2019-07-04 +- 3.8.0 beta 2: Thursday, 2019-07-04 - 3.8.0 beta 3: Monday, 2019-07-29 - 3.8.0 beta 4: Friday, 2019-08-30 - 3.8.0 candidate 1: Tuesday, 2019-10-01 @@ -92,6 +91,12 @@ Provided irregularly on an "as-needed" basis until October 2024. - 3.8.11: Monday, 2021-06-28 - 3.8.12: Monday, 2021-08-30 +- 3.8.13: Wednesday, 2022-03-16 +- 3.8.14: Tuesday, 2022-09-06 +- 3.8.15: Tuesday, 2022-10-11 +- 3.8.16: Tuesday, 2022-12-06 +- 3.8.17: Tuesday, 2023-06-06 +- 3.8.18: Thursday, 2023-08-24 Features for 3.8 @@ -99,14 +104,14 @@ Features for 3.8 Some of the notable features of Python 3.8 include: -* PEP 570, Positional-only arguments -* PEP 572, Assignment Expressions -* PEP 574, Pickle protocol 5 with out-of-band data -* PEP 578, Runtime audit hooks -* PEP 587, Python Initialization Configuration -* PEP 590, Vectorcall: a fast calling protocol for CPython -* Typing-related: PEP 591 (Final qualifier), PEP 586 (Literal types), - and PEP 589 (TypedDict) +* :pep:`570`, Positional-only arguments +* :pep:`572`, Assignment Expressions +* :pep:`574`, Pickle protocol 5 with out-of-band data +* :pep:`578`, Runtime audit hooks +* :pep:`587`, Python Initialization Configuration +* :pep:`590`, Vectorcall: a fast calling protocol for CPython +* Typing-related: :pep:`591` (Final qualifier), :pep:`586` (Literal types), + and :pep:`589` (TypedDict) * Parallel filesystem cache for compiled bytecode * Debug builds share ABI as release builds * f-strings support a handy ``=`` specifier for debugging @@ -129,13 +134,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 72 - coding: utf-8 - End: diff --git a/pep-0570.rst b/peps/pep-0570.rst similarity index 98% rename from pep-0570.rst rename to peps/pep-0570.rst index 7beb389e5..f3a437d6b 100644 --- a/pep-0570.rst +++ b/peps/pep-0570.rst @@ -8,7 +8,7 @@ Author: Larry Hastings , Eric N. Vander Weele BDFL-Delegate: Guido van Rossum Discussions-To: https://discuss.python.org/t/pep-570-python-positional-only-parameters/1078 -Status: Accepted +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 20-Jan-2018 @@ -264,7 +264,7 @@ the positional syntax would improve consistency. The ``/`` syntax is already exp in the existing documentation such as when builtins and interfaces are generated by the argument clinic. -Another essential aspect to consider is PEP 399, which mandates that +Another essential aspect to consider is :pep:`399`, which mandates that pure Python versions of modules in the standard library *must* have the same interface and semantics that the accelerator modules implemented in C. For example, if ``collections.defaultdict`` were to have a pure Python @@ -282,7 +282,7 @@ The new syntax will enable library authors to further control how their API can be called. It will allow designating which parameters must be called as positional-only, while preventing them from being called as keyword arguments. -Previously, (informational) PEP 457 defined the syntax, but with a much more vague +Previously, (informational) :pep:`457` defined the syntax, but with a much more vague scope. This PEP takes the original proposal a step further by justifying the syntax and providing an implementation for the ``/`` syntax in function definitions. @@ -349,7 +349,7 @@ intended order:: Compatibility for Pure Python and C Modules ------------------------------------------- -Another critical motivation for positional-only parameters is PEP 399: +Another critical motivation for positional-only parameters is :pep:`399`: Pure Python/C Accelerator Module Compatibility Requirements. This PEP states that: @@ -373,7 +373,7 @@ other Python implementations. For example:: 2918445923 Other Python implementations can reproduce the CPython APIs manually, but this -goes against the spirit of PEP 399 to avoid duplication of effort by +goes against the spirit of :pep:`399` to avoid duplication of effort by mandating that all modules added to Python's standard library **must** have a pure Python implementation with the same interface and semantics. @@ -655,7 +655,7 @@ passed by position or keyword:: >>> standard_arg(arg=2) 2 -The second function ``pos_only_arg` is restricted to only use positional +The second function ``pos_only_arg`` is restricted to only use positional parameters as there is a ``/`` in the function definition:: >>> pos_only_arg(1) @@ -824,7 +824,7 @@ unpacked automatically. An example is:: def fxn(a, (b, c), d): pass -Tuple argument unpacking was removed in Python 3 (PEP 3113). There has been a +Tuple argument unpacking was removed in Python 3 (:pep:`3113`). There has been a proposition to reuse this syntax to implement positional-only parameters. We have rejected this syntax for indicating positional only parameters for several reasons: @@ -861,7 +861,7 @@ Thanks ====== Credit for some of the content of this PEP is contained in Larry Hastings’s -PEP 457. +:pep:`457`. Credit for the use of ``/`` as the separator between positional-only and positional-or-keyword parameters go to Guido van Rossum, in a proposal from @@ -900,9 +900,6 @@ Braulio Valdivieso. and https://mail.python.org/pipermail/python-ideas/2012-March/014417.html -.. [#PEP399] - https://www.python.org/dev/peps/pep-0399/ - .. [#python-ideas-decorator-based] https://mail.python.org/pipermail/python-ideas/2017-February/044888.html diff --git a/pep-0571.rst b/peps/pep-0571.rst similarity index 91% rename from pep-0571.rst rename to peps/pep-0571.rst index 433d53d79..8aed6d6d9 100644 --- a/pep-0571.rst +++ b/peps/pep-0571.rst @@ -1,14 +1,13 @@ PEP: 571 Title: The manylinux2010 Platform Tag -Version: $Revision$ -Last-Modified: $Date$ Author: Mark Williams , Geoffrey Thomas , Thomas Kluyver BDFL-Delegate: Nick Coghlan -Discussions-To: Distutils SIG +Discussions-To: distutils-sig@python.org Status: Superseded Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 05-Feb-2018 Post-History: @@ -20,7 +19,7 @@ Abstract ======== This PEP proposes the creation of a ``manylinux2010`` platform tag to -succeed the ``manylinux1`` tag introduced by PEP 513 [1]_. It also +succeed the ``manylinux1`` tag introduced by :pep:`513`. It also proposes that PyPI and ``pip`` both be updated to support uploading, downloading, and installing ``manylinux2010`` distributions on compatible platforms. @@ -44,7 +43,7 @@ built against version 2.5 or earlier; they may then be run systems that provide more recent ``glibc`` version that still export the required symbols at version 2.5. -PEP 513 drew its whitelisted shared libraries and their symbol +:pep:`513` drew its whitelisted shared libraries and their symbol versions from CentOS 5.11, which was the oldest supported CentOS release at the time of its writing. Unfortunately, CentOS 5.11 reached its end-of-life on March 31st, 2017 with a clear warning @@ -55,12 +54,12 @@ packagers who use the ``manylinux1`` Docker image. CentOS 6 is now the oldest supported CentOS release, and will receive maintenance updates through November 30th, 2020. [5]_ We propose that -a new PEP 425-style [6]_ platform tag called ``manylinux2010`` be derived +a new :pep:`425`-style platform tag called ``manylinux2010`` be derived from CentOS 6 and that the ``manylinux`` toolchain, PyPI, and ``pip`` be updated to support it. This was originally proposed as ``manylinux2``, but the versioning has -been changed to use calendar years (also known as CalVer [23]_). This +been changed to use calendar years (also known as CalVer [22]_). This makes it easier to define future *manylinux* tags out of order: for example, a hypothetical ``manylinux2017`` standard may be defined via a new PEP before ``manylinux2014``, or a ``manylinux2007`` standard @@ -109,7 +108,7 @@ the ``manylinux2010`` tag: This list is identical to the externally-provided libraries whitelisted for ``manylinux1``, minus ``libncursesw.so.5`` and ``libpanelw.so.5``. [7]_ ``libpythonX.Y`` remains ineligible for - inclusion for the same reasons outlined in PEP 513. + inclusion for the same reasons outlined in :pep:`513`. ``libcrypt.so.1`` was retrospectively removed from the whitelist after Fedora 30 was released with ``libcrypt.so.2`` instead. @@ -167,7 +166,7 @@ the ``manylinux2010`` tag: against Python 2, then, must include either the ``cpy27mu`` tag indicating it was built against an interpreter with the UCS-4 ABI or the ``cpy27m`` tag indicating an interpreter with the UCS-2 - ABI. [8]_ [9]_ + ABI. (:pep:`3149`, [9]_) 5. A wheel *must not* require the ``PyFPE_jbuf`` symbol. This is achieved by building it against a Python compiled *without* the ``--with-fpectl`` ``configure`` flag. @@ -237,11 +236,11 @@ distribution's upcoming releases. [17]_ If Travis CI, for example, begins running jobs under a kernel that does not provide the ``vsyscall`` interface, Python packagers will not be able to use our Docker images there to build -``manylinux`` wheels. [19]_ +``manylinux`` wheels. [18]_ We have derived a patch from the ``glibc`` git repository that backports the removal of all dependencies on ``vsyscall`` to the -version of ``glibc`` included with our ``manylinux2010`` image. [20]_ +version of ``glibc`` included with our ``manylinux2010`` image. [19]_ Rebuilding ``glibc``, and thus building ``manylinux2010`` image itself, still requires a host kernel that provides the ``vsyscall`` mechanism, but the resulting image can be both run on hosts that provide it and @@ -256,15 +255,15 @@ Auditwheel ---------- The ``auditwheel`` tool has also been updated to produce -``manylinux2010`` wheels. [21]_ Its behavior and purpose are otherwise -unchanged from PEP 513. +``manylinux2010`` wheels. [20]_ Its behavior and purpose are otherwise +unchanged from :pep:`513`. Platform Detection for Installers ================================= Platforms may define a ``manylinux2010_compatible`` boolean attribute on -the ``_manylinux`` module described in PEP 513. A platform is +the ``_manylinux`` module described in :pep:`513`. A platform is considered incompatible with ``manylinux2010`` if the attribute is ``False``. @@ -297,14 +296,14 @@ Specifically, the algorithm we propose is:: Backwards compatibility with ``manylinux1`` wheels ================================================== -As explained in PEP 513, the specified symbol versions for +As explained in :pep:`513`, the specified symbol versions for ``manylinux1`` whitelisted libraries constitute an *upper bound*. The same is true for the symbol versions defined for ``manylinux2010`` in this PEP. As a result, ``manylinux1`` wheels are considered ``manylinux2010`` wheels. A ``pip`` that recognizes the ``manylinux2010`` platform tag will thus install ``manylinux1`` wheels for ``manylinux2010`` platforms -- even when explicitly set -- when no -``manylinux2010`` wheels are available. [22]_ +``manylinux2010`` wheels are available. [21]_ PyPI Support ============ @@ -314,8 +313,8 @@ to be uploaded in the same way that it permits ``manylinux1``. It should not attempt to verify the compatibility of ``manylinux2010`` wheels. -Summary of changes to \PEP 571 -============================== +Summary of changes to PEP 571 +============================= The following changes were made to this PEP based on feedback received after it was approved: @@ -328,8 +327,6 @@ it was approved: References ========== -.. [1] PEP 513 -- A Platform Tag for Portable Linux Built Distributions - (https://www.python.org/dev/peps/pep-0513/) .. [2] pyca/cryptography (https://cryptography.io/) .. [3] numpy @@ -338,13 +335,9 @@ References (https://lists.centos.org/pipermail/centos-announce/2017-April/022350.html) .. [5] CentOS Product Specifications (https://web.archive.org/web/20180108090257/https://wiki.centos.org/About/Product) -.. [6] PEP 425 -- Compatibility Tags for Built Distributions - (https://www.python.org/dev/peps/pep-0425/) .. [7] ncurses 5 -> 6 transition means we probably need to drop some libraries from the manylinux whitelist (https://github.com/pypa/manylinux/issues/94) -.. [8] PEP 3149 - https://www.python.org/dev/peps/pep-3149/ .. [9] SOABI support for Python 2.X and PyPy https://github.com/pypa/pip/pull/3075 .. [10] manylinux2010 Docker image @@ -363,28 +356,18 @@ References (https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852620) .. [17] [Wheel-builders] Heads-up re: new kernel configurations breaking the manylinux docker image (https://mail.python.org/pipermail/wheel-builders/2016-December/000239.html) -.. [18] No longer used -.. [19] Travis CI +.. [18] Travis CI (https://travis-ci.org/) -.. [20] remove-vsyscall.patch +.. [19] remove-vsyscall.patch https://github.com/markrwilliams/manylinux/commit/e9493d55471d153089df3aafca8cfbcb50fa8093#diff-3eda4130bdba562657f3ec7c1b3f5720 -.. [21] auditwheel manylinux2 branch +.. [20] auditwheel manylinux2 branch (https://github.com/markrwilliams/auditwheel/tree/manylinux2) -.. [22] pip manylinux2 branch +.. [21] pip manylinux2 branch https://github.com/markrwilliams/pip/commits/manylinux2 -.. [23] Calendar Versioning +.. [22] Calendar Versioning http://calver.org/ Copyright ========= This document has been placed into the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0572.rst b/peps/pep-0572.rst similarity index 98% rename from pep-0572.rst rename to peps/pep-0572.rst index aaba4f637..4d04081cc 100644 --- a/pep-0572.rst +++ b/peps/pep-0572.rst @@ -1,8 +1,8 @@ PEP: 572 Title: Assignment Expressions Author: Chris Angelico , Tim Peters , - Guido van Rossum -Status: Accepted + Guido van Rossum +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 28-Feb-2018 @@ -290,7 +290,7 @@ The same reason makes these examples invalid too:: While it's technically possible to assign consistent semantics to these cases, it's difficult to determine whether those semantics actually make *sense* in the -absence of real use cases. Accordingly, the reference implementation will ensure +absence of real use cases. Accordingly, the reference implementation [1]_ will ensure that such cases raise ``SyntaxError``, rather than executing with implementation defined behaviour. @@ -388,7 +388,7 @@ may miss the distinction. But simple cases are not objectionable:: print("Extremely long line:", longline) This PEP recommends always putting spaces around ``:=``, similar to -PEP 8's recommendation for ``=`` when used for assignment, whereas the +:pep:`8`'s recommendation for ``=`` when used for assignment, whereas the latter disallows spaces around ``=`` used for keyword arguments.) Change to evaluation order @@ -765,7 +765,7 @@ Broadly the same semantics as the current proposal, but spelled differently. Execution order is inverted (the indented body is performed first, followed by the "header"). This requires a new keyword, unless an existing keyword - is repurposed (most likely ``with:``). See PEP 3150 for prior discussion + is repurposed (most likely ``with:``). See :pep:`3150` for prior discussion on this subject (with the proposed keyword being ``given:``). 5. ``TARGET from EXPR``:: @@ -898,7 +898,7 @@ The less confusing option is to make ``:=`` bind more tightly than comma. Always requiring parentheses ---------------------------- -It's been proposed to just always require parenthesize around an +It's been proposed to just always require parentheses around an assignment expression. This would resolve many ambiguities, and indeed parentheses will frequently be needed to extract the desired subexpression. But in the following cases the extra parentheses feel @@ -955,7 +955,7 @@ Style guide recommendations As expression assignments can sometimes be used equivalently to statement assignments, the question of which should be preferred will arise. For the -benefit of style guides such as PEP 8, two recommendations are suggested. +benefit of style guides such as :pep:`8`, two recommendations are suggested. 1. If either assignment statements or assignment expressions can be used, prefer statements; they are a clear declaration of intent. @@ -1326,14 +1326,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0573.rst b/peps/pep-0573.rst similarity index 95% rename from pep-0573.rst rename to peps/pep-0573.rst index 20d5fb6e1..6007d0015 100644 --- a/pep-0573.rst +++ b/peps/pep-0573.rst @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Petr Viktorin , Nick Coghlan , - Eric Snow + Eric Snow , Marcel Plch BDFL-Delegate: Stefan Behnel Discussions-To: import-sig@python.org @@ -27,12 +27,12 @@ rather than PyState_FindModule for looking up module state, reducing or eliminating the performance cost of using module-scoped state over process global state. -This fixes one of the remaining roadblocks for adoption of PEP 3121 (Extension -module initialization and finalization) and PEP 489 +This fixes one of the remaining roadblocks for adoption of :pep:`3121` (Extension +module initialization and finalization) and :pep:`489` (Multi-phase extension module initialization). While this PEP takes an additional step towards fully solving the problems that -PEP 3121 and PEP 489 started tackling, it does not attempt to resolve *all* +:pep:`3121` and :pep:`489` started tackling, it does not attempt to resolve *all* remaining concerns. In particular, access to the module state from slot methods (``nb_add``, etc) is not solved. @@ -46,8 +46,6 @@ Process-Global State C-level static variables. Since this is very low-level memory storage, it must be managed carefully. -.. _per-module state: - Per-module State ---------------- @@ -101,7 +99,7 @@ CPython implements the C-API, but other implementations exist. Rationale ========= -PEP 489 introduced a new way to initialize extension modules, which brings +:pep:`489` introduced a new way to initialize extension modules, which brings several advantages to extensions that implement it: * The extension modules behave more like their Python counterparts. @@ -112,7 +110,7 @@ several advantages to extensions that implement it: makes it possible to test module isolation (a key feature for proper sub-interpreter support) from a single interpreter. -The biggest hurdle for adoption of PEP 489 is allowing access to module state +The biggest hurdle for adoption of :pep:`489` is allowing access to module state from methods of extension types. Currently, the way to access this state from extension methods is by looking up the module via ``PyState_FindModule`` (in contrast to module level functions in @@ -124,7 +122,7 @@ deterring module authors from using it. Also, ``PyState_FindModule`` relies on the assumption that in each subinterpreter, there is at most one module corresponding to a given ``PyModuleDef``. This assumption does not hold for modules that use -PEP 489's multi-phase initialization, so ``PyState_FindModule`` is unavailable +:pep:`489`'s multi-phase initialization, so ``PyState_FindModule`` is unavailable for these modules. A faster, safer way of accessing module-level state from extension methods @@ -182,7 +180,7 @@ their shared state in C-level process globals, causing problems when: * reloading modules (e.g. to test conditional imports) * loading extension modules in subinterpreters -PEP 3121 attempted to resolve this by offering the ``PyState_FindModule`` API, +:pep:`3121` attempted to resolve this by offering the ``PyState_FindModule`` API, but this still has significant problems when it comes to extension methods (rather than module level functions): @@ -435,7 +433,7 @@ Solving this problem is left to future discussions. Easy creation of types with module references --------------------------------------------- -It would be possible to add a PEP 489 execution slot type to make +It would be possible to add a :pep:`489` execution slot type to make creating heap types significantly easier than calling ``PyType_FromModuleAndSpec``. This is left to a future PEP. @@ -470,22 +468,9 @@ References .. [#gh-patch] https://github.com/Dormouse759/cpython/compare/master...Dormouse759:pep-c-rebase_newer -.. [#pep-590] - https://www.python.org/dev/peps/pep-0590/ - Copyright ========= This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0574.rst b/peps/pep-0574.rst similarity index 97% rename from pep-0574.rst rename to peps/pep-0574.rst index d665e5f0c..891ca3fb3 100644 --- a/pep-0574.rst +++ b/peps/pep-0574.rst @@ -185,7 +185,7 @@ PickleBuffer objects -------------------- The ``PickleBuffer`` class supports a very simple Python API. Its constructor -takes a single PEP 3118-compatible object [#pep-3118]_. ``PickleBuffer`` +takes a single :pep:`3118`-compatible object. ``PickleBuffer`` objects themselves support the buffer protocol, so consumers can call ``memoryview(...)`` on them to get additional information about the underlying buffer (such as the original type, shape, etc.). @@ -207,7 +207,7 @@ PickleBuffer objects: ``PyObject *PyPickleBuffer_FromObject(PyObject *obj)`` - Create a ``PickleBuffer`` object holding a view over the PEP 3118-compatible + Create a ``PickleBuffer`` object holding a view over the :pep:`3118`-compatible *obj*. ``PyPickleBuffer_Check(PyObject *obj)`` @@ -229,7 +229,7 @@ Buffer requirements ``PickleBuffer`` can wrap any kind of buffer, including non-contiguous buffers. However, it is required that ``__reduce__`` only returns a -contiguous ``PickleBuffer`` (*contiguity* here is meant in the PEP 3118 +contiguous ``PickleBuffer`` (*contiguity* here is meant in the :pep:`3118` sense: either C-ordered or Fortran-ordered). Non-contiguous buffers will raise an error when pickled. @@ -347,7 +347,7 @@ Caveats Mutability ---------- -PEP 3118 buffers [#pep-3118]_ can be readonly or writable. Some objects, +:pep:`3118` buffers can be readonly or writable. Some objects, such as Numpy arrays, need to be backed by a mutable buffer for full operation. Pickle consumers that use the ``buffer_callback`` and ``buffers`` arguments will have to be careful to recreate mutable buffers. When doing @@ -504,9 +504,9 @@ to pickle [#dask-serialization]_. PyArrow implements zero-copy component-based serialization for a few selected types [#pyarrow-serialization]_. -PEP 554 proposes hosting multiple interpreters in a single process, with +:pep:`554` proposes hosting multiple interpreters in a single process, with provisions for transferring buffers between interpreters as a communication -scheme [#pep-554]_. +scheme. Acknowledgements @@ -538,12 +538,6 @@ References .. [#pyarrow-serialization] PyArrow IPC and component-based serialization https://arrow.apache.org/docs/python/ipc.html#component-based-serialization -.. [#pep-3118] PEP 3118 -- Revising the buffer protocol - https://www.python.org/dev/peps/pep-3118/ - -.. [#pep-554] PEP 554 -- Multiple Interpreters in the Stdlib - https://www.python.org/dev/peps/pep-0554/ - .. [#pickle5-git] ``pickle5`` branch on GitHub https://github.com/pitrou/cpython/tree/pickle5 diff --git a/pep-0575.rst b/peps/pep-0575.rst similarity index 98% rename from pep-0575.rst rename to peps/pep-0575.rst index 407b51317..35218ea47 100644 --- a/pep-0575.rst +++ b/peps/pep-0575.rst @@ -6,15 +6,15 @@ Type: Standards Track Content-Type: text/x-rst Created: 27-Mar-2018 Python-Version: 3.8 -Post-History: 31-Mar-2018, 12-Apr-2018, 27-Apr-2018, 5-May-2018 +Post-History: 31-Mar-2018, 12-Apr-2018, 27-Apr-2018, 05-May-2018 Withdrawal notice ================= -See PEP 580 for a better solution to allowing fast calling of custom classes. +See :pep:`580` for a better solution to allowing fast calling of custom classes. -See PEP 579 for a broader discussion of some of the other issues from this PEP. +See :pep:`579` for a broader discussion of some of the other issues from this PEP. Abstract @@ -67,7 +67,7 @@ subclasses are also allowed for functions implemented in C. In the latter case, this can be done with the same performance as true built-in functions. All functions can access the function object -(the ``self`` in ``__call__``), paving the way for PEP 573. +(the ``self`` in ``__call__``), paving the way for :pep:`573`. New classes @@ -121,7 +121,7 @@ but with the following differences and new features: This is meant to be backwards compatible with ``method_descriptor``. #. The field ``ml_doc`` and the attributes ``__doc__`` and - ``__text_signature__`` (see Argument Clinic [#clinic]_) + ``__text_signature__`` (see :pep:`Argument Clinic <436>`) are not supported. #. A new flag ``METH_PASS_FUNCTION`` for ``ml_flags``. @@ -203,11 +203,11 @@ The class ``cfunction`` is a copy of ``base_function``, with the following diffe const char *ml_doc; } PyMethodDef; - Note that ``PyMethodDef`` is part of the Python Stable ABI [#ABI]_ + Note that ``PyMethodDef`` is part of the :pep:`Python Stable ABI <384>` and it is used by practically all extension modules, so we absolutely cannot change this structure. -#. Argument Clinic [#clinic]_ is supported. +#. :pep:`Argument Clinic <436>` is supported. #. ``__self__`` always exists. In the cases where ``base_function.__self__`` would raise ``AttributeError``, instead ``None`` is returned. @@ -792,7 +792,7 @@ to use the ``base_function`` implementation instead. It also makes sense to use ``METH_PASS_FUNCTION`` without ``METH_CALL_UNBOUND`` in cases where the C function simply needs access to additional metadata from the function, such as the ``__parent__``. -This is for example needed to support PEP 573. +This is for example needed to support :pep:`573`. Converting existing methods to use ``METH_PASS_FUNCTION`` is trivial: it only requires adding an extra argument to the C function. @@ -1097,10 +1097,6 @@ References .. [#bpo33265] Python bug 33265, contextlib.ExitStack abuses __self__ (https://bugs.python.org/issue33265 and https://github.com/python/cpython/pull/6456) -.. [#ABI] PEP 384, Defining a Stable ABI, Löwis (https://www.python.org/dev/peps/pep-0384) - -.. [#clinic] PEP 436, The Argument Clinic DSL, Hastings (https://www.python.org/dev/peps/pep-0436) - .. [#methoddoc] PyMethodDef documentation (https://docs.python.org/3.7/c-api/structures.html#c.PyMethodDef) .. [#proposal] PEP proposal: unifying function/method classes (https://mail.python.org/pipermail/python-ideas/2018-March/049398.html) diff --git a/pep-0576.rst b/peps/pep-0576.rst similarity index 97% rename from pep-0576.rst rename to peps/pep-0576.rst index bd0f4c95f..6ea831506 100644 --- a/pep-0576.rst +++ b/peps/pep-0576.rst @@ -7,10 +7,10 @@ Type: Standards Track Content-Type: text/x-rst Created: 10-May-2018 Python-Version: 3.8 -Post-History: 17-May-2018 - 23-June-2018 - 08-July-2018 - 29-Mar-1028 +Post-History: 17-May-2018, + 23-Jun-2018, + 08-Jul-2018, + 29-Mar-2019 Abstract ======== @@ -136,7 +136,7 @@ Adding a function pointer to each callable, rather than each class of callable, Alternative Suggestions ======================= -PEP 580 is an alternative approach to solving the same problem as this PEP. +:pep:`580` is an alternative approach to solving the same problem as this PEP. diff --git a/pep-0577.rst b/peps/pep-0577.rst similarity index 93% rename from pep-0577.rst rename to peps/pep-0577.rst index b605c76f7..91684a025 100644 --- a/pep-0577.rst +++ b/peps/pep-0577.rst @@ -13,13 +13,13 @@ PEP Withdrawal ============== While working on this PEP, I realised that it didn't really address what was -actually bothering me about PEP 572's proposed scoping rules for previously +actually bothering me about :pep:`572`'s proposed scoping rules for previously unreferenced assignment targets, and also had some significant undesirable -consequences (most notably, allowing ``>>=` and ``<<=`` as inline augmented +consequences (most notably, allowing ``>>=`` and ``<<=`` as inline augmented assignment operators that meant something entirely different from the ``>=`` and ``<=`` comparison operators). -I also realised that even without dedicated syntax of their own, PEP 572 +I also realised that even without dedicated syntax of their own, :pep:`572` technically allows inline augmented assignments to be written using the ``operator`` module:: @@ -39,7 +39,7 @@ the time I also started writing a replacement PEP that focused specifically on the handling of assignment targets which hadn't already been declared as local variables in the current scope (for both regular block scopes, and for scoped expressions), but that draft never even reached a stage where *I* liked it -better than the ultimately accepted proposal in PEP 572, so it was never +better than the ultimately accepted proposal in :pep:`572`, so it was never posted anywhere, nor assigned a PEP number. @@ -56,7 +56,7 @@ statements do. The question of allowing expression level local variable declarations at function scope is deliberately separated from the question of allowing expression level name bindings, and deferred to a later PEP. -This PEP is a direct competitor to PEP 572 (although it borrows heavily from that +This PEP is a direct competitor to :pep:`572` (although it borrows heavily from that PEP's motivation, and even shares the proposed syntax for inline assignments). See `Relationship with PEP 572`_ for more details on the connections between the two PEPs. @@ -145,7 +145,7 @@ used today (in combination with ``operator.itemsetter``) to work around the lack of expression level assignments. Rather than requiring such workarounds, this PEP instead proposes that -PEP 572's "NAME := EXPR" syntax be adopted as a new inline assignment +:pep:`572`'s "NAME := EXPR" syntax be adopted as a new inline assignment expression that uses the augmented assignment scoping rules described below. This cleanly handles cases where only the new value is of interest, and the @@ -169,12 +169,12 @@ that when used as a top level expression the entire right hand side of the expression is still interpreted as the value to be processed (even when that value is a tuple without parentheses). -The difference this introduces relative to PEP 572 is that where -``(n := first, second)`` sets ``n = first`` in PEP 572, in this PEP it would set -``n = (first, second)`, and getting the first meaning would require an extra +The difference this introduces relative to :pep:`572` is that where +``(n := first, second)`` sets ``n = first`` in :pep:`572`, in this PEP it would set +``n = (first, second)``, and getting the first meaning would require an extra set of parentheses (``((n := first), second)``). -PEP 572 quite reasonably notes that this results in ambiguity when assignment +:pep:`572` quite reasonably notes that this results in ambiguity when assignment expressions are used as function call arguments. This PEP resolves that concern a different way by requiring that assignment expressions be parenthesised when used as arguments to a function call (unless they're the sole argument). @@ -368,7 +368,7 @@ Design discussion Allowing complex assignment targets ----------------------------------- -The initial drafts of this PEP kept PEP 572's restriction to single name targets +The initial drafts of this PEP kept :pep:`572`'s restriction to single name targets when augmented assignments were used as expressions, allowing attribute and subscript targets solely for the statement form. @@ -385,7 +385,7 @@ that also gained support for assignment and subscript targets. Augmented assignment or name binding only? ------------------------------------------ -PEP 572 makes a reasonable case that the potential use cases for inline +:pep:`572` makes a reasonable case that the potential use cases for inline augmented assignment are notably weaker than those for inline assignment in general, so it's acceptable to require that they be spelled as ``x := x + 1``, bypassing any in-place augmented assignment methods. @@ -442,7 +442,7 @@ currently executing module and modifying its attributes). For class scopes, the answer to both questions is also "Yes" in practice, although less unequivocally so, since the semantics of ``locals()`` are currently formally unspecified. However, if the current behaviour of ``locals()`` -at class scope is taken as normative (as PEP 558 proposes), then this is +at class scope is taken as normative (as :pep:`558` proposes), then this is essentially the same scenario as manipulating the module globals, just using ``locals()`` instead. @@ -489,8 +489,8 @@ level bindings are sufficient. Ignoring scoped expressions when determining augmented assignment targets ------------------------------------------------------------------------- -When discussing possible binding semantics for PEP 572's assignment expressions, -Tim Peters made a plausible case [1_,2_,3_] for assignment expressions targeting +When discussing possible binding semantics for :pep:`572`'s assignment expressions, +Tim Peters made a plausible case [1]_, [2]_, [3]_ for assignment expressions targeting the containing block scope, essentially ignoring any intervening scoped expressions. @@ -505,7 +505,7 @@ way:: while any(n % (factor := p) == 0 for p in small_primes): n //= factor -Guido also expressed his approval for this general approach [4_]. +Guido also expressed his approval for this general approach [4]_. The proposal in this PEP differs from Tim's original proposal in three main areas: @@ -531,12 +531,12 @@ to detect it), so there's also no need to tinker with that. Treating inline assignment as an augmented assignment variant ------------------------------------------------------------- -One of the challenges with PEP 572 is the fact that ``NAME = EXPR`` and +One of the challenges with :pep:`572` is the fact that ``NAME = EXPR`` and ``NAME := EXPR`` are entirely semantically equivalent at every scope. This makes the two forms hard to teach, since there's no inherent nudge towards choosing one over the other at the statement level, so you end up having to resort to "``NAME = EXPR`` is preferred because it's been around longer" -(and PEP 572 proposes to enfore that historical idiosyncrasy at the compiler +(and :pep:`572` proposes to enforce that historical idiosyncrasy at the compiler level). That semantic equivalence is difficult to avoid at module and class scope while @@ -554,7 +554,7 @@ design requirement, then this PEP would be updated to propose that ``EXPR given NAME`` also be introduced as a way to support inline name declarations after arbitrary expressions (this would allow the inline name declarations to be deferred until the end of a complex expression rather than needing to be -embedded in the middle of it, and PEP 8 would gain a recommendation encouraging +embedded in the middle of it, and :pep:`8` would gain a recommendation encouraging that style). @@ -586,7 +586,7 @@ operation:: x >= y # Greater-than-or-equal-to x <= y # Less-than-or-equal-to -Both this PEP and PEP 572 propose adding at least one operator that's somewhat +Both this PEP and :pep:`572` propose adding at least one operator that's somewhat similar in appearance, but defines an assignment instead:: x := y # Becomes @@ -687,7 +687,7 @@ be written using nested if-else statements:: else: ... -As with PEP 572, this PEP allows the else/if portions of that chain to be +As with :pep:`572`, this PEP allows the else/if portions of that chain to be condensed, making their consistent and mutually exclusive structure more readily apparent:: @@ -701,7 +701,7 @@ readily apparent:: else: ... -Unlike PEP 572, this PEP requires that the assignment target be explicitly +Unlike :pep:`572`, this PEP requires that the assignment target be explicitly indicated as local before the first use as a ``:=`` target, either by binding it to a value (as shown above), or else by including an appropriate explicit type declaration:: @@ -761,27 +761,27 @@ practice. Relationship with PEP 572 ========================= -The case for allowing inline assignments at all is made in PEP 572. This +The case for allowing inline assignments at all is made in :pep:`572`. This competing PEP was initially going to propose an alternate surface syntax (``EXPR given NAME = EXPR``), while retaining the expression semantics from -PEP 572, but that changed when discussing one of the initial motivating use +:pep:`572`, but that changed when discussing one of the initial motivating use cases for allowing embedded assignments at all: making it possible to easily calculate cumulative sums in comprehensions and generator expressions. -As a result of that, and unlike PEP 572, this PEP focuses primarily on use +As a result of that, and unlike :pep:`572`, this PEP focuses primarily on use cases for inline augmented assignment. It also has the effect of converting cases that currently inevitably raise ``UnboundLocalError`` at function call time to report a new compile time ``TargetNameError``. New syntax for a name rebinding expression (``NAME := TARGET``) is then added -not only to handle the same use cases as are identified in PEP 572, but also +not only to handle the same use cases as are identified in :pep:`572`, but also as a lower level primitive to help illustrate, implement and explain the new augmented assignment semantics, rather than being the sole change being proposed. The author of this PEP believes that this approach makes the value of the new flexibility in name rebinding clearer, while also mitigating many of the -potential concerns raised with PEP 572 around explaining when to use +potential concerns raised with :pep:`572` around explaining when to use ``NAME = EXPR`` over ``NAME := EXPR`` (and vice-versa), without resorting to prohibiting the bare statement form of ``NAME := EXPR`` outright (such that ``NAME := EXPR`` is a compile error, but ``(NAME := EXPR)`` is permitted). @@ -790,7 +790,7 @@ that ``NAME := EXPR`` is a compile error, but ``(NAME := EXPR)`` is permitted). Acknowledgements ================ -The PEP author wishes to thank Chris Angelico for his work on PEP 572, and his +The PEP author wishes to thank Chris Angelico for his work on :pep:`572`, and his efforts to create a coherent summary of the great many sprawling discussions that spawned on both python-ideas and python-dev, as well as Tim Peters for the in-depth discussion of parent local scoping that prompted the above @@ -820,14 +820,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0578.rst b/peps/pep-0578.rst similarity index 99% rename from pep-0578.rst rename to peps/pep-0578.rst index c1c63f5e0..9e5d25ec6 100644 --- a/pep-0578.rst +++ b/peps/pep-0578.rst @@ -9,7 +9,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 16-Jun-2018 Python-Version: 3.8 -Post-History: 28-March-2019, 07-May-2019 +Post-History: 28-Mar-2019, 07-May-2019 Abstract ======== @@ -29,7 +29,7 @@ are unspecified here to allow implementations the freedom to determine how best to provide information to their users. Some examples likely to be used in CPython are provided for explanatory purposes. -See PEP 551 for discussion and recommendations on enhancing the +See :pep:`551` for discussion and recommendations on enhancing the security of a Python runtime making use of these auditing APIs. Background @@ -545,13 +545,13 @@ Relationship to PEP 551 ======================= This API was originally presented as part of -`PEP 551 `_ Security +:pep:`551` Security Transparency in the Python Runtime. For simpler review purposes, and due to the broader applicability of these APIs beyond security, the API design is now presented separately. -PEP 551 is an informational PEP discussing how to integrate Python into +:pep:`551` is an informational PEP discussing how to integrate Python into a secure or audited environment. References diff --git a/pep-0579.rst b/peps/pep-0579.rst similarity index 97% rename from pep-0579.rst rename to peps/pep-0579.rst index 800695714..6edaac6e2 100644 --- a/pep-0579.rst +++ b/peps/pep-0579.rst @@ -12,10 +12,10 @@ Post-History: 20-Jun-2018 Approval Notice =============== -This PEP describes design issues addressed in PEP 575, PEP 580, PEP 590 +This PEP describes design issues addressed in :pep:`575`, :pep:`580`, :pep:`590` (and possibly later proposals). -As noted in `PEP 1 `_: +As noted in :pep:`PEP 1 <1#pep-types>`: Informational PEPs do not necessarily represent a Python community consensus or recommendation, so users and implementers are free to @@ -119,7 +119,7 @@ be the *only* remaining purpose of the ``PyMethodDef`` structure. Additionally, we can also make some function classes subclassable. However, this seems less important once we have ``tp_ccalloffset``. -**Reference**: PEP 580 +**Reference**: :pep:`580` 3. cfunctions do not become methods @@ -158,7 +158,7 @@ should be part of the new C call protocol. For backwards compatibility, we would keep the existing non-binding behavior of cfunctions. We would just allow it in custom classes. -**Reference**: PEP 580 +**Reference**: :pep:`580` 4. Semantics of inspect.isfunction @@ -195,7 +195,7 @@ C call protocol for Python functions (``types.FunctionType``): the C function which implements calling Python functions needs access to the ``__code__`` attribute of the function. -This is also needed for PEP 573 +This is also needed for :pep:`573` where all cfunctions require access to their "parent" (the module for functions of a module or the defining class for methods). @@ -204,7 +204,7 @@ for methods). that the C function takes an additional argument (as first argument), namely the function object. -**References**: PEP 580, PEP 573 +**References**: :pep:`580`, :pep:`573` 6. METH_FASTCALL is private and undocumented @@ -226,7 +226,7 @@ As part of the C call protocol, we should also add a C API function :: PyObject *PyCCall_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords) -**Reference**: PEP 580 +**Reference**: :pep:`580` 7. Allowing native C arguments @@ -303,7 +303,7 @@ instead of doing type checks. Furthermore, it should be investigated whether some of these classes can be merged and whether ``method`` can be re-used also for bound methods of extension types -(see PEP 576 for the latter, +(see :pep:`576` for the latter, keeping in mind that this may have some minor backwards compatibility issues). This is not a goal by itself but just something to keep in mind when working on these classes. @@ -316,7 +316,7 @@ The typical way to create a cfunction or cmethod in an extension module is by using a ``PyMethodDef`` to define it. These are then stored in an array ``PyModuleDef.m_methods`` (for cfunctions) or ``PyTypeObject.tp_methods`` (for cmethods). -However, because of the stable ABI (PEP 384), +However, because of the stable ABI (:pep:`384`), we cannot change the ``PyMethodDef`` structure. So, this means that we cannot add new fields for creating cfunctions/cmethods diff --git a/pep-0580.rst b/peps/pep-0580.rst similarity index 96% rename from pep-0580.rst rename to peps/pep-0580.rst index 9b95abe22..d2822a43b 100644 --- a/pep-0580.rst +++ b/peps/pep-0580.rst @@ -13,7 +13,7 @@ Post-History: 20-Jun-2018, 22-Jun-2018, 16-Jul-2018 Rejection Notice ================ -This PEP is rejected in favor of PEP 590, which proposes a simpler public +This PEP is rejected in favor of :pep:`590`, which proposes a simpler public C API for callable objects. @@ -63,7 +63,7 @@ enabling even further simplifications. We also design the C call protocol such that it can easily be extended with new features in the future. -For more background and motivation, see PEP 579. +For more background and motivation, see :pep:`579`. Overview @@ -176,7 +176,7 @@ For methods of extension types, ``cc_parent`` points to the class that defines the method (which may be a superclass of ``type(self)``). This is currently non-trivial to retrieve from a method's code. In the future, this can be used to access the module state via -the defining class. See the rationale of PEP 573 for details. +the defining class. See the rationale of :pep:`573` for details. When the flag ``CCALL_OBJCLASS`` is set (as it will be for methods of extension types), ``cc_parent`` is used for type checks like the following:: @@ -212,7 +212,7 @@ becomes ``tp_ccalloffset`` unconditionally, drop the ``Py_TPFLAGS_HAVE_CCALL`` flag and instead check for ``tp_ccalloffset != 0``. -**NOTE**: the exact layout of ``PyTypeObject`` is not part of the stable ABI ([#pep384]_). +**NOTE**: the exact layout of ``PyTypeObject`` is not part of the :pep:`stable ABI <384>`). Therefore, changing the ``tp_print`` field from a ``printfunc`` (a function pointer) to a ``Py_ssize_t`` should not be a problem, even if this changes the memory layout of the ``PyTypeObject`` structure. @@ -479,7 +479,7 @@ compatibility problems. C API functions --------------- -The following function is added (also to the stable ABI [#pep384]_): +The following function is added (also to the :pep:`stable ABI <384>`): - ``PyObject * PyCFunction_ClsNew(PyTypeObject *cls, PyMethodDef *ml, PyObject *self, PyObject *module, PyObject *parent)``: create a new object with object structure ``PyCFunctionObject`` and class ``cls``. @@ -497,7 +497,7 @@ and ``PyCFunction_GET_FLAGS`` are deprecated. They are still artificially supported by storing the original ``METH_...`` flags in a bitfield inside ``cc_flags``. Despite the fact that ``PyCFunction_GetFlags`` is technically -part of the stable ABI [#pep384]_, +part of the :pep:`stable ABI <384>`, it is highly unlikely to be used that way: first of all, it is not even documented. Second, the flag ``METH_FASTCALL`` @@ -544,7 +544,7 @@ performance improvements are discussed: Stable ABI ========== -The function ``PyCFunction_ClsNew`` is added to the stable ABI [#pep384]_. +The function ``PyCFunction_ClsNew`` is added to the :pep:`stable ABI <384>`. None of the functions, structures or constants dealing with the C call protocol are added to the stable ABI. @@ -552,7 +552,7 @@ are added to the stable ABI. There are two reasons for this: first of all, the most useful feature of the C call protocol is probably the ``METH_FASTCALL`` calling convention. -Given that this is not even part of the public API (see also PEP 579, issue 6), +Given that this is not even part of the public API (see also :pep:`579`, issue 6), it would be strange to add anything else from the C call protocol to the stable ABI. @@ -579,7 +579,7 @@ Rationale Why is this better than PEP 575? -------------------------------- -One of the major complaints of PEP 575 was that is was coupling +One of the major complaints of :pep:`575` was that is was coupling functionality (the calling and introspection protocol) with the class hierarchy: a class could only benefit from the new features @@ -587,7 +587,7 @@ if it was a subclass of ``base_function``. It may be difficult for existing classes to do that because they may have other constraints on the layout of the C object structure, coming from an existing base class or implementation details. -For example, ``functools.lru_cache`` cannot implement PEP 575 as-is. +For example, ``functools.lru_cache`` cannot implement :pep:`575` as-is. It also complicated the implementation precisely because changes were needed both in the implementation details and in the class hierarchy. @@ -687,7 +687,7 @@ Why CCALL_DEFARG? The flag ``CCALL_DEFARG`` gives the callee access to the ``PyCCallDef *``. There are various use cases for this: -1. The callee can use the ``cc_parent`` field, which is useful for PEP 573. +1. The callee can use the ``cc_parent`` field, which is useful for :pep:`573`. 2. Applications are free to extend the ``PyCCallDef`` structure with user-defined fields, which can then be accessed analogously. @@ -720,9 +720,9 @@ In particular, the Cython project has shown interest in doing that Alternative suggestions ======================= -PEP 576 is an alternative approach to solving the same problem as this PEP. +:pep:`576` is an alternative approach to solving the same problem as this PEP. See https://mail.python.org/pipermail/python-dev/2018-July/154238.html -for comments on the difference between PEP 576 and PEP 580. +for comments on the difference between :pep:`576` and :pep:`580`. Discussion @@ -757,16 +757,13 @@ The reference implementation can be found at https://github.com/jdemeyer/cpython/tree/pep580 For an example of using the C call protocol, -the following branch implements ``functools.lru_cache`` using PEP 580: +the following branch implements ``functools.lru_cache`` using :pep:`580`: https://github.com/jdemeyer/cpython/tree/lru580 References ========== -.. [#pep384] Löwis, PEP 384 – Defining a Stable ABI, - https://www.python.org/dev/peps/pep-0384/ - .. [#bpo29259] Add tp_fastcall to PyTypeObject: support FASTCALL calling convention for all callable objects, https://bugs.python.org/issue29259 diff --git a/pep-0581.rst b/peps/pep-0581.rst similarity index 96% rename from pep-0581.rst rename to peps/pep-0581.rst index 71f65eb54..6ef0843de 100644 --- a/pep-0581.rst +++ b/peps/pep-0581.rst @@ -4,12 +4,12 @@ Version: $Revision$ Last-Modified: $Date$ Author: Mariatta BDFL-Delegate: Barry Warsaw -Discussions-To: Core-Workflow Category on Discourse +Discussions-To: https://discuss.python.org/t/535 Status: Accepted Type: Process Content-Type: text/x-rst Created: 20-Jun-2018 -Post-History: 7-Mar-2019 +Post-History: 07-Mar-2019 Resolution: https://mail.python.org/pipermail/python-dev/2019-May/157399.html @@ -17,7 +17,7 @@ Abstract ======== This PEP outlines the rationale for migration from Python's issue -tracker on Roundup to GitHub issues. See PEP 588 for the detailed +tracker on Roundup to GitHub issues. See :pep:`588` for the detailed migration plan. @@ -111,10 +111,10 @@ Issues with Roundup / bpo - Note: Exposing email address to registered and logged in users was a decision taken when bpo instance was setup. This behavior has been recently modified - after PEP 581's acceptance. + after :pep:`581`'s acceptance. - REST API is not currently available in bpo. There was an open issue in Roundup - for adding REST API [#]_. At the time PEP 581 was proposed, the ticket received + for adding REST API [#]_. At the time :pep:`581` was proposed, the ticket received no activity since 2016. REST API has been integrated in Roundup in February 2019, however it is not yet integrated to bpo. @@ -188,7 +188,7 @@ GitHub is not the perfect issue tracker. Several issues we need to be aware of: - Using GitHub could possibly increase the triaging effort. This was first raised as a Zulip topic [#]_, and also brought up during Core Python sprint in September 2018 [#]_. A few solutions have been proposed and considered, such as - creating a special triage team [#]_. After PEP 581's acceptance, GitHub released a + creating a special triage team [#]_. After :pep:`581`'s acceptance, GitHub released a new triaging role, currently in beta. The PSF has been in touch with GitHub to have this enabled for Python organization. This is pending GitHub's review [#]_. @@ -208,7 +208,7 @@ GitHub is not the perfect issue tracker. Several issues we need to be aware of: - bpo uses a number of fields to specify several metadata, and these might not be easily transferable to GitHub. The intended way to handle custom metadata on GitHub is by using labels. The details of which labels to create will be - further discussed in PEP 588. + further discussed in :pep:`588`. Further questions and discussions diff --git a/peps/pep-0582.rst b/peps/pep-0582.rst new file mode 100644 index 000000000..d21c30b0e --- /dev/null +++ b/peps/pep-0582.rst @@ -0,0 +1,351 @@ +PEP: 582 +Title: Python local packages directory +Version: $Revision$ +Last-Modified: $Date$ +Author: Kushal Das , Steve Dower , + Donald Stufft , Nick Coghlan +Discussions-To: https://discuss.python.org/t/pep-582-python-local-packages-directory/963/ +Status: Rejected +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 16-May-2018 +Python-Version: 3.12 +Post-History: `01-Mar-2019 `__, +Resolution: https://discuss.python.org/t/pep-582-python-local-packages-directory/963/430 + + +Abstract +======== + +This PEP proposes extending the existing mechanism for setting up ``sys.path`` +to include a new ``__pypackages__`` directory, in addition to the existing +locations. The new directory will be added at the start of ``sys.path``, after +the current working directory and just before the system site-packages, to give +packages installed there priority over other locations. + +This is similar to the existing mechanism of adding the current directory (or +the directory the script is located in), but by using a subdirectory, +additional libraries are kept separate from the user's work. + + +Motivation +========== + +New Python programmers can benefit from being taught the value of isolating an +individual project's dependencies from their system environment. However, the +existing mechanism for doing this, virtual environments, is known to be complex +and error-prone for beginners to understand. Explaining virtual environments is +often a distraction when trying to get a group of beginners set up - differences +in platform and shell environments require individual assistance, and the need +for activation in every new shell session makes it easy for students to make +mistakes when coming back to work after a break. This proposal offers a lightweight +solution that gives isolation without the user needing to understand more +advanced concepts. + +Furthermore, standalone Python applications usually need 3rd party libraries to +function. Typically, they are either designed to be run from a virtual environment, +where the dependencies are installed into the environment alongside the application, +or they bundle their dependencies in a subdirectory, and modify ``sys.path`` at +application startup. Virtual environments, while a common and effective solution +(used, for example, by the ``pipx`` tool), are somewhat awkward to set up and manage, +and are not relocatable. On the other hand, manual manipulation of ``sys.path`` is +boilerplate that developers need to get right, and (being a runtime behaviour) +it is not understood by tools like linters and type checkers. The ``__pypackages__`` +proposal formalises the idea of a "bundled dependencies" location, avoiding the +boilerplate and providing a standard location that development tools can be taught +to recognise. + +It should be noted that in general, Python libraries cannot be simply copied +between machines, platforms, or even necessarily between Python versions. This +proposal does nothing to change that fact, and while it is tempting to assume +that bundling a script and its ``__pypackages__`` is a mechanism for +distributing applications, this is explicitly *not* a goal of this proposal. +Developers remain responsible for the portability of their code. + +Rationale +========= + +While ``sys.path`` can be manipulated at runtime, the default value is important, as +it establishes a common baseline that users and tools can agree on. The current default +does not include a location that could be viewed as "private to the current project", +and yet that is a useful concept. + +This is similar to the npm ``node_modules`` directory, which is popular in the +Javascript community, and something that developers familiar with that +ecosystem often ask for from Python. + + +Specification +============= + + +This PEP proposes to add a new step in the process of calculating ``sys.path`` at +startup. + +When the interactive interpreter starts, if a ``__pypackages__`` directory is +found in the current working directory, then it will be included in +``sys.path`` after the entry for current working directory and just before the +system site-packages. + +When the interpreter runs a script, Python will try to find ``__pypackages__`` +in the same directory as the script. If found (along with the current Python +version directory inside), then it will be used, otherwise Python will behave +as it does currently. + +The behaviour should work exactly the same as the way the existing mechanism +for adding the current working directory or script directory to ``sys.path`` +works. For example, ``__pypackages__`` will be ignored if the ``-P`` option or +the ``PYTHONSAFEPATH`` environment variable is set. + +In order to be recognised, the ``__pypackages__`` directory must be laid out +according to a new ``localpackages`` scheme in the sysconfig module. +Specifically, both of the ``purelib`` and ``platlib`` directories must be +present, using the following code to determine the locations of those +directories:: + + scheme = "localpackages" + purelib = sysconfig.get_path("purelib", scheme, vars={"base": "__pypackages__", "platbase": "__pypackages__"}) + platlib = sysconfig.get_path("platlib", scheme, vars={"base": "__pypackages__", "platbase": "__pypackages__"}) + +These two locations will be added to ``sys.path``, other directories or +files in the ``__pypackages__`` directory will be silently ignored. The +paths will be based on Python versions. + +.. note:: There is a possible option of having a separate new API, it is documented at `issue #3013 `_. + + +Example +------- + +The following shows an example project directory structure, and different ways +the Python executable and any script will behave. The example is for Unix-like +systems - on Windows the subdirectories will be different. + +:: + + foo + __pypackages__ + lib + python3.10 + site-packages + bottle + myscript.py + + /> python foo/myscript.py + sys.path[0] == 'foo' + sys.path[1] == 'foo/__pypackages__/lib/python3.10/site-packages/' + + + cd foo + + foo> /usr/bin/ansible + #! /usr/bin/env python3 + foo> python /usr/bin/ansible + + foo> python myscript.py + + foo> python + sys.path[0] == '.' + sys.path[1] == './__pypackages__/lib/python3.10/site-packages' + + foo> python -m bottle + +We have a project directory called ``foo`` and it has a ``__pypackages__`` +inside of it. We have ``bottle`` installed in that +``__pypackages__/lib/python3.10/site-packages/``, and have a ``myscript.py`` +file inside of the project directory. We have used whatever tool we generally +use to install ``bottle`` in that location. + +For invoking a script, Python will try to find a ``__pypackages__`` inside of +the directory that the script resides [1]_, ``/usr/bin``. The same will happen +in case of the last example, where we are executing ``/usr/bin/ansible`` from +inside of the ``foo`` directory. In both cases, it will **not** use the +``__pypackages__`` in the current working directory. + +Similarly, if we invoke ``myscript.py`` from the first example, it will use the +``__pypackages__`` directory that was in the ``foo`` directory. + +If we go inside of the ``foo`` directory and start the Python executable (the +interpreter), it will find the ``__pypackages__`` directory inside of the +current working directory and use it in the ``sys.path``. The same happens if we +try to use the ``-m`` and use a module. In our example, ``bottle`` module will +be found inside of the ``__pypackages__`` directory. + +The above two examples are only cases where ``__pypackages__`` from current +working directory is used. + +In another example scenario, a trainer of a Python class can say "Today we are +going to learn how to use Twisted! To start, please checkout our example +project, go to that directory, and then run a given command to install Twisted." + +That will install Twisted into a directory separate from ``python3``. There's no +need to discuss virtual environments, global versus user installs, etc. as the +install will be local by default. The trainer can then just keep telling them to +use ``python3`` without any activation step, etc. + + +.. [1] In the case of symlinks, it is the directory where the actual script + resides, not the symlink pointing to the script + + +Relationship to virtual environments +==================================== + +At its heart, this proposal is simply to modify the calculation of the default +value of ``sys.path``, and does not relate at all to the virtual environment +mechanism. However, ``__pypackages__`` can be viewed as providing an isolation +capability, and in that sense, it "competes" with virtual environments. + +However, there are significant differences: + + * Virtual environments are isolated from the system environment, whereas + ``__pypackages__`` simply adds to the system environment. + * Virtual environments include a full "installation scheme", with directories + for binaries, C header files, etc., whereas ``__pypackages__`` is solely + for Python library code. + * Virtual environments work most smoothly when "activated". This proposal + needs no activation. + +This proposal should be seen as independent of virtual environments, not competing +with them. At best, some use cases currently only served by virtual environments +can also be served (possibly better) by ``__pypackages__``. + +It should be noted that libraries installed in ``__pypackages__`` will be visible +in a virtual environment. This arguably breaks the isolation of virtual environments, +but it is no different in principle to the presence of the current directory on +``sys.path`` (or mechanisms like the ``PYTHONPATH`` environment variable). The only +difference is in degree, as the expectation is that people will more commonly install +packages in ``__pypackages__``. The alternative would be to explicitly detect virtual +environments and disable ``__pypackages__`` in that case - however that would break +scripts with bundled dependencies. The PEP authors believe that developers using +virtual environments should be experienced enough to understand the issue and +anticipate and avoid any problems. + +Security Considerations +======================= + +In theory, it is possible to add a library to the ``__pypackages__`` directory +that overrides a stdlib module or an installed 3rd party library. For the +``__pypackages__`` associated with a script, this is assumed not to be a +significant issue, as it is unlikely that anyone would be able to write to +``__pypackages__`` unless they also had the ability to write to the script itself. + +For a ``__pypackages__`` directory in the current working directory, the +interactive interpreter could be affected. However, this is not significantly +different than the existing issue of someone having a ``math.py`` mdule in their +current directory, and while (just like that case) it can cause user confusion, +it does not introduce any new security implications. + +When running a script, any ``__pypackages__`` directory in the current working +directory is ignored. This is the same approach Python uses for adding the +current working directory to ``sys.path`` and ensures that it is not possible +to change the behaviour of a script by modifying files in the current +directory. + +Also, a ``__pypackages__`` directory is only recognised in the current (or +script) directory. The interpreter will *not* scan for ``__pypackages__`` in +parent directories. Doing so would open up the risk of security issues if +directory permissions on parents differ. In particular, scripts in the ``bin`` +directory or ``__pypackages__`` (the ``scripts`` location in ``sysconfig`` +terms) have no special access to the libraries installed in ``__pypackages__``. +Putting executable scripts in a ``bin`` directory is not supported by this +proposal. + +How to Teach This +================= + +The original motivation for this proposal was to make it easier to teach Python +to beginners. To that end, it needs to be easy to explain, and simple to use. + +At the most basic level, this is similar to the existing mechanism where the +script directory is added to ``sys.path`` and can be taught in a similar manner. +However, for its intended use of "lightweight isolation", it would likely be taught +in terms of "things you put in a ``__pypackages__`` directory are private to your +script". The experience of the PEP authors suggests that this would be significantly +easier to teach than the current alternative of introducing virtual environments. + + +Impact on Tools +=============== + +As the intended use of the feature is to install 3rd party libraries in the new +directory, it is important that tools, particularly installers, understand how to +manage ``__pypackages__``. + +It is hoped that tools will introduce a dedicated "pypackages" installation +mode that *is* guaranteed to match the expected layout in all cases. However, +the question of how best to support the ``__pypackages__`` layout is ultimately +left to individual tool maintainers to consider and decide on. + +Tools that locate packages without actually running Python code (IDEs, linters, +type checkers, etc.) would need updating to recognise ``__pypackages__``. In the +absence of such updates, the ``__pypackages__`` directory would work similarly +to directories currently added to ``sys.path`` at runtime (i.e., the tool would +probably ignore it). + + +Backwards Compatibility +======================= + +The directory name ``__pypackages__`` was chosen because it is unlikely to be in +common use. It is true that users who have chosen to use that name for their own +purposes will be impacted, but at the time this PEP was written, this was viewed +as a relatively low risk. + +Unfortunately, in the time this PEP has been under discussion, a number of tools +have chosen to implement variations on what is being proposed here, which are not +all compatible with the final form of the PEP. As a result, the risk of clashes is +now higher than originally anticipated. + +It would be possible to mitigate this by choosing a *different* name, hopefully as +uncommon as ``__pypackages__`` originally was. But realistically, any compatibility +issues can be viewed as simply the consequences of people trying to implement +draft proposals, without making the effort to track changes in the proposal. As such, +it seems reasonable to retain the ``__pypackages__`` name, and put the burden of +addressing the compatibility issue on the tools that implemented the draft version. + + +Impact on other Python implementations +-------------------------------------- + +Other Python implementations will need to replicate the new behavior of the +interpreter bootstrap, including locating the ``__pypackages__`` directory and +adding it the ``sys.path`` just before site packages, if it is present. This is +no different to any other Python change. + + +Reference Implementation +======================== + +`Here `_ is a small script which will +enable the implementation for ``Cpython`` & in ``PyPy``. + + +Rejected Ideas +============== + +* Alternative names, such as ``__pylocal__`` and ``python_modules``. Ultimately, the name is arbitrary and the chosen name is good enough. + +* Additional features of virtual environments. This proposal is not a replacement for virtual environments, and such features are therefore out of scope. + +* We will not scan any parent directory to find ``__pypackages__``. If we want to execute scripts inside of the ``~/bin/`` directory, then the ``__pypackages__`` directory must be inside of the ``~/bin/`` directory. Doing any such scan for ``__pypackages__`` (for the interpreter or a script) will have security implications and also increase startup time. + +* Raise an error if unexpected files or directories are present in ``__pypackages__``. This is considered too strict, particularly as transitional approaches like ``pip install --prefix`` can create additional files in ``__pypackages__``. + +* Using a different ``sysconfig`` scheme, or a dedicated ``pypackages`` scheme. While this is attractive in theory, it makes transition harder, as there will be no readily-available way of installing to ``__pypackages__`` until tools implement explicit support. And while the PEP authors hope and assume that such support would be added, having the proposal dependent on such support in order to be usable seems like an unacceptable risk. + +Copyright +========= + +This document has been placed in the public domain. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 80 + coding: utf-8 + End: diff --git a/pep-0583.rst b/peps/pep-0583.rst similarity index 98% rename from pep-0583.rst rename to peps/pep-0583.rst index 99dc2b8af..3a5574ffb 100644 --- a/pep-0583.rst +++ b/peps/pep-0583.rst @@ -223,7 +223,7 @@ interface would specify that puts happen before gets, and we'd reason directly from that. -.. _hazards: +.. _PEP 583 hazards: Surprising behaviors with races =============================== @@ -657,10 +657,11 @@ than ordinary python code. If we want to get those tiers back, we could: Sequential Consistency ---------------------- -We could just adopt sequential consistency for Python. This avoids -all of the hazards_ mentioned above, but it prohibits lots of -optimizations too. As far as I know, this is the current model of -CPython, but if CPython learned to optimize out some variable reads, +We could just adopt sequential consistency for Python. +This avoids all of the `hazards `_ mentioned above, +but it prohibits lots of optimizations too. +As far as I know, this is the current model of CPython, +but if CPython learned to optimize out some variable reads, it would lose this property. If we adopt this, Jython's ``dict`` implementation may no longer be @@ -827,15 +828,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: - diff --git a/pep-0584.rst b/peps/pep-0584.rst similarity index 99% rename from pep-0584.rst rename to peps/pep-0584.rst index bd15bae58..d0a002a77 100644 --- a/pep-0584.rst +++ b/peps/pep-0584.rst @@ -987,7 +987,7 @@ The above examples show that sometimes the ``|`` operator leads to a clear increase in readability, reducing the number of lines of code and improving clarity. However other examples using the ``|`` operator lead to long, complex single expressions, possibly well over -the PEP 8 maximum line length of 80 columns. As with any other +the :pep:`8` maximum line length of 80 columns. As with any other language feature, the programmer should use their own judgement about whether ``|`` improves their code. diff --git a/pep-0585.rst b/peps/pep-0585.rst similarity index 97% rename from pep-0585.rst rename to peps/pep-0585.rst index 2b1b08158..efd78761d 100644 --- a/pep-0585.rst +++ b/peps/pep-0585.rst @@ -3,9 +3,10 @@ Title: Type Hinting Generics In Standard Collections Version: $Revision$ Last-Modified: $Date$ Author: Ɓukasz Langa -Discussions-To: Typing-Sig -Status: Accepted +Discussions-To: typing-sig@python.org +Status: Final Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 03-Mar-2019 Python-Version: 3.9 @@ -68,7 +69,7 @@ collections directly. Example:: def find(haystack: dict[str, list[int]]) -> int: ... -Usefulness of this syntax before PEP 585 is limited as external tooling +Usefulness of this syntax before :pep:`585` is limited as external tooling like Mypy does not recognize standard collections as generic. Moreover, certain features of typing like type aliases or casting require putting types outside of annotations, in runtime context. While these are @@ -116,16 +117,16 @@ Python 3.9, the following collections become generic using * ``re.Pattern`` # typing.Pattern, typing.re.Pattern * ``re.Match`` # typing.Match, typing.re.Match -Importing those from ``typing`` is deprecated. Due to PEP 563 and the +Importing those from ``typing`` is deprecated. Due to :pep:`563` and the intention to minimize the runtime impact of typing, this deprecation will not generate DeprecationWarnings. Instead, type checkers may warn about such deprecated usage when the target version of the checked program is signalled to be Python 3.9 or newer. It's recommended to allow for those warnings to be silenced on a project-wide basis. -The deprecated functionality will be removed from the ``typing`` module -in the first Python version released 5 years after the release of -Python 3.9.0. +The deprecated functionality may eventually be removed from the ``typing`` +module. Removal will occur no sooner than Python 3.9's end of life, +scheduled for October 2025. Parameters to generics are available at runtime diff --git a/pep-0586.rst b/peps/pep-0586.rst similarity index 97% rename from pep-0586.rst rename to peps/pep-0586.rst index 564b1c184..a3f9fba6c 100644 --- a/pep-0586.rst +++ b/peps/pep-0586.rst @@ -2,9 +2,10 @@ PEP: 586 Title: Literal Types Author: Michael Lee , Ivan Levkivskyi , Jukka Lehtosalo BDFL-Delegate: Guido van Rossum -Discussions-To: Typing-Sig +Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 14-Mar-2019 Python-Version: 3.8 @@ -15,7 +16,7 @@ Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/ Abstract ======== -This PEP proposes adding *Literal types* to the PEP 484 ecosystem. +This PEP proposes adding *Literal types* to the :pep:`484` ecosystem. Literal types indicate that some expression has literally a specific value. For example, the following function will accept only expressions that have literally the value "4":: @@ -55,7 +56,7 @@ The typing issue tracker contains some `additional examples and discussion `_. There is currently no way of expressing the type signatures of these -functions: PEP 484 does not include any mechanism for writing signatures +functions: :pep:`484` does not include any mechanism for writing signatures where the return type varies depending on the value passed in. Note that this problem persists even if we redesign these APIs to instead accept enums: ``MyEnum.FOO`` and ``MyEnum.BAR`` are both @@ -94,7 +95,7 @@ it’s safe to do things like ``foo + 5`` since ``foo`` inherits int’s ``__add__`` method. The resulting type of ``foo + 5`` is ``int``. This "inheriting" behavior is identical to how we -`handle NewTypes. `_. +:pep:`handle NewTypes <484#newtype-helper-function>`. Equivalence of two Literals --------------------------- @@ -223,7 +224,7 @@ The following parameters are intentionally disallowed by design: ``Literal["foo".replace("o", "b")]``. - Rationale: Literal types are meant to be a - minimal extension to the PEP 484 typing ecosystem and requiring type + minimal extension to the :pep:`484` typing ecosystem and requiring type checkers to interpret potentially expressions inside types adds too much complexity. Also see `Rejected or out-of-scope ideas`_. @@ -527,7 +528,8 @@ values of the ``Status`` enum have already been exhausted:: print("Got custom status: " + s) The interaction described above is not new: it's already -`already codified within PEP 484 `_. However, many type +:pep:`codified within PEP 484 <484#support-for-singleton-types-in-unions>`. +However, many type checkers (such as mypy) do not yet implement this due to the expected complexity of the implementation work. @@ -585,7 +587,7 @@ involving Literal bools. For example, we can combine ``Literal[True]``, Interactions with Final ----------------------- -`PEP 591 `_ proposes adding a "Final" qualifier to the typing +:pep:`591` proposes adding a "Final" qualifier to the typing ecosystem. This qualifier can be used to declare that some variable or attribute cannot be reassigned:: @@ -603,7 +605,7 @@ is valid to use in any context that expects a ``Literal[3]``:: The ``Final`` qualifier serves as a shorthand for declaring that a variable is *effectively Literal*. -If both this PEP and PEP 591 are accepted, type checkers are expected to +If both this PEP and :pep:`591` are accepted, type checkers are expected to support this shortcut. Specifically, given a variable or attribute assignment of the form ``var: Final = value`` where ``value`` is a valid parameter for ``Literal[...]``, type checkers should understand that ``var`` may be used in @@ -629,7 +631,7 @@ True dependent types/integer generics ------------------------------------- This proposal is essentially describing adding a very simplified -dependent type system to the PEP 484 ecosystem. One obvious extension +dependent type system to the :pep:`484` ecosystem. One obvious extension would be to implement a full-fledged dependent type system that lets users predicate types based on their values in arbitrary ways. That would let us write signatures like the below:: @@ -734,11 +736,6 @@ something similar to how .. _typescript-index-types: https://www.typescriptlang.org/docs/handbook/advanced-types.html#index-types -.. _newtypes: https://www.python.org/dev/peps/pep-0484/#newtype-helper-function - -.. _pep-484-enums: https://www.python.org/dev/peps/pep-0484/#support-for-singleton-types-in-unions - -.. _pep-591: https://www.python.org/dev/peps/pep-0591/ Acknowledgements ================ diff --git a/pep-0587.rst b/peps/pep-0587.rst similarity index 98% rename from pep-0587.rst rename to peps/pep-0587.rst index 430c8f11a..134d1248d 100644 --- a/pep-0587.rst +++ b/peps/pep-0587.rst @@ -32,7 +32,7 @@ easier using the new ``Py_RunMain()`` function. Moreover, using the same way the regular Python parses command line arguments, and ``PyConfig.xoptions`` are handled as ``-X opt`` command line options. -This extracts a subset of the API design from the PEP 432 development +This extracts a subset of the API design from the :pep:`432` development and refactoring work that is now considered sufficiently stable to make public (allowing 3rd party embedding applications access to the same configuration APIs that the native CPython CLI is now using). @@ -71,7 +71,7 @@ Moreover, ``Py_Main()`` could exit directly the process rather than returning an exit code. Proposed new API reports the error or exit code to the caller which can decide how to handle it. -Implementing the PEP 540 (UTF-8 Mode) and the new ``-X dev`` correctly +Implementing the :pep:`540` (UTF-8 Mode) and the new ``-X dev`` correctly was almost impossible in Python 3.6. The code base has been deeply reworked in Python 3.7 and then in Python 3.8 to read the configuration into a structure with no side effect. It becomes possible to clear the @@ -88,9 +88,9 @@ behavior way more easily, especially for corner cases like that, and ensure that the configuration remains consistent: see `Priority and Rules`_. -This PEP is a partial implementation of PEP 432 which is the overall +This PEP is a partial implementation of :pep:`432` which is the overall design. New fields can be added later to ``PyConfig`` structure to -finish the implementation of the PEP 432 (e.g. by adding a new partial +finish the implementation of the :pep:`432` (e.g. by adding a new partial initialization API which allows to configure Python using Python objects to finish the full initialization). However, those features are omitted from this PEP as even the native CPython CLI doesn't work that way - the public API @@ -447,7 +447,7 @@ exceptions (error or exit) using ``PyStatus_Exception()`` and ``bytearray`` with ``str``, or comparing ``bytes`` with ``int``. If equal or greater to 2, raise a ``BytesWarning`` exception. * ``check_hash_pycs_mode`` (``wchar_t*``): - ``--check-hash-based-pycs`` command line option value (see PEP 552). + ``--check-hash-based-pycs`` command line option value (see :pep:`552`). Valid values: ``always``, ``never`` and ``default``. The default value is ``default``. * ``configure_c_stdio`` (``int``): @@ -585,7 +585,7 @@ Options`_. Install importlib? * ``_init_main`` (``int``): If equal to 0, stop Python initialization before the "main" phase - (see PEP 432). + (see :pep:`432`). More complete example modifying the default configuration, read the configuration, and then override some parameters:: @@ -669,7 +669,7 @@ behaves as the regular Python. Environments variables and command line arguments are used to configure Python, whereas global configuration variables are ignored. -This function enables C locale coercion (PEP 538) and UTF-8 Mode (PEP +This function enables C locale coercion (:pep:`538`) and UTF-8 Mode (PEP 540) depending on the LC_CTYPE locale, ``PYTHONUTF8`` and ``PYTHONCOERCECLOCALE`` environment variables. @@ -710,7 +710,7 @@ Example of customized Python always running in isolated mode:: } This example is a basic implementation of the "System Python Executable" -discussed in PEP 432. +discussed in :pep:`432`. Path Configuration @@ -801,7 +801,7 @@ Multi-Phase Initialization Private Provisional API -------------------------------------------------- This section is a private provisional API introducing multi-phase -initialization, the core feature of the PEP 432: +initialization, the core feature of the :pep:`432`: * "Core" initialization phase, "bare minimum Python": @@ -890,7 +890,7 @@ This PEP only adds a new API: it leaves the existing API unchanged and has no impact on the backwards compatibility. The Python 3.7 ``Py_Initialize()`` function now disable the C locale -coercion (PEP 538) and the UTF-8 Mode (PEP 540) by default to prevent +coercion (:pep:`538`) and the UTF-8 Mode (:pep:`540`) by default to prevent mojibake. The new API using the `Python Configuration`_ is needed to enable them automatically. @@ -1528,7 +1528,7 @@ Version History Acceptance ========== -PEP 587 was `accepted by Thomas Wouters on May 26, 2019 +:pep:`587` was `accepted by Thomas Wouters on May 26, 2019 `_. diff --git a/pep-0588.rst b/peps/pep-0588.rst similarity index 97% rename from pep-0588.rst rename to peps/pep-0588.rst index fac37f52c..20e93b23c 100644 --- a/pep-0588.rst +++ b/peps/pep-0588.rst @@ -2,7 +2,7 @@ PEP: 588 Title: GitHub Issues Migration Plan Author: Mariatta BDFL-Delegate: Barry Warsaw -Discussions-To: Core-Workflow Category on Discourse +Discussions-To: https://discuss.python.org/t/13791 Status: Draft Type: Informational Content-Type: text/x-rst @@ -13,8 +13,8 @@ Abstract ======== This PEP describes the detailed plan for migrating from Python's issue -tracker on Roundup to GitHub issues. See PEP 581 for rationale and -background. PEP 588 also describes the detailed timeline for the +tracker on Roundup to GitHub issues. See :pep:`581` for rationale and +background. :pep:`588` also describes the detailed timeline for the migration. @@ -45,7 +45,7 @@ core-workflow [#]_. We're using GitHub's Migrations API [#]_ to download GitHub data for CPython on a daily basis. The archives will be dropped in a S3 bucket. -Thanks to Ee W. Durbin III for working on this. +Thanks to Ee Durbin for working on this. Update the CLA host diff --git a/pep-0589.rst b/peps/pep-0589.rst similarity index 94% rename from pep-0589.rst rename to peps/pep-0589.rst index dd7550e31..913f9bb49 100644 --- a/pep-0589.rst +++ b/peps/pep-0589.rst @@ -6,6 +6,7 @@ BDFL-Delegate: Guido van Rossum Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 20-Mar-2019 Python-Version: 3.8 @@ -16,7 +17,7 @@ Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/ Abstract ======== -PEP 484 [#PEP-484]_ defines the type ``Dict[K, V]`` for uniform +:pep:`484` defines the type ``Dict[K, V]`` for uniform dictionaries, where each value has the same type, and arbitrary key values are supported. It doesn't properly support the common pattern where the type of a dictionary value depends on the string value of @@ -24,7 +25,7 @@ the key. This PEP proposes a type constructor ``typing.TypedDict`` to support the use case where a dictionary object has a specific set of string keys, each with a value of a specific type. -Here is an example where PEP 484 doesn't allow us to annotate +Here is an example where :pep:`484` doesn't allow us to annotate satisfactorily:: movie = {'name': 'Blade Runner', @@ -63,10 +64,10 @@ when not using JSON. They trivially support various useful operations with no extra effort, including pretty-printing (through ``str()`` and the ``pprint`` module), iteration, and equality comparisons. -PEP 484 doesn't properly support the use cases mentioned above. Let's +:pep:`484` doesn't properly support the use cases mentioned above. Let's consider a dictionary object that has exactly two valid string keys, ``'name'`` with value type ``str``, and ``'year'`` with value type -``int``. The PEP 484 type ``Dict[str, Any]`` would be suitable, but +``int``. The :pep:`484` type ``Dict[str, Any]`` would be suitable, but it is too lenient, as arbitrary string keys can be used, and arbitrary values are valid. Similarly, ``Dict[str, Union[str, int]]`` is too general, as the value for key ``'name'`` could be an ``int``, and @@ -95,7 +96,7 @@ This PEP proposes two ways of defining TypedDict types. The first uses a class-based syntax. The second is an alternative assignment-based syntax that is provided for backwards compatibility, to allow the feature to be backported to older Python versions. The -rationale is similar to why PEP 484 supports a comment-based +rationale is similar to why :pep:`484` supports a comment-based annotation syntax for Python 2.7: type hinting is particularly useful for large existing codebases, and these often need to run on older Python versions. The two syntax options parallel the syntax variants @@ -105,7 +106,7 @@ required or not). This PEP also provides a sketch of how a type checker is expected to support type checking operations involving TypedDict objects. -Similar to PEP 484, this discussion is left somewhat vague on purpose, +Similar to :pep:`484`, this discussion is left somewhat vague on purpose, to allow experimentation with a wide variety of different type checking approaches. In particular, type compatibility should be based on structural compatibility: a more specific TypedDict type can @@ -224,7 +225,7 @@ are the only uses of the type a type checker is expected to allow: In particular, TypedDict type objects cannot be used in ``isinstance()`` tests such as ``isinstance(d, Movie)``. The reason is that there is no existing support for checking types of dictionary -item values, since ``isinstance()`` does not work with many PEP 484 +item values, since ``isinstance()`` does not work with many :pep:`484` types, including common ones like ``List[str]``. This would be needed for cases like this:: @@ -336,7 +337,7 @@ Alternative Syntax This PEP also proposes an alternative syntax that can be backported to older Python versions such as 3.5 and 2.7 that don't support the -variable definition syntax introduced in PEP 526 [#PEP-526]_. It +variable definition syntax introduced in :pep:`526`. It resembles the traditional syntax for defining named tuples:: Movie = TypedDict('Movie', {'name': str, 'year': int}) @@ -364,7 +365,7 @@ Type Consistency Informally speaking, *type consistency* is a generalization of the is-subtype-of relation to support the ``Any`` type. It is defined -more formally in PEP 483 [#PEP-483]_. This section introduces the +more formally in :pep:`483`. This section introduces the new, non-trivial rules needed to support type consistency for TypedDict types. @@ -436,7 +437,7 @@ Discussion: y: str def f(a: A) -> None: - a[y] = 1 + a['y'] = 1 def g(b: B) -> None: f(b) # Type check error: 'B' incompatible with 'A' @@ -566,7 +567,7 @@ alternative is to generate false positive errors for idiomatic code. Use of Final Values and Literal Types ------------------------------------- -Type checkers should allow final names (PEP 591 [#PEP-591]_) with +Type checkers should allow final names (:pep:`591`) with string values to be used instead of string literals in operations on TypedDict objects. For example, this is valid:: @@ -576,7 +577,7 @@ TypedDict objects. For example, this is valid:: years_since_epoch = m[YEAR] - 1970 Similarly, an expression with a suitable literal type -(PEP 586 [#PEP-586]_) can be used instead of a literal value:: +(:pep:`586`) can be used instead of a literal value:: def get_value(movie: Movie, key: Literal['year', 'name']) -> Union[int, str]: @@ -658,7 +659,7 @@ extensions to be added in the future: * TypedDict can't be used for specifying the type of a ``**kwargs`` argument. This would allow restricting the allowed keyword - arguments and their types. According to PEP 484, using a TypedDict + arguments and their types. According to :pep:`484`, using a TypedDict type as the type of ``**kwargs`` means that the TypedDict is valid as the *value* of arbitrary keyword arguments, but it doesn't restrict which keyword arguments should be allowed. The syntax @@ -677,25 +678,9 @@ Michael Lee, Dominik Miedzinski, Roy Williams and Max Moroz. References ========== -.. [#PEP-484] PEP 484, Type Hints, van Rossum, Lehtosalo, Langa - (http://www.python.org/dev/peps/pep-0484) - .. [#dataclasses-json] Dataclasses JSON (https://github.com/lidatong/dataclasses-json) -.. [#PEP-526] PEP 526, Syntax for Variable Annotations, Gonzalez, - House, Levkivskyi, Roach, van Rossum - (http://www.python.org/dev/peps/pep-0484) - -.. [#PEP-483] PEP 483, The Theory of Type Hints, van Rossum, Levkivskyi - (http://www.python.org/dev/peps/pep-0483) - -.. [#PEP-591] PEP 591, Adding a final qualifier to typing, Sullivan, - Levkivskyi (http://www.python.org/dev/peps/pep-0591) - -.. [#PEP-586] PEP 586, Literal Types, Lee, Levkivskyi, Lehtosalo - (http://www.python.org/dev/peps/pep-0586) - .. [#mypy] http://www.mypy-lang.org/ .. [#typing_extensions] diff --git a/pep-0590.rst b/peps/pep-0590.rst similarity index 95% rename from pep-0590.rst rename to peps/pep-0590.rst index fabd4df7b..a3fa010fa 100644 --- a/pep-0590.rst +++ b/peps/pep-0590.rst @@ -250,7 +250,7 @@ Performance of functions using ``METH_VARARGS`` will become slightly worse. Stable ABI ========== -Nothing from this PEP is added to the stable ABI (PEP 384). +Nothing from this PEP is added to the stable ABI (:pep:`384`). Alternative Suggestions @@ -259,7 +259,7 @@ Alternative Suggestions bpo-29259 --------- -PEP 590 is close to what was proposed in bpo-29259 [#bpo29259]_. +:pep:`590` is close to what was proposed in bpo-29259 [#bpo29259]_. The main difference is that this PEP stores the function pointer in the instance rather than in the class. This makes more sense for implementing functions in C, @@ -269,15 +269,15 @@ It also allows optimizing ``type.__call__``, which is not possible with bpo-2925 PEP 576 and PEP 580 ------------------- -Both PEP 576 and PEP 580 are designed to enable 3rd party objects to be both expressive and performant (on a par with +Both :pep:`576` and :pep:`580` are designed to enable 3rd party objects to be both expressive and performant (on a par with CPython objects). The purpose of this PEP is provide a uniform way to call objects in the CPython ecosystem that is both expressive and as performant as possible. -This PEP is broader in scope than PEP 576 and uses variable rather than fixed offset function-pointers. -The underlying calling convention is similar. Because PEP 576 only allows a fixed offset for the function pointer, +This PEP is broader in scope than :pep:`576` and uses variable rather than fixed offset function-pointers. +The underlying calling convention is similar. Because :pep:`576` only allows a fixed offset for the function pointer, it would not allow the improvements to any objects with constraints on their layout. -PEP 580 proposes a major change to the ``PyMethodDef`` protocol used to define builtin functions. +:pep:`580` proposes a major change to the ``PyMethodDef`` protocol used to define builtin functions. This PEP provides a more general and simpler mechanism in the form of a new calling convention. This PEP also extends the ``PyMethodDef`` protocol, but merely to formalise existing conventions. @@ -308,7 +308,7 @@ References .. [2] tp_call/PyObject_Call calling convention https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_call .. [3] Using PY_VECTORCALL_ARGUMENTS_OFFSET in callee - https://github.com/markshannon/cpython/blob/vectorcall-minimal/Objects/classobject.c#L53 + https://github.com/markshannon/cpython/blob/815cc1a30d85cdf2e3d77d21224db7055a1f07cb/Objects/classobject.c#L53 .. [4] Argument Clinic https://docs.python.org/3/howto/clinic.html diff --git a/pep-0591.rst b/peps/pep-0591.rst similarity index 95% rename from pep-0591.rst rename to peps/pep-0591.rst index 6b6bacffc..60ad02071 100644 --- a/pep-0591.rst +++ b/peps/pep-0591.rst @@ -5,6 +5,7 @@ BDFL-Delegate: Guido van Rossum Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 15-Mar-2019 Python-Version: 3.8 @@ -62,7 +63,7 @@ situations: * Allowing a name to be used in situations where ordinarily a literal is expected (for example as a field name for ``NamedTuple``, a tuple of types passed to ``isinstance``, or an argument to a function - with arguments of ``Literal`` type [#PEP-586]_). + with arguments of ``Literal`` type (:pep:`586`)). Specification ============= @@ -279,16 +280,6 @@ desired is to use this idiom (possibly in a support module):: References ========== -.. [#PEP-484] PEP 484, Type Hints, van Rossum, Lehtosalo, Langa - (http://www.python.org/dev/peps/pep-0484) - -.. [#PEP-526] PEP 526, Syntax for Variable Annotations, Gonzalez, - House, Levkivskyi, Roach, van Rossum - (http://www.python.org/dev/peps/pep-0526) - -.. [#PEP-586] PEP 586, Literal Types, Lee, Levkivskyi, Lehtosalo - (http://www.python.org/dev/peps/pep-0586) - .. [#mypy] http://www.mypy-lang.org/ .. [#typing_extensions] https://github.com/python/typing/tree/master/typing_extensions diff --git a/pep-0592.rst b/peps/pep-0592.rst similarity index 98% rename from pep-0592.rst rename to peps/pep-0592.rst index 961b6d7c6..298720e6a 100644 --- a/pep-0592.rst +++ b/peps/pep-0592.rst @@ -2,9 +2,10 @@ PEP: 592 Title: Adding "Yank" Support to the Simple API Author: Donald Stufft BDFL-Delegate: Paul Moore -Discussions-To: https://discuss.python.org/c/packaging +Discussions-To: https://discuss.python.org/t/1629 Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 07-May-2019 Resolution: https://discuss.python.org/t/pep-592-support-for-yanked-files-in-the-simple-repository-api/1629/30 @@ -108,7 +109,7 @@ suggested approaches to take: matches a version specifier that "pins" to an exact version using either ``==`` (without any modifiers that make it a range, such as ``.*``) or ``===``. Matching this version specifier should otherwise - be done as per PEP 440 for things like local versions, zero padding, + be done as per :pep:`440` for things like local versions, zero padding, etc. 2. Yanked files are always ignored, unless they are the only file that matches what a lock file (such as ``Pipfile.lock`` or ``poetry.lock``) diff --git a/pep-0593.rst b/peps/pep-0593.rst similarity index 89% rename from pep-0593.rst rename to peps/pep-0593.rst index 2a8da4bc1..6522ba2b1 100644 --- a/pep-0593.rst +++ b/peps/pep-0593.rst @@ -5,6 +5,7 @@ Sponsor: Ivan Levkivskyi Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 26-Apr-2019 Python-Version: 3.9 @@ -19,12 +20,12 @@ This PEP introduces a mechanism to extend the type annotations from PEP Motivation ---------- -PEP 484 provides a standard semantic for the annotations introduced in -PEP 3107. PEP 484 is prescriptive but it is the de facto standard +:pep:`484` provides a standard semantic for the annotations introduced in +:pep:`3107`. :pep:`484` is prescriptive but it is the de facto standard for most of the consumers of annotations; in many statically checked code bases, where type annotations are widely used, they have effectively crowded out any other form of annotation. Some of the use -cases for annotations described in PEP 3107 (database mapping, +cases for annotations described in :pep:`3107` (database mapping, foreign languages bridge) are not currently realistic given the prevalence of type annotations. Furthermore, the standardisation of type annotations rules out advanced features only supported by specific type @@ -43,13 +44,14 @@ should ignore it and simply treat the type as ``T``. Unlike the ``no_type_check`` functionality that currently exists in the ``typing`` module which completely disables typechecking annotations on a function or a class, the ``Annotated`` type allows for both static typechecking -of ``T`` (e.g., via mypy [mypy]_ or Pyre [pyre]_, which can safely ignore ``x``) +of ``T`` (e.g., via `mypy `_ or `Pyre `_, +which can safely ignore ``x``) together with runtime access to ``x`` within a specific application. The introduction of this type would address a diverse set of use cases of interest to the broader Python community. -This was originally brought up as issue 600 [issue-600]_ in the typing github -and then discussed in Python ideas [python-ideas]_. +This was originally brought up as `issue 600 `_ in the typing github +and then discussed in `Python ideas `_. Motivating examples ------------------- @@ -81,21 +83,21 @@ Lowering barriers to developing new typing constructs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Typically when adding a new type, a developer need to upstream that type to the -typing module and change mypy, PyCharm [pycharm]_, Pyre, pytype [pytype]_, +typing module and change mypy, `PyCharm `_, Pyre, `pytype `_, etc... This is particularly important when working on open-source code that makes use of these types, seeing as the code would not be immediately transportable to other developers' tools without additional logic. As a result, there is a high cost to developing and trying out new types in a codebase. Ideally, authors should be able to introduce new types in a manner that allows -for graceful degradation (e.g.: when clients do not have a custom mypy plugin -[mypy-plugin]_), which would lower the barrier to development and ensure some +for graceful degradation (e.g.: when clients do not have a custom `mypy plugin +`_), which would lower the barrier to development and ensure some degree of backward compatibility. -For example, suppose that an author wanted to add support for tagged unions -[tagged-union]_ to Python. One way to accomplish would be to annotate -``TypedDict`` [typed-dict]_ in Python such that only one field is allowed to be -set:: +For example, suppose that an author wanted to add support for `tagged unions +`_ to Python. One way to accomplish would be to `annotate +`_ ``TypedDict`` in Python such that only one field is allowed +to be set:: Currency = Annotated[ TypedDict('Currency', {'dollars': float, 'pounds': float}, total=False), @@ -223,7 +225,7 @@ don't expect clients to have to write lots of boilerplate code:: T = TypeVar('T') Const = Annotated[T, my_annotations.CONST] - Class C: + class C: def const_method(self: Const[List[int]]) -> int: ... @@ -236,7 +238,7 @@ cause ``Annotated`` to not integrate cleanly with the other typing annotations: * ``Annotated`` cannot infer the decorated type. You could imagine that ``Annotated[..., Immutable]`` could be used to mark a value as immutable while still inferring its type. Typing does not support using the - inferred type anywhere else [issue-276]_; it's best to not add this as a + inferred type `anywhere else `_; it's best to not add this as a special case. * Using ``(Type, Ann1, Ann2, ...)`` instead of @@ -253,40 +255,34 @@ This feature was left out to keep the design simple: little benefit. -References ----------- - -.. [issue-600] +.. _issue-600: https://github.com/python/typing/issues/600 -.. [python-ideas] +.. _python-ideas: https://mail.python.org/pipermail/python-ideas/2019-January/054908.html -.. [struct-doc] - https://docs.python.org/3/library/struct.html#examples +.. _mypy: + http://www.mypy-lang.org/ -.. [mypy] - http://www.mypy-lang.org/ - -.. [pyre] +.. _pyre: https://pyre-check.org/ -.. [pycharm] +.. _pycharm: https://www.jetbrains.com/pycharm/ -.. [pytype] +.. _pytype: https://github.com/google/pytype -.. [mypy-plugin] +.. _mypy-plugin: https://github.com/python/mypy_extensions -.. [tagged-union] +.. _tagged-union: https://en.wikipedia.org/wiki/Tagged_union -.. [typed-dict] +.. _typed-dict: https://mypy.readthedocs.io/en/latest/more_types.html#typeddict -.. [issue-276] +.. _issue-276: https://github.com/python/typing/issues/276 Copyright diff --git a/pep-0594.rst b/peps/pep-0594.rst similarity index 57% rename from pep-0594.rst rename to peps/pep-0594.rst index 3cb41288b..1a89458c4 100644 --- a/pep-0594.rst +++ b/peps/pep-0594.rst @@ -1,12 +1,15 @@ PEP: 594 Title: Removing dead batteries from the standard library -Author: Christian Heimes -Discussions-To: https://discuss.python.org/t/pep-594-removing-dead-batteries-from-the-standard-library/1704 -Status: Draft +Author: Christian Heimes , + Brett Cannon +Discussions-To: https://discuss.python.org/t/13508 +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 20-May-2019 -Post-History: 21-May-2019 +Python-Version: 3.11 +Post-History: 21-May-2019, 04-Feb-2022 +Resolution: https://discuss.python.org/t/13508/22 Abstract @@ -18,7 +21,7 @@ and SUN file formats), APIs and operating systems that have been superseded a long time ago (e.g. Mac OS 9), or modules that have security implications and better alternatives (e.g. password and login). -The PEP follows in the foot steps of other PEPS like :pep:`3108`. The +The PEP follows in the footsteps of other PEPS like :pep:`3108`. The *Standard Library Reorganization* proposal removed a bunch of modules from Python 3.0. In 2007, the PEP referred to maintenance burden as: @@ -29,7 +32,7 @@ Python 3.0. In 2007, the PEP referred to maintenance burden as: up an undue amount of time and effort." The withdrawn :pep:`206` from 2000 expresses issues with the Python standard -library unvarnished and fortright: +library in an unvarnished and forthright manner: "[...] the standard library modules aren't always the best choices for a job. Some library modules were quick hacks (e.g. ``calendar``, @@ -47,13 +50,13 @@ philosophy and was one of the cornerstones to Python's success story. Users didn't have to figure out how to download and install separate packages in order to write a simple web server or parse email. -Times have changed. The introduction of the cheese shop (PyPI), setuptools, +Times have changed. With the introduction of PyPI (nĂ©e Cheeseshop), setuptools, and later pip, it became simple and straightforward to download and install packages. Nowadays Python has a rich and vibrant ecosystem of third-party packages. It's pretty much standard to either install packages from PyPI or use one of the many Python or Linux distributions. -On the other hand, Python's standard library is piling up cruft, unnecessary +On the other hand, Python's standard library is piling up with cruft, unnecessary duplication of functionality, and dispensable features. This is undesirable for several reasons. @@ -71,7 +74,7 @@ for several reasons. Micro:bit). Python on mobile platforms like BeeWare or WebAssembly (e.g. pyodide) also benefit from reduced download size. -The modules in the PEP have been selected for deprecation because their +The modules in this PEP have been selected for deprecation because their removal is either least controversial or most beneficial. For example, least controversial are 30-year-old multimedia formats like the ``sunau`` audio format, which was used on SPARC and NeXT workstations in the late @@ -91,7 +94,7 @@ code is not permitted without legal approval. * The ``optparse`` and ``getopt`` modules are widely used. They are mature modules with very low maintenance overhead. * According to David Beazley [5]_ the ``wave`` module is easy to teach to - kids and can make crazy sounds. Making a computer generate crazy sounds is + kids and can make crazy sounds. Making a computer generate sounds is a powerful and highly motivating exercise for a nine-year-old aspiring developer. It's a fun battery to keep. @@ -99,59 +102,25 @@ code is not permitted without legal approval. Deprecation schedule ==================== -3.8 ---- - -This PEP targets Python 3.8. Version 3.8.0 final is scheduled to be released -a few months before Python 2.7 will reach its end of lifetime. We expect that -Python 3.8 will be targeted by users that migrate to Python 3 in 2019 and -2020. To reduce churn and to allow a smooth transition from Python 2, -Python 3.8 will neither raise ``DeprecationWarning`` nor remove any -modules that have been scheduled for removal. Instead deprecated modules will -just be *documented* as deprecated. Optionally modules may emit a -``PendingDeprecationWarning``. - -All deprecated modules will also undergo a feature freeze. No additional -features should be added *unless* python-dev agrees that the deprecation of -the module is reverted and the code will not be removed. Bug should still be -fixed. - -3.9 ---- - -Starting with Python 3.9, deprecated modules will start issuing -``DeprecationWarning``. The `parser`_ module is removed and potentially -replaced with a new module. - -All other deprecated modules are fully supported and will receive security -updates until Python 3.9 reaches its end-of-life. Python 3.9.0 will -be released about 18 months after 3.8.0 (April 2021?) and most likely -be supported for five years after the release. The estimated EOL of Python 3.9 -is in 2026. - -3.10 +3.11 ---- -In 3.10 all deprecated modules will be removed from the CPython repository -together with tests, documentation, and autoconf rules. +Starting with Python 3.11, deprecated modules will start issuing +``DeprecationWarning``. The estimated EOL of Python 3.10, the last +version without the warning, is October 2026. +3.12 +---- -PEP acceptance process -====================== +There should be no specific change compared to Python 3.11. +This is the last version of Python with the deprecated modules, +with an estimated EOL of October 2028. -3.8.0b1 is scheduled to be release shortly after the PEP is officially -submitted. Since it's improbable that the PEP will pass all stages of the -PEP process in time, I propose a two-step acceptance process that is -analogous to Python's two-release deprecation process. +3.13 +---- -The first *provisionally-accepted* phase targets Python 3.8.0b1. In the first -phase no code is changed or removed. Modules are only documented as -deprecated. The only exception is the `parser`_ module. It has been -documented as deprecated since Python 2.5 and is scheduled for removal for -3.9 to make place for a more advanced parser. - -The final decision, which modules will be removed and how the removed code -is preserved, can be delayed for another year. +All modules deprecated by this PEP are removed from the ``main`` branch +of the CPython repository and are no longer distributed as part of Python. Deprecated modules @@ -167,39 +136,34 @@ audio processing. :header: "Module", "Deprecated in", "To be removed", "Added in", "Has maintainer?", "Replacement" :widths: 2, 1, 1, 1, 1, 2 - aifc,3.8 (3.0\*),3.10,1993,**yes (inactive)**,\- - asynchat,**3.6** (3.0\*),3.10,1999,**yes**,asyncio_ - asyncore,**3.6** (3.0\*),3.10,1999,**yes**,asyncio_ - audioop,3.8 (3.0\*),3.10,1992,**yes**,\- - binhex,3.8,3.10,1995,no,\- - cgi,3.8 (2.0\*\*),3.10,1995,no,\- - cgitb,3.8 (2.0\*\*),3.10,1995,no,\- - chunk,3.8,3.10,1999,no,\- - crypt,3.8,3.10,1994,**yes (inactive)**,"legacycrypt_, bcrypt_, argon2cffi_, hashlib_, passlib_" - formatter,**3.4**,3.10,1995,no,\- - fpectl,**3.7**,**3.7**,1997,n/a,\- - imghdr,3.8,3.10,1992,no,"filetype_, puremagic_, python-magic_" - imp,**3.4**,3.10,1990/1995,no,importlib_ - macpath,**3.7**,**3.8**,1990,n/a,\- - msilib,3.8,3.10,2006,no,\- - nntplib,3.8,3.10,1992,no,\- - nis,3.8 (3.0\*),3.10,1992,no,\- - ossaudiodev,3.8,3.10,2002,no,\- - parser,**2.5**,**3.9**,1993,**yes**,"ast_, astroid_, lib2to3.pgen2" - pipes,3.8,3.10,1992,no,"subprocess_" - smtpd,"**3.4.7**, **3.5.4**",3.10,2001,**yes**,"aiosmtpd_" - sndhdr,3.8,3.10,1994,no,"filetype_, puremagic_, python-magic_" - spwd,3.8,3.10,2005,no,"python-pam_, simplepam_" - sunau,3.8 (3.0\*),3.10,1993,no,\- - telnetlib,3.8 (3.0\*),3.10,1997,no,"telnetlib3_, Exscript_" - uu,3.8,3.10,1994,no,\- - xdrlib,3.8,3.10,1992/1996,no,\- + aifc,3.11 (3.0\*),3.13,1993,**yes (inactive)**,\- + asynchat,**3.6** (3.0\*),3.12,1999,**yes**,asyncio_ + asyncore,**3.6** (3.0\*),3.12,1999,**yes**,asyncio_ + audioop,3.11 (3.0\*),3.13,1992,**yes**,\- + cgi,3.11 (2.0\*\*),3.13,1995,no,\- + cgitb,3.11 (2.0\*\*),3.13,1995,no,\- + chunk,3.11,3.13,1999,no,\- + crypt,3.11,3.13,1994,**yes (inactive)**,"legacycrypt_, bcrypt_, argon2-cffi_, hashlib_, passlib_" + imghdr,3.11,3.13,1992,no,"filetype_, puremagic_, python-magic_" + mailcap,3.11,3.13,1995,no,\- + msilib,3.11,3.13,2006,no,\- + nntplib,3.11,3.13,1992,no,\- + nis,3.11 (3.0\*),3.13,1992,no,\- + ossaudiodev,3.11,3.13,2002,no,\- + pipes,3.11,3.13,1992,no,"subprocess_" + smtpd,"**3.4.7**, **3.5.4**",3.12,2001,**yes**,"aiosmtpd_" + sndhdr,3.11,3.13,1994,no,"filetype_, puremagic_, python-magic_" + spwd,3.11,3.13,2005,no,"python-pam_" + sunau,3.11 (3.0\*),3.13,1993,no,\- + telnetlib,3.11 (3.0\*),3.13,1997,no,"telnetlib3_, Exscript_" + uu,3.11,3.13,1994,no,\- + xdrlib,3.11,3.13,1992/1996,no,\- .. _aiosmtpd: https://pypi.org/project/aiosmtpd/ -.. _argon2cffi: https://pypi.org/project/argon2-cffi/ +.. _argon2-cffi: https://pypi.org/project/argon2-cffi/ .. _ast: https://docs.python.org/3/library/ast.html .. _astroid: https://pypi.org/project/astroid/ -.. _asyncio: https://docs.python.org/3/library/hashlib.html +.. _asyncio: https://docs.python.org/3/library/asyncio.html .. _bcrypt: https://pypi.org/project/bcrypt/ .. _Exscript: https://pypi.org/project/Exscript/ .. _filetype: https://pypi.org/project/filetype/ @@ -224,24 +188,6 @@ experts and maintainers in the DevGuide. Data encoding modules --------------------- -binhex -~~~~~~ - -The `binhex `_ module encodes -and decodes Apple Macintosh binhex4 data. It was originally developed for -TRS-80. In the 1980s and early 1990s it was used on classic Mac OS 9 to -encode binary email attachments. - -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** uu and the uu encoding ~~~~~~~~~~~~~~~~~~~~~~ @@ -252,16 +198,6 @@ format has been replaced by MIME. The uu codec is provided by the ``binascii`` module. There's also ``encodings/uu_codec.py`` which is a codec for the same encoding; it should also be deprecated. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** xdrlib ~~~~~~ @@ -271,17 +207,6 @@ the Sun External Data Representation Standard. XDR is an old binary serialization format from 1987. These days it's rarely used outside specialized domains like NFS. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** - Multimedia modules ------------------ @@ -295,23 +220,13 @@ File Format is an old audio format from 1988 based on Amiga IFF. It was most commonly used on the Apple Macintosh. These days only few specialized application use AIFF. -A user disclosed [8]_ that the post production film industry makes heavy +A user disclosed [6]_ that the post production film industry makes heavy use of the AIFC file format. The usage of the ``aifc`` module in closed source and internal software was unknown prior to the first posting of this PEP. This may be a compelling argument to keep the ``aifc`` module in the standard library. The file format is stable and the module does not require much maintenance. The strategic benefits for Python may outmatch the burden. -Module type - pure Python (depends on some functions from `audioop`_ C extension) -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - yes, but expert is currently inactive. -Substitute - **none** audioop ~~~~~~~ @@ -328,16 +243,6 @@ the ``audioop`` module is converted into a private implementation detail, e.g. ``_audioop`` with ``byteswap``, ``alaw2lin``, ``ulaw2lin``, ``lin2alaw``, ``lin2ulaw``, and ``lin2adpcm``. -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - yes -Substitute - **none** chunk ~~~~~ @@ -347,16 +252,6 @@ support for reading and writing Electronic Arts' Interchange File Format. IFF is an old audio file format originally introduced for Commodore and Amiga. The format is no longer relevant. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** imghdr ~~~~~~ @@ -366,18 +261,6 @@ simple tool to guess the image file format from the first 32 bytes of a file or buffer. It supports only a limited number of formats and neither returns resolution nor color depth. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - `puremagic `_, - `filetype `_, - `python-magic `_ ossaudiodev ~~~~~~~~~~~ @@ -386,7 +269,7 @@ The `ossaudiodev `_ module provides support for Open Sound System, an interface to sound playback and capture devices. OSS was initially free software, but later support for newer sound devices and improvements were proprietary. Linux -community abandoned OSS in favor of ALSA [1]_. Some operation systems like +community abandoned OSS in favor of ALSA [1]_. Some operating systems like OpenBSD and NetBSD provide an incomplete [2]_ emulation of OSS. To best of my knowledge, FreeBSD is the only widespread operating system @@ -397,19 +280,9 @@ for both FreeBSD community and core development, if the module would be maintained and distributed by people that care for it and use it. The standard library used to have more audio-related modules. The other -audio device interface (``audiodev``, ``linuxaudiodev``, ``sunaudiodev``) +audio device interfaces (``audiodev``, ``linuxaudiodev``, ``sunaudiodev``) were removed in 2007 as part of the :pep:`3108` stdlib re-organization. -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** sndhdr ~~~~~~ @@ -420,36 +293,13 @@ format, channels, frame rate, and sample widths from the first 512 bytes of a file or buffer. The module only supports AU, AIFF, HCOM, VOC, WAV, and other ancient formats. -Module type - pure Python (depends on `audioop`_ C extension for some operations) -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - `puremagic `_, - `filetype `_, - `python-magic `_ sunau ~~~~~ -The `sunau `_ module provides +The `sunau `_ module provides support for Sun AU sound format. It's yet another old, obsolete file format. -Module type - pure Python (depends on `audioop`_ C extension for some operations) -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** - Networking modules ------------------ @@ -460,16 +310,6 @@ asynchat The `asynchat `_ module is built on top of `asyncore`_ and has been deprecated since Python 3.6. -Module type - pure Python -Deprecated in - 3.6 -Removed in - 3.10 -Has a designated expert - yes -Substitute - asyncio asyncore ~~~~~~~~ @@ -483,17 +323,6 @@ The ``asyncore`` module is also used in stdlib tests. The tests for based on ``asyncore``. These tests must be updated to use asyncio or threading. -Module type - pure Python -Deprecated in - 3.6 -Removed in - 3.10 -Has a designated expert - yes -Substitute - asyncio - cgi ~~~ @@ -505,24 +334,34 @@ inefficient because every incoming request is handled in a new process. "[...] designed poorly and are now near-impossible to fix (``cgi``) [...]" -Several people proposed to either keep the ``cgi`` module for features like -``cgi.parse_qs`` or move ``cgi.escape`` to a different module. The -functions ``cgi.parse_qs`` and ``cgi.parse_qsl`` have been -deprecated for a while and are actually aliases for -``urllib.parse.parse_qs`` and ``urllib.parse.parse_qsl``. The -function ``cgi.quote`` has been deprecated in favor of ``html.quote`` -with secure default values. +Replacements for the various parts of ``cgi`` which are not directly +related to executing code are: -Module type - pure Python -Deprecated in - 3.8 (originally proposed for 2.0 by :pep:`206`) -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** +- ``parse`` with ``urllib.parse.parse_qs`` (``parse`` is just a wrapper) +- ``parse_header`` with ``email.message.Message`` (see example below) +- ``parse_multipart`` with ``email.message.Message`` (same MIME RFCs) +- ``FieldStorage``/``MiniFieldStorage`` has no direct replacement, but can + typically be replaced by using `multipart + `_ (for ``POST`` and ``PUT`` + requests) or ``urllib.parse.parse_qsl`` (for ``GET`` and ``HEAD`` + requests) +- ``valid_boundary`` (undocumented) with ``re.compile("^[ -~]{0,200}[!-~]$")`` + +As an explicit example of how close ``parse_header`` and +``email.message.Message`` are: + +.. code-block:: pycon + + >>> from cgi import parse_header + >>> from email.message import Message + >>> parse_header(h) + ('application/json', {'charset': 'utf8'}) + >>> m = Message() + >>> m['content-type'] = h + >>> m.get_params() + [('application/json', ''), ('charset', 'utf8')] + >>> m.get_param('charset') + 'utf8' cgitb @@ -535,16 +374,6 @@ The ``cgitb`` module is not used by any major Python web framework (Django, Pyramid, Plone, Flask, CherryPy, or Bottle). Only Paste uses it in an optional debugging middleware. -Module type - pure Python -Deprecated in - 3.8 (originally proposed for 2.0 by :pep:`206`) -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** smtpd ~~~~~ @@ -554,16 +383,6 @@ a simple implementation of a SMTP mail server. The module documentation marks the module as deprecated and recommends ``aiosmtpd`` instead. The deprecation message was added in releases 3.4.7, 3.5.4, and 3.6.1. -Module type - pure Python -Deprecated in - **3.4.7**, **3.5.4**, **3.6.1** -To be removed in - 3.10 -Has a designated expert - yes -Substitute - aiosmtpd nntplib ~~~~~~~ @@ -579,21 +398,11 @@ activity since 2014. This is a good indicator that the public interest in NNTP support is declining. The ``nntplib`` tests have been the cause of additional work in the recent -past. Python only contains client side of NNTP. The tests connect to -external news server. The servers are sometimes unavailable, too slow, or do +past. Python only contains the client side of NNTP, so the tests connect to +external news servers. The servers are sometimes unavailable, too slow, or do not work correctly over IPv6. The situation causes flaky test runs on buildbots. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** telnetlib ~~~~~~~~~ @@ -601,16 +410,6 @@ telnetlib The `telnetlib `_ module provides a Telnet class that implements the Telnet protocol. -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Substitute - telnetlib3_, - Exscript_ - Operating system interface -------------------------- @@ -621,7 +420,7 @@ crypt The `crypt `_ module implements password hashing based on the ``crypt(3)`` function from ``libcrypt`` or ``libxcrypt`` on Unix-like platforms. The algorithms are mostly old, of poor -quality and insecure. Users are discouraged to use them. +quality and insecure. Users are discouraged from using them. * The module is not available on Windows. Cross-platform applications need an alternative implementation anyway. @@ -636,61 +435,19 @@ quality and insecure. Users are discouraged to use them. * The module was never useful to interact with system user and password databases. On BSD, macOS, and Linux, all user authentication and password modification operations must go through PAM (pluggable - authentication module), see `spwd`_ deprecation. + authentication module); see the `spwd`_ deprecation. -Module type - C extension + Python module -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - yes, but expert is currently inactive. -Substitute - `legacycrypt `_ (ctypes wrapper), - `bcrypt `_, - `passlib `_, - `argon2cffi `_, - hashlib module (PBKDF2, scrypt) - -macpath -~~~~~~~ - -The `macpath `_ module -provides Mac OS 9 implementation of ``os.path`` routines. Mac OS 9 is no longer -supported. - -Module type - pure Python -Deprecated in - 3.7 -Removed in - 3.8 -Has a designated expert - n/a -Substitute - **none** nis ~~~ The `nis `_ module provides NIS/YP support. Network Information Service / Yellow Pages is an old and -deprecated directory service protocol developed by Sun Microsystems. It's +deprecated directory service protocol developed by Sun Microsystems. Its designed successor NIS+ from 1992 never took off. For a long time, libc's -Name Service Switch, LDAP, and Kerberos/GSSAPI are considered a more powerful -and more secure replacement of NIS. +Name Service Switch, LDAP, and Kerberos/GSSAPI have been considered a more powerful +and more secure replacement for NIS. -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** spwd ~~~~ @@ -698,69 +455,33 @@ spwd The `spwd `_ module provides direct access to Unix shadow password database using non-standard APIs. -In general it's a bad idea to use spwd. It circumvents system +In general, it's a bad idea to use ``spwd``. It circumvents system security policies, does not use the PAM stack, and is only compatible with local user accounts, because it ignores NSS. The use of the ``spwd`` module for access control must be considered a *security bug*, as it bypasses PAM's access control. -Further more the ``spwd`` module uses the +Furthermore, the ``spwd`` module uses the `shadow(3) `_ APIs. Functions like ``getspnam(3)`` access the ``/etc/shadow`` file directly. This is dangerous and even forbidden for confined services on systems with a security engine like SELinux or AppArmor. -Module type - C extension -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - `python-pam `_, - `simpleplam `_ Misc modules ------------ -formatter -~~~~~~~~~ +mailcap +~~~~~~~ -The `formatter `_ module -is an old text formatting module which has been deprecated since Python 3.4. +The `mailcap `__ package +reads *mail capability* files to assist in handling a file attachment in +an email. In most modern operating systems the email client itself handles reacting to +file attachments. Operating systems also have their own way to register +handling files by their file name extension. Finally, the module has +`CVE-2015-20107 `__ filed +against it while having no maintainer to help fix it. -Module type - pure Python -Deprecated in - 3.4 -To be removed in - 3.10 -Has a designated expert - no -Substitute - *n/a* - -imp -~~~ - -The `imp `_ module is the -predecessor of the -`importlib `_ module. Most -functions have been deprecated since Python 3.3 and the module since -Python 3.4. - -Module type - C extension -Deprecated in - 3.4 -To be removed in - 3.10 -Has a designated expert - yes, experts have deprecated the module -Substitute - importlib msilib ~~~~~~ @@ -773,49 +494,8 @@ module is used to facilitate distutils to create MSI installers with the Windows installer, too. Microsoft is slowly moving away from MSI in favor of Windows 10 Apps (AppX) -as new deployment model [3]_. +as a new deployment model [3]_. -Module type - C extension + Python code -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - **none** - -parser -~~~~~~ - -The `parser `_ module provides -an interface to Python’s internal parser and bytecode compiler. The stdlib -has superior ways to interact with the parse tree. From Python 2.5 onward, -it's much more convenient to cut in at the Abstract Syntax Tree (AST) -generation and compilation stage. - -The ``parser`` module causes additional work. It's C code that must be -kept in sync with any change to Python's grammar and internal parser. -Pablo wants to remove the parser module and promote lib2to3's pgen2 instead -[6]_. - -Most importantly the presence of the ``parser`` module makes it harder to -switch to something more powerful than a LL(1) parser [7]_. Since the -``parser`` module is documented as deprecated since Python 2.5 and a new -parsing technology is planned for 3.9, the ``parser`` module is scheduled for -removal in 3.9. - -Module type - C extension -Deprecated in - 3.8, documented as deprecated since **2.5** -To be removed in - **3.9** -Has a designated expert - yes, experts have deprecated the module. -Substitute - ast, lib2to3.pgen2 pipes ~~~~~ @@ -823,47 +503,14 @@ pipes The `pipes `_ module provides helpers to pipe the input of one command into the output of another command. The module is built on top of ``os.popen``. Users are encouraged to use -the subprocess module instead. - -Module type - pure Python -Deprecated in - 3.8 -To be removed in - 3.10 -Has a designated expert - no -Substitute - subprocess module - -Removed modules -=============== - -fpectl ------- - -The `fpectl `_ module was -never built by default, its usage was discouraged and considered dangerous. -It also required a configure flag that caused an ABI incompatibility. The -module was removed in 3.7 by Nathaniel J. Smith in -`bpo-29137 `_. - -Module type - C extension + CAPI -Deprecated in - 3.7 -Removed in - 3.7 -Has a designated expert - n/a -Substitute - **none** +the ``subprocess`` module instead. Modules to keep =============== -Some modules were originally proposed for deprecation. +Some modules were originally proposed for deprecation but are no longer +listed as such in this PEP. .. csv-table:: Table 2: Withdrawn deprecations :header: "Module", "Deprecated in", "Replacement" @@ -872,7 +519,6 @@ Some modules were originally proposed for deprecation. colorsys,\-,"colormath, colour, colorspacious, Pillow" fileinput,\-,argparse getopt,\-,"argparse, optparse" - lib2to3,\-, optparse,**3.2**,argparse wave,\-, @@ -892,45 +538,18 @@ The PyPI packages ``colormath``, ``colour``, and ``colorspacious`` provide more advanced features. The Pillow library is better suited to transform images between color systems. -Module type - pure Python -Has a designated expert - no -Substitute - `colormath `_, - `colour `_ - `colorspacious `_, - `Pillow `_ fileinput --------- The `fileinput `_ module implements helpers to iterate over a list of files from ``sys.argv``. The -module predates the ``optparser`` and ``argparser`` modules. The same functionality -can be implemented with the ``argparser`` module. +module predates the ``optparse`` and ``argparse`` modules. The same functionality +can be implemented with the ``argparse`` module. Several core developers expressed their interest to keep the module in the standard library, as it is handy for quick scripts. -Module type - pure Python -Has a designated expert - no - -lib2to3 -------- - -The `lib2to3 `_ package provides -the ``2to3`` command to transpile Python 2 code to Python 3 code. - -The package is useful for other tasks besides porting code from Python 2 to -3. For example, `Black`_ uses it for code reformatting. - -Module type - pure Python -Has a designated expert - no getopt ------ @@ -942,12 +561,6 @@ Although users are encouraged to use ``argparse`` instead, the ``getopt`` module still widely used. The module is small, simple, and handy for C developers to write simple Python scripts. -Module type - pure Python -Has a designated expert - no -Substitute - argparse optparse -------- @@ -958,14 +571,6 @@ the predecessor of the ``argparse`` module. Although it has been deprecated for many years, it's still too widely used to remove it. -Module type - pure Python -Deprecated in - 3.2 -Has a designated expert - yes -Substitute - argparse wave ---- @@ -973,7 +578,7 @@ wave The `wave `_ module provides support for the WAV sound format. -The module is not deprecated, because The WAV format is still relevant these +The module is not deprecated, because the WAV format is still relevant these days. The ``wave`` module is also used in education, e.g. to show kids how to make noise with a computer. @@ -984,55 +589,16 @@ module. To remove ``wave``'s dependency on ``audioop``, the byte swap function could be either be moved to another module (e.g. ``operator``) or the ``array`` module could gain support for 24-bit (3-byte) arrays. -Module type - pure Python (depends on *byteswap* from `audioop`_ C extension) -Has a designated expert - no - - -Future maintenance of removed modules -===================================== - -The main goal of the PEP is to reduce the burden and workload on the Python -core developer team. Therefore, removed modules will not be maintained by -the core team as separate PyPI packages. However the removed code, tests and -documentation may be moved into a new Git repository, so community members -have a place from which they can pick up and fork code. - -A first draft of a `legacylib `_ -repository is available on my private GitHub account. The modules could be -made available on PyPI. The Python core team will not publish or maintain -the packages. It is my hope that members of the Python community will -adopt, maintain, and perhaps improve the deprecated modules. - -It's my hope that some of the deprecated modules will be picked up and -adopted by users that actually care about them. For example, ``colorsys`` and -``imghdr`` are useful modules, but have limited feature set. A fork of -``imghdr`` can add new features and support for more image formats, without -being constrained by Python's release cycle. - -Most of the modules are in pure Python and can be easily packaged. Some -depend on a simple C module, e.g. `audioop`_ and `crypt`_. Since `audioop`_ -does not depend on any external libraries, it can be shipped as binary -wheels with some effort. Other C modules can be replaced with ``ctypes`` or ``cffi``. -For example, I created `legacycrypt`_, which provides a full implementation of -``crypt``. It is implemented on top of a ctypes wrapper around ``libxcrypt`` -and ``libcrypt`` instead of a C extension like the original ``_crypt`` -module. - Discussions =========== * Elana Hashman and Nick Coghlan suggested to keep the ``getopt`` module. -* Berker Peksag proposed to deprecate and removed ``msilib``. +* Berker Peksag proposed to deprecate and remove ``msilib``. * Brett Cannon recommended to delay active deprecation warnings and removal of modules like ``imp`` until Python 3.10. Version 3.8 will be released shortly before Python 2 reaches end-of-life. A delay reduced churn for users that migrate from Python 2 to 3.8. -* Brett also came up with the idea to keep ``lib2to3``. The package is useful - for other purposes, e.g. `Black `_ uses - it to reformat Python code. * At one point, distutils was mentioned in the same sentence as this PEP. To avoid lengthy discussion and delay of the PEP, I decided against dealing with distutils. Deprecation of the distutils package will be handled by @@ -1045,13 +611,29 @@ Discussions stdlib doesn't have a replacement for the servers, yet. +Rejected ideas +============== + +Creating/maintaining a separate repo for the deprecated modules +--------------------------------------------------------------- + +It was previously proposed to create a separate repository containing the +deprecated modules packaged for installation. One of the PEP authors went so far +as to create a `demo repository `_. In the +end, though, it was decided that the added workload to create and maintain such +a repo officially wasn't justified, as the source code will continue to be +available in the CPython repository for people to vendor as necessary. Similar +work has also not been done when previous modules were deprecated and removed, +and it seemingly wasn't an undue burden on the community. + + Update history ============== Update 1 -------- -* Deprecate `parser`_ module +* Deprecate ``parser`` module * Keep `fileinput`_ module * Elaborate why ``crypt`` and ``spwd`` are dangerous and bad * Improve sections for `cgitb`_, `colorsys`_, `nntplib`_, and `smtpd`_ modules @@ -1069,16 +651,25 @@ Update 2 * Add experts * Redirect discussions to discuss.python.org * Deprecate `telnetlib`_ -* Deprecate compat32 policy of email package +* Deprecate compat32 policy of ``email`` package * Add creation year to overview table * Mention :pep:`206` and :pep:`3108` * Update sections for ``aifc``, ``audioop``, ``cgi``, and ``wave``. Update 3 -------- + * Keep the legacy email API modules. Internal deprecations will be handled separately. +Update 4 +-------- + +* Add Brett as a co-author. +* Retarget the PEP for Python 3.11. +* Examples of how to replace the relevant parts of ``cgi`` + (thanks Martijn Pieters). + References ========== @@ -1088,15 +679,14 @@ References .. [3] https://blogs.msmvps.com/installsite/blog/2015/05/03/the-future-of-windows-installer-msi-in-the-light-of-windows-10-and-the-universal-windows-platform/ .. [4] https://twitter.com/ChristianHeimes/status/1130257799475335169 .. [5] https://twitter.com/dabeaz/status/1130278844479545351 -.. [6] https://mail.python.org/pipermail/python-dev/2019-May/157464.html -.. [7] https://discuss.python.org/t/switch-pythons-parsing-tech-to-something-more-powerful-than-ll-1/379 -.. [8] https://mail.python.org/pipermail/python-dev/2019-May/157634.html +.. [6] https://mail.python.org/pipermail/python-dev/2019-May/157634.html Copyright ========= -This document has been placed in the public domain. +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0595.rst b/peps/pep-0595.rst similarity index 77% rename from pep-0595.rst rename to peps/pep-0595.rst index 61964da01..7a06554fd 100644 --- a/pep-0595.rst +++ b/peps/pep-0595.rst @@ -20,7 +20,7 @@ switching to GitHub Issues, as proposed by :pep:`581`. Resolution ========== -2020-06-25: With the acceptance of PEP 581, the move to GitHub for +2020-06-25: With the acceptance of :pep:`581`, the move to GitHub for issues is proceeding, this PEP is being marked as a withdrawn informational PEP. @@ -28,9 +28,11 @@ informational PEP. Motivation ========== -On May 14th, 2019 :pep:`581` has been accepted [#]_ without much -public discussion and without a clear consensus [#]_. The PEP -contains factual errors and doesn't address some of the +On May 14th, 2019 :pep:`581` has been `accepted +`_ +without much public discussion and `without a clear consensus +`_. +The PEP contains factual errors and doesn't address some of the issues that the migration to GitHub Issues might present. Given the scope of the migration, the amount of work required, @@ -98,13 +100,16 @@ on GitHub Issues. * **Nosy list autocomplete.** The nosy list has an autocomplete feature that suggests maintainers and experts. The suggestions - are automatically updated when the experts index [#]_ changes. + are automatically updated when the `experts index + `_ changes. * **Dependencies and Superseders.** Roundup allows to specify dependencies that must be addressed before the current issues can be closed and a superseder issue to easily mark duplicates - [#]_. The list of dependencies can also be used to create - meta-issues that references several other sub-issues [#]_. + (for example, `bpo-12078 `_). + The list of dependencies can also be used to create + meta-issues that references several other sub-issues + (for example, `bpo-26865 `_). Improving Roundup @@ -115,7 +120,7 @@ and other desired features and discusses how they can be implemented by improving Roundup and/or our instance. * **REST API support.** A REST API will make integration with other - services and the development of new tools and applications easiers. + services and the development of new tools and applications easier. Upstream Roundup now supports a REST API. Updating the tracker will make the REST API available. @@ -125,15 +130,17 @@ by improving Roundup and/or our instance. It will also solve issues with confirmation emails being marked as spam, and provide two-factor authentication. - A patch to add this functionality is already available and is - being integrated at the time of writing [#]_. + A patch to add this functionality is `already available + `_ + and is being integrated at the time of writing. * **Markdown support and message preview and editing.** This feature will allow the use of Markdown in messages and the ability to preview the message before the submission and edit it afterward. This can be done, but it will take some work. Possible solutions - have been proposed on the roundup-devel mailing list [#]_. + have been proposed on the `roundup-devel mailing list + `_. * **"Remove me from nosy list" button.** Add a button on issue pages to remove self from the nosy list. @@ -160,14 +167,18 @@ by improving Roundup and/or our instance. * **Add PR link to BPO emails.** Currently bpo emails don't include links to the corresponding PRs. - A patch [#]_ is available to change the content of the bpo emails - from:: + A `patch `_ + is available to change the content of the bpo emails from: + + .. code-block:: text components: +Tkinter versions: +Python 3.4 pull_requests: +42 - to:: + to: + + .. code-block:: text components: +Tkinter versions: +Python 3.4 @@ -190,7 +201,7 @@ PEP 581 issues This section addresses some errors and inaccuracies found in :pep:`581`. -The "Why GitHub?" section of PEP 581 lists features currently +The "Why GitHub?" section of :pep:`581` lists features currently available on GitHub Issues but not on Roundup. Some of this features are currently supported: @@ -222,7 +233,8 @@ are currently supported: * The GitHub integration of Roundup automatically closes issues when a commit that contains "fixes issue " is merged. (Alternative spellings such as "closes" or "bug" are also supported.) - See [#]_ for a recent example of this feature. + See `this message `_ + for a recent example of this feature. * "Support for permalinks, allowing easy quoting and copying & pasting of source code." @@ -239,15 +251,19 @@ are currently supported: write and maintain bots. In some cases, bots are required to workaround GitHub's lack of - features rather than expanding. [#]_ was written - specifically to workaround GitHub's email integration. + features rather than expanding. `This webhook + `_ + was written specifically to workaround GitHub's email integration. Updating our bots to stay up-to-date with changes in the GitHub API - has also maintenance cost. [#]_ took two days to be fixed. + has also maintenance cost. `This recent incident caused by GitHub + `_ + took two days to be fixed. In addition, we will still need to maintain Roundup for bpo (even if it becomes read-only) and for the other trackers - we currently host/maintain (Jython [#]_ and Roundup [#]_). + we currently host/maintain (`Jython `_ + and `Roundup `_). The "Issues with Roundup / bpo" section of :pep:`581` lists some issues that have already been fixed: @@ -256,8 +272,11 @@ that have already been fixed: it puts heavy burden on the few existing maintainers in terms of reviewing, testing, and applying patches." - * While Roundup uses Mercurial by default, there is a git clone - available on GitHub [#]_. Roundup also has CI available [#]_ [#]_. + * While Roundup uses Mercurial by default, there is a `git clone + available on GitHub `_. + Roundup also has CI available on `Travis CI + `_ and `Codecov + `_. * "There is no REST API available. There is an open issue in Roundup for adding REST API. Last activity was in 2016." @@ -271,7 +290,8 @@ that have already been fixed: This has now been changed to make the email addresses hidden for regular users too (Developers and Coordinators can still see them). - The "Email address" column from the user listing page [#]_ has been + The "Email address" column from the `user listing page + `_ has been removed too. * "It sends a number of unnecessary emails and notifications, and it is @@ -362,11 +382,12 @@ need to be addressed regardless of the approach used: redirects and preserve external links to GitHub issues. * **References preservation and updating.** In addition to issue - references, bpo converts a number of other references into links, + references, bpo `converts a number of other references into links + `_, including message and PR IDs, changeset numbers, legacy SVN revision numbers, paths to files in the repo, files in tracebacks (detecting the correct branch), and links to devguide pages and - sections [#]_. + sections. Since Roundup converts references to links when messages are requested, it is possible to update the target and generate the @@ -403,145 +424,38 @@ need to be addressed regardless of the approach used: * **Signal to noise ratio.** Switching to GitHub Issues will likely increase the number of invalid reports and increase the triaging effort. This concern has been raised in the past - in a Zulip topic [#]_. + in a `Zulip topic + `_. There have been already cases where people posted comments on PRs that required moderators to mark them as off-topic or disruptive, delete them altogether, and even lock the - conversation [#]_. + conversation (for example, `this PR + `_. * **Weekly tracker reports and stats.** Roundup sends weekly reports to python-dev with a summary that includes new issues, recent issues with no replies, recent issues waiting for review, most discussed issues, closed issues, and deltas for open/closed/total - issue counts [#]_. The report provides an easy way to keep track + issue counts (for example, see `this summary + `_). + The report provides an easy way to keep track of the tracker activity and to make sure that issues that require attention are noticed. - The data collect by the weekly report is also use to generate - statistics and graphs that can be used to gain new insights [#]_. + The data collect by the weekly report is also used to generate + `statistics and graphs `_ + that can be used to gain new insights. * **bpo-related MLs.** There are currently two mailing lists where bpo posts new tracker issues and all messages respectively: - ``new-bugs-announce`` [#]_ and ``python-bugs-list`` [#]_. A new system - will need to be developed to preserve this functionality. These MLs + `new-bugs-announce `_ + and `python-bugs-list `_. + A new system will need to be developed to preserve this functionality. These MLs offer additional ways to keep track of the tracker activity. -References -========== - -.. [#] [Python-Dev] PEP 581 (Using GitHub issues for CPython) is accepted - - https://mail.python.org/pipermail/python-dev/2019-May/157399.html - -.. [#] [python-committers] [Python-Dev] PEP 581 (Using GitHub issues - for CPython) is accepted - - https://mail.python.org/pipermail/python-committers/2019-May/006755.html - -.. [#] Experts Index -- Python Devguide - - https://devguide.python.org/experts/ - -.. [#] An example of superseded issues: - "re.sub() replaces only several matches" - - https://bugs.python.org/issue12078 - -.. [#] An example of meta issue using dependencies to track sub-issues: - "Meta-issue: support of the android platform"" - - https://bugs.python.org/issue26865 - -.. [#] Support logging in with GitHub - - https://github.com/python/bugs.python.org/issues/7 - -.. [#] Re: [Roundup-devel] PEP 581 and Google Summer of Code - - https://sourceforge.net/p/roundup/mailman/message/36667828/ - -.. [#] [Tracker-discuss] [issue624] bpo emails contain useless non-github - pull_request number - users want a link to actual github PR - - https://mail.python.org/pipermail/tracker-discuss/2018-June/004547.html - -.. [#] The commit reported in msg342882 closes the issue (see the history below) - - https://bugs.python.org/issue36951#msg342882 - -.. [#] The cpython-emailer-webhook project - - https://github.com/berkerpeksag/cpython-emailer-webhook - -.. [#] A recent incident caused by GitHub - - https://github.com/python/bedevere/pull/163 - -.. [#] Jython issue tracker - - https://bugs.jython.org/ - -.. [#] Roundup issue tracker - - https://issues.roundup-tracker.org/ - -.. [#] GitHub clone of Roundup - - https://github.com/roundup-tracker/roundup - -.. [#] Travis-CI for Roundup - - https://travis-ci.org/roundup-tracker/roundup) and codecov - -.. [#] Codecov for Roundup - - https://codecov.io/gh/roundup-tracker/roundup/commits - -.. [#] User listing -- Python tracker - - https://bugs.python.org/user?@sort=username - -.. [#] Generating Special Links in a Comment -- Python Devguide - - https://devguide.python.org/triaging/#generating-special-links-in-a-comment - -.. [#] The New-bugs-announce mailing list - - https://mail.python.org/mailman/listinfo/new-bugs-announce - -.. [#] The Python-bugs-list mailing list - - https://mail.python.org/mailman/listinfo/python-bugs-list - -.. [#] An example of [Python-Dev] Summary of Python tracker Issues - - https://mail.python.org/pipermail/python-dev/2019-May/157483.html - -.. [#] Issues stats -- Python tracker - - https://bugs.python.org/issue?@template=stats - -.. [#] s/n ratio -- Python -- Zulip - - https://python.zulipchat.com/#narrow/stream/130206-pep581/topic/s.2Fn.20ratio - -.. [#] For example, this and other related PRs: - - https://github.com/python/cpython/pull/9099 - - Copyright ========= This document has been placed in the public domain. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0596.rst b/peps/pep-0596.rst similarity index 81% rename from pep-0596.rst rename to peps/pep-0596.rst index c618cff6f..62b4d10d1 100644 --- a/pep-0596.rst +++ b/peps/pep-0596.rst @@ -1,11 +1,10 @@ PEP: 596 Title: Python 3.9 Release Schedule -Version: $Revision$ -Last-Modified: $Date$ Author: Ɓukasz Langa Discussions-To: https://discuss.python.org/t/pep-596-python-3-9-release-schedule-doubling-the-release-cadence/1828 Status: Active Type: Informational +Topic: Release Content-Type: text/x-rst Created: 04-Jun-2019 Python-Version: 3.9 @@ -39,8 +38,8 @@ Release Schedule -------------- Note: the dates below use a 17-month development period that results -in a 12-month release cadence between major versions, as defined by -PEP 602. +in a 12-month release cadence between feature versions, as defined by +:pep:`602`. Actual: @@ -69,30 +68,32 @@ Actual: - 3.9.1 candidate 1: Tuesday, 2020-11-24 - 3.9.1 final: Monday, 2020-12-07 -- 3.9.2 candidate 1: Monday, 2021-02-16 -- 3.9.2 final: Monday, 2021-02-19 +- 3.9.2 candidate 1: Tuesday, 2021-02-16 +- 3.9.2 final: Friday, 2021-02-19 - 3.9.3: Friday, 2021-04-02 (security hotfix; recalled due to bpo-43710) - 3.9.4: Sunday, 2021-04-04 (ABI compatibility hotfix) - 3.9.5: Monday, 2021-05-03 - 3.9.6: Monday, 2021-06-28 - 3.9.7: Monday, 2021-08-30 - -Expected: - -- 3.9.8: Monday, 2021-11-01 -- 3.9.9: Monday, 2022-01-03 -- 3.9.10: Monday, 2022-02-28 - -Final regular bugfix release with binary installers: - -- 3.9.11: Monday, 2022-05-02 +- 3.9.8: Friday, 2021-11-05 (recalled due to bpo-45235) +- 3.9.9: Monday, 2021-11-15 +- 3.9.10: Friday, 2022-01-14 +- 3.9.11: Wednesday, 2022-03-16 +- 3.9.12: Wednesday, 2022-03-23 +- 3.9.13: Tuesday, 2022-05-17 (final regular bugfix release with binary + installers) Source-only security fix releases --------------------------------- -Provided irregularly on an "as-needed" basis until October 2025, -starting with 3.9.12. +Provided irregularly on an "as-needed" basis until October 2025. + +- 3.9.14: Tuesday, 2022-09-06 +- 3.9.15: Tuesday, 2022-10-11 +- 3.9.16: Tuesday, 2022-12-06 +- 3.9.17: Tuesday, 2023-06-06 +- 3.9.18: Thursday, 2023-08-24 3.9 Lifespan @@ -125,13 +126,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 72 - coding: utf-8 - End: diff --git a/pep-0597.rst b/peps/pep-0597.rst similarity index 100% rename from pep-0597.rst rename to peps/pep-0597.rst diff --git a/pep-0598.rst b/peps/pep-0598.rst similarity index 98% rename from pep-0598.rst rename to peps/pep-0598.rst index 2a6feabf5..0271d56a8 100644 --- a/pep-0598.rst +++ b/peps/pep-0598.rst @@ -14,7 +14,7 @@ Python-Version: 3.9 Abstract ======== -PEP 602 proposes reducing the feature delivery latency for the Python +:pep:`602` proposes reducing the feature delivery latency for the Python standard library and CPython reference interpreter by increasing the frequency of CPython feature releases from every 18-24 months to instead occur every 9-12 months. @@ -31,12 +31,12 @@ PEP Withdrawal ============== This PEP has been withdrawn in favour of the rolling beta release stream -proposal in PEP 605. +proposal in :pep:`605`. However, the concerns raised in this PEP are likely to apply to any other "Long Term Support branch" proposals that allow feature backports to improve the developer experience of supporting such releases (such as the EL Python -draft at [3_]), so the ideas presented here may provide useful design +draft at [3]_), so the ideas presented here may provide useful design suggestions for such proposals. @@ -221,7 +221,7 @@ The pre-release beta period would be relaxed to use the incremental feature release policy for changes, rather than the stricter maintenance release policy. For governance purposes, baseline feature releases are the only releases that -would qualify as a "feature release" in the PEP 13 sense (incremental feature +would qualify as a "feature release" in the :pep:`13` sense (incremental feature releases wouldn't count). @@ -516,11 +516,11 @@ Motivation ========== The motivation for change in this PEP is essentially the same as the motivation -for change in PEP 596: the current 18-24 month gap between feature releases has +for change in :pep:`596`: the current 18-24 month gap between feature releases has a lot of undesirable consequences, especially for the standard library (see -PEP 596 for further articulation of the details). +:pep:`596` for further articulation of the details). -This PEP's concern with the specific proposal in PEP 596 is that it doubles the +This PEP's concern with the specific proposal in :pep:`596` is that it doubles the number of actively supported Python branches, increasing the complexity of compatibility testing matrices for the entire Python community, increasing the number of binary Python wheels to be uploaded to PyPI when not using the stable @@ -612,7 +612,7 @@ in that release series. Implications for the proposed Scientific Python ecosystem support period ------------------------------------------------------------------------ -Based on discussions at SciPy 2019, a NEP is currently being drafted [2_] to +Based on discussions at SciPy 2019, a NEP is currently being drafted [2]_ to define a common convention across the Scientific Python ecosystem for dropping support for older Python versions. @@ -734,7 +734,7 @@ from the 2019-10 release of Python 3.8.0 and a final Python 3.9.x incremental feature release in 2021-10 evenly between pre-release development and subsequent incremental feature releases. -This is an area where this PEP could adopt part of the proposal in PEP 596, +This is an area where this PEP could adopt part of the proposal in :pep:`596`, by instead making that split ~9 months of pre-release development, and ~15 months of incremental feature releases: @@ -799,7 +799,7 @@ branched. An alternative way of handling this would be to start publishing alpha releases for the next baseline feature release during the feature addition period (similar -to the way that PEP 596 proposes to starting publishing Python 3.9.0 alpha +to the way that :pep:`596` proposes to starting publishing Python 3.9.0 alpha releases during the Python 3.8.0 release candidate period). However, rather than setting specific timelines for that at a policy level, @@ -833,7 +833,7 @@ the subsequent October 2022 baseline feature release, as there are already inher compatibility risks associated with the choice of either "Python 4.0" (erroneous checks for the major version being exactly 3 rather than 3 or greater), or "Python 3.10" (code incorrectly assuming that the minor version will always -contain exactly one decimal digit) [1_]. +contain exactly one decimal digit) [1]_. While the text of this PEP assumes that the release published in 2022 will be 3.10 (as the PEP author personally considers that the more reasonable and most @@ -883,7 +883,7 @@ References ========== .. [1] Anthony Sottile created a pseudo "Python 3.10" to find and fix such issues - (https://github.com/asottile/python3.10) + (https://github.com/asottile-archive/python3.10) .. [2] NEP proposing a standard policy for dropping support of old Python versions (https://github.com/numpy/numpy/pull/14086) @@ -895,13 +895,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 80 - coding: utf-8 - End: diff --git a/pep-0599.rst b/peps/pep-0599.rst similarity index 96% rename from pep-0599.rst rename to peps/pep-0599.rst index 676484ebf..1c22a643e 100644 --- a/pep-0599.rst +++ b/peps/pep-0599.rst @@ -5,12 +5,13 @@ Last-Modified: $Date$ Author: Dustin Ingram Sponsor: Paul Moore BDFL-Delegate: Paul Moore -Discussions-To: https://discuss.python.org/t/the-next-manylinux-specification/ +Discussions-To: https://discuss.python.org/t/the-next-manylinux-specification/1043 Status: Superseded Type: Informational +Topic: Packaging Content-Type: text/x-rst Created: 29-Apr-2019 -Post-History: 29-April-2019 +Post-History: 29-Apr-2019 Superseded-By: 600 Resolution: https://discuss.python.org/t/the-next-manylinux-specification/1043/199 @@ -35,7 +36,7 @@ patches will be made available. All wheels built under the point. Therefore, we propose the continuation of the existing manylinux -standard, and that a new PEP 425-style [2]_ platform tag called +standard, and that a new :pep:`425`-style platform tag called ``manylinux2014`` be derived from CentOS 7 and that the ``manylinux`` toolchain, PyPI, and ``pip`` be updated to support it. @@ -178,7 +179,7 @@ the ``manylinux2014`` tag: built against Python 2, then, must include either the ``cpy27mu`` tag indicating it was built against an interpreter with the UCS-4 ABI or the ``cpy27m`` tag indicating an interpreter with the UCS-2 - ABI. [6]_ [7]_ + ABI. (:pep:`3149` [7]_) 5. A wheel *must not* require the ``PyFPE_jbuf`` symbol. This is achieved by building it against a Python compiled *without* the ``--with-fpectl`` ``configure`` flag. @@ -281,16 +282,12 @@ References .. [1] CentOS Product Specifications (https://wiki.centos.org/About/Product) -.. [2] PEP 425: Compatibility Tags for Built Distributions - (https://www.python.org/dev/peps/pep-0425/) .. [3] Tracking issue for manylinux2010 rollout (https://github.com/pypa/manylinux/issues/179) .. [4] Red Hat Universal Base Image 7 (https://access.redhat.com/containers/?tab=overview#/registry.access.redhat.com/ubi7) .. [5] The CentOS Alternative Architecture Special Interest Group (https://wiki.centos.org/SpecialInterestGroup/AltArch) -.. [6] PEP 3149: ABI version tagged .so files - (https://www.python.org/dev/peps/pep-3149/) .. [7] SOABI support for Python 2.X and PyPy (https://github.com/pypa/pip/pull/3075) .. [8] auditwheel @@ -299,7 +296,7 @@ References Acceptance ========== -PEP 599 was `accepted by Paul Moore on July 31, 2019 +:pep:`599` was `accepted by Paul Moore on July 31, 2019 `_. Copyright diff --git a/pep-0600.rst b/peps/pep-0600.rst similarity index 99% rename from pep-0600.rst rename to peps/pep-0600.rst index ddeb6d6bd..5fa3996b0 100644 --- a/pep-0600.rst +++ b/peps/pep-0600.rst @@ -2,16 +2,17 @@ PEP: 600 Title: Future 'manylinux' Platform Tags for Portable Linux Built Distributions Version: $Revision$ Last-Modified: $Date$ -Author: Nathaniel J. Smith +Author: Nathaniel J. Smith , Thomas Kluyver Sponsor: Paul Moore BDFL-Delegate: Paul Moore -Discussions-To: Discourse https://discuss.python.org/t/the-next-manylinux-specification/1043 -Status: Accepted -Type: Informational +Discussions-To: https://discuss.python.org/t/the-next-manylinux-specification/1043 +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 03-May-2019 -Post-History: 3-May-2019 +Post-History: 03-May-2019 Replaces: 513, 571, 599 Resolution: https://discuss.python.org/t/pep-600-future-manylinux-platform-tags-for-portable-linux-built-distributions/2414/27 diff --git a/pep-0601.txt b/peps/pep-0601.rst similarity index 59% rename from pep-0601.txt rename to peps/pep-0601.rst index fa6054d26..046c898b2 100644 --- a/pep-0601.txt +++ b/peps/pep-0601.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 26-Aug-2019 Python-Version: 3.8 -Post-History: 26-Aug-2019 23-Sep-2019 +Post-History: 26-Aug-2019, 23-Sep-2019 Resolution: https://discuss.python.org/t/pep-601-forbid-return-break-continue-breaking-out-of-finally/2239/32 Rejection Note @@ -18,28 +18,28 @@ This PEP was rejected by the Steering Council by a vote of 4/4. Guido's arguments for rejecting the PEP are: "it seems to me that most languages implement this kind of construct but have style guides and/or linters that -reject it. I would support a proposal to add this to PEP 8", and "I note that +reject it. I would support a proposal to add this to :pep:`8`", and "I note that the toy examples are somewhat misleading – the functionality that may be useful is a conditional return (or break etc.) inside a finally block.". Abstract ======== -This PEP proposes to forbid return, break and continue statements within -a finally suite where they would break out of the finally. Their use in +This PEP proposes to forbid ``return``, ``break`` and ``continue`` statements within +a ``finally`` suite where they would break out of the ``finally``. Their use in such a location silently cancels any active exception being raised through -the finally, leading to unclear code and possible bugs. +the ``finally``, leading to unclear code and possible bugs. -Continue is currently not supported in a finally in Python 3.7 (due to +``continue`` is currently not supported in a ``finally`` in Python 3.7 (due to implementation issues) and the proposal is to not add support for it in -Python 3.8. For return and break the proposal is to deprecate their use +Python 3.8. For ``return`` and ``break`` the proposal is to deprecate their use in Python 3.9, emit a compilation warning in Python 3.10 and then forbid their use after that. Motivation ========== -The use of return, break and continue within a finally suite leads to behaviour +The use of ``return``, ``break`` and ``continue`` within a ``finally`` suite leads to behaviour which is not at all obvious. Consider the following function:: def foo(): @@ -49,9 +49,9 @@ which is not at all obvious. Consider the following function:: return This will return cleanly (without an exception) even though it has infinite -recursion and raises an exception within the try. The reason is that the return -within the finally will silently cancel any exception that propagates through -the finally suite. Such behaviour is unexpected and not at all obvious. +recursion and raises an exception within the ``try``. The reason is that the ``return`` +within the ``finally`` will silently cancel any exception that propagates through +the ``finally`` suite. Such behaviour is unexpected and not at all obvious. This function is equivalent to:: def foo(): @@ -61,8 +61,8 @@ This function is equivalent to:: pass return -Break and continue have similar behaviour (they silence exceptions) if they -jump to code outside the finally suite. For example:: +``break`` and ``continue`` have similar behaviour (they silence exceptions) if they +jump to code outside the ``finally`` suite. For example:: def bar(): while True: @@ -83,35 +83,35 @@ This behaviour goes against the following parts of The Zen of Python: If this behaviour of silencing exceptions is really needed then the explicit form of a try-except can be used instead, and this makes the code clearer. -Independent to the semantics, implementing return/break/continue within a -finally suite is non-trivial as it requires to correctly track any active -exceptions at runtime (an executing finally suite may or may not have an +Independent to the semantics, implementing ``return``/``break``/``continue`` within a +``finally`` suite is non-trivial as it requires to correctly track any active +exceptions at runtime (an executing ``finally`` suite may or may not have an active exception) and cancel them as appropriate. CPython did have a bug in -this for the case of continue and so originally disallowed it [1]. Requiring -correct behaviour for return/break/continue within a finally puts an +this for the case of ``continue`` and so originally disallowed it [1]_. Requiring +correct behaviour for ``return``/``break``/``continue`` within a ``finally`` puts an unnecessary burden on alternative implementations of Python. Other languages =============== -Java allows to return from within a finally block, but its use is discouraged -according to [2], [3], [4]. The Java compiler later on included a linting -option -Xlint:finally to warn against the use of return within a finally block. +Java allows to return from within a ``finally`` block, but its use is discouraged +according to [2]_, [3]_, [4]_. The Java compiler later on included a linting +option ``-Xlint:finally`` to warn against the use of return within a ``finally`` block. The Eclipse editor also warns about this use. Ruby allows return from inside ensure (Python's finally), but it should be an -explicit return. It is discouraged and handled by linters [5], [6]. +explicit return. It is discouraged and handled by linters [5]_, [6]_. -Like Ruby, JavaScript also allows use of return/break/continue within a finally -but it is seen as unsafe and it is handled by eslint [7]. +Like Ruby, JavaScript also allows use of ``return``/``break``/``continue`` within a ``finally`` +but it is seen as unsafe and it is handled by eslint [7]_. -C# forbids the use of ending statements like return/goto/break within a finally -[8], [9]. +C# forbids the use of ending statements like ``return``/``goto``/``break`` within a ``finally`` +[8]_, [9]_. Rationale ========= -Since the behaviour of return/break/continue within a finally is unclear, the +Since the behaviour of ``return``/``break``/``continue`` within a ``finally`` is unclear, the pattern is rarely used, and there is a simple alternative to writing equivalent code (which is more explicit), forbidding the syntax is the most straightforward approach. @@ -119,19 +119,19 @@ approach. Specification ============= -This is a change to the compiler, not the grammar. The compiler should -check for the following in a finally suite: +This is a change to the compiler, not the grammar. The compiler should +check for the following in a ``finally`` suite: -* A return in any statement, at any level of nesting. +* A ``return`` in any statement, at any level of nesting. -* A break/continue in any statement, at any level of nesting, that would - transfer control flow outside the finally suite. +* A ``break``/``continue`` in any statement, at any level of nesting, that would + transfer control flow outside the ``finally`` suite. Upon finding such a case it should emit the appropriate exception: -* For continue, a SyntaxError (this is the current behaviour of 3.7). +* For ``continue``, a ``SyntaxError`` (this is the current behaviour of 3.7). -* For return/break, a SyntaxWarning in 3.10, and a SyntaxError after that. +* For ``return``/``break``, a ``SyntaxWarning`` in 3.10, and a ``SyntaxError`` after that. For example, the following are all forbidden by this proposal:: @@ -160,8 +160,8 @@ For example, the following are all forbidden by this proposal:: for x in range(10): return -The following is still allowed because the continue doesn't escape the -finally:: +The following is still allowed because the ``continue`` doesn't escape the +``finally``:: try: pass @@ -169,17 +169,17 @@ finally:: for x in range(10): continue -Note that yielding from within a finally remains acceptable by this PEP -because resuming the generator will resume the finally and eventually +Note that yielding from within a ``finally`` remains acceptable by this PEP +because resuming the generator will resume the ``finally`` and eventually raise any active exceptions (so they are never silenced by yielding). Backwards Compatibility ======================= -This is a backwards incompatible change, for return and break. +This is a backwards incompatible change, for ``return`` and ``break``. The following locations in the CPython standard library (at -v3.8.0b1-651-g7fcc2088a5) use return within finally: +v3.8.0b1-651-g7fcc2088a5) use ``return`` within ``finally``: * Lib/subprocess.py:921 - the use here looks like a bug @@ -189,11 +189,11 @@ v3.8.0b1-651-g7fcc2088a5) use return within finally: * Lib/multiprocessing/connection.py:318 - the use here looks legitimate but the intention is not clear -* Lib/test/test_sys_settrace.py:837 - a test for return within finally +* Lib/test/test_sys_settrace.py:837 - a test for ``return`` within ``finally`` -* Lib/test/test_sys_settrace.py:1346 - a test for return within finally +* Lib/test/test_sys_settrace.py:1346 - a test for ``return`` within ``finally`` -There are no uses of break within a finally (that break out of the finally) +There are no uses of ``break`` within a ``finally`` (that break out of the ``finally``) in the standard library. Security Implications @@ -207,20 +207,20 @@ How to Teach This This feature is very rarely used so forbidding it will likely only impact advanced users, not beginners and probably not any existing teaching -material. Since this is the removal of a feature teaching users will be -one by the raising of a SyntaxError if/when the forbidden feature is used. +material. Since this is the removal of a feature teaching users will be +one by the raising of a ``SyntaxError`` if/when the forbidden feature is used. Reference Implementation ======================== There is currently no reference implementation, although the way continue -is currently handled in a finally (raising a SyntaxError) can be extended -to return and break. +is currently handled in a ``finally`` (raising a ``SyntaxError``) can be extended +to ``return`` and ``break``. References ========== -.. [1] https://bugs.python.org/issue37830 +.. [1] https://github.com/python/cpython/issues/82011 .. [2] https://stackoverflow.com/questions/48088/returning-from-a-finally-block-in-java @@ -228,9 +228,9 @@ References .. [4] https://wiki.sei.cmu.edu/confluence/display/java/ERR04-J.+Do+not+complete+abruptly+from+a+finally+block -.. [5] https://github.com/rubocop-hq/rubocop/issues/5949 +.. [5] https://github.com/rubocop/rubocop/issues/5949 -.. [6] https://www.rubydoc.info/gems/rubocop/RuboCop/Cop/Lint/EnsureReturn +.. [6] https://www.rubydoc.info/gems/rubocop/0.74.0/RuboCop/Cop/Lint/EnsureReturn .. [7] https://eslint.org/docs/rules/no-unsafe-finally @@ -243,14 +243,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/peps/pep-0602-example-release-calendar.png b/peps/pep-0602-example-release-calendar.png new file mode 100644 index 000000000..fdbf1a0f5 Binary files /dev/null and b/peps/pep-0602-example-release-calendar.png differ diff --git a/pep-0602-example-release-calendar.pptx b/peps/pep-0602-example-release-calendar.pptx similarity index 100% rename from pep-0602-example-release-calendar.pptx rename to peps/pep-0602-example-release-calendar.pptx diff --git a/peps/pep-0602-overlapping-support-matrix.png b/peps/pep-0602-overlapping-support-matrix.png new file mode 100644 index 000000000..4620a10de Binary files /dev/null and b/peps/pep-0602-overlapping-support-matrix.png differ diff --git a/pep-0602-overlapping-support-matrix.pptx b/peps/pep-0602-overlapping-support-matrix.pptx similarity index 100% rename from pep-0602-overlapping-support-matrix.pptx rename to peps/pep-0602-overlapping-support-matrix.pptx diff --git a/pep-0602.rst b/peps/pep-0602.rst similarity index 94% rename from pep-0602.rst rename to peps/pep-0602.rst index ef5f8bf0b..2cbf8c29c 100644 --- a/pep-0602.rst +++ b/peps/pep-0602.rst @@ -3,7 +3,7 @@ Title: Annual Release Cycle for Python Version: $Revision$ Last-Modified: $Date$ Author: Ɓukasz Langa -BDFL-Delegate: Brett Cannon (on behalf of the steering council) +PEP-Delegate: Brett Cannon Discussions-To: https://discuss.python.org/t/pep-602-annual-release-cycle-for-python/2296/ Status: Accepted Type: Informational @@ -17,15 +17,15 @@ Abstract This document describes a change in the release calendar for Python starting with Python 3.9. This change accelerates the release cadence -such that major versions are released predictably every twelve months, +such that feature versions are released predictably every twelve months, in October every year. Implementation ============== -Seventeen months to develop a major version -------------------------------------------- +Seventeen months to develop a feature version +--------------------------------------------- This PEP proposes that Python 3.X.0 will be developed for around 17 months: @@ -65,7 +65,7 @@ Annual release cadence Feature development of Python 3.(X+1).0 starts as soon as Python 3.X.0 Beta 1 is released. This creates a twelve-month delta -between major Python versions. +between Python feature versions. Example @@ -120,7 +120,7 @@ Deprecations The current policy around breaking changes assumes at least two releases before a deprecated feature is removed from Python or a ``__future__`` -behavior is enabled by default. This is documented in PEP 387. +behavior is enabled by default. This is documented in :pep:`387`. This PEP proposes to keep this policy of **at least** two releases before making a breaking change. @@ -128,7 +128,7 @@ before making a breaking change. The term of the Steering Council -------------------------------- -The current wording of PEP 13 states that "a new council is elected +The current wording of :pep:`13` states that "a new council is elected after each feature release". This PEP proposes to keep this policy as it will lead to a consistent election schedule. @@ -236,7 +236,7 @@ by one or two: Figure 2. Testing matrix in the 18-month cadence vs. the 12-month The "extended bugfix support at the discretion of the Release Manager" -stage of the current release cycle is not codified. If fact, PEP 101 +stage of the current release cycle is not codified. If fact, :pep:`101` currently states that after the release of Python 3.(X+1).0 only one last bugfix release is made for Python 3.X.0. However, in practice at least the last four versions of Python 3 overlapped with stable releases @@ -288,10 +288,10 @@ More importantly, from the perspective of the user: users. The bigger a release is feature-wise, the more potential problems are hiding in "point zero releases". -Double the release cadence to achieve 9 months between major versions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Double the release cadence to achieve 9 months between feature versions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This was originally proposed in PEP 596 and rejected as both too +This was originally proposed in :pep:`596` and rejected as both too irregular and too short. This would not give any of the benefits of a regular release calendar but it would shorten all development phases, especially the beta + RC phases. This was considered dangerous. @@ -311,7 +311,7 @@ better. Slow down releases but don't freeze feature development with Beta 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This is described in PEP 598. This proposal includes non-standard +This is described in :pep:`598`. This proposal includes non-standard concepts like the "incremental feature release" which makes it hard to understand. The presented advantages are unclear while the unfamiliarity of the scheme poses a real risk of user and integrator diff --git a/pep-0603-hamt_vs_dict.png b/peps/pep-0603-hamt_vs_dict.png similarity index 100% rename from pep-0603-hamt_vs_dict.png rename to peps/pep-0603-hamt_vs_dict.png diff --git a/pep-0603-lookup_hamt.png b/peps/pep-0603-lookup_hamt.png similarity index 100% rename from pep-0603-lookup_hamt.png rename to peps/pep-0603-lookup_hamt.png diff --git a/pep-0603.rst b/peps/pep-0603.rst similarity index 98% rename from pep-0603.rst rename to peps/pep-0603.rst index f91a89a6e..1ca9d1bd0 100644 --- a/pep-0603.rst +++ b/peps/pep-0603.rst @@ -3,11 +3,11 @@ Title: Adding a frozenmap type to collections Version: $Revision$ Last-Modified: $Date$ Author: Yury Selivanov +Discussions-To: https://discuss.python.org/t/pep-603-adding-a-frozenmap-type-to-collections/2318/ Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 12-Sep-2019 -Python-Version: 3.9 Post-History: 12-Sep-2019 @@ -298,6 +298,7 @@ Performance .. figure:: pep-0603-hamt_vs_dict.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 1. Benchmark code can be found here: [3]_. @@ -312,6 +313,7 @@ The above chart demonstrates that: .. figure:: pep-0603-lookup_hamt.png :align: center :width: 100% + :class: invert-in-dark-mode Figure 2. Benchmark code can be found here: [4]_. diff --git a/pep-0604.rst b/peps/pep-0604.rst similarity index 94% rename from pep-0604.rst rename to peps/pep-0604.rst index 0796e94da..b3d773106 100644 --- a/pep-0604.rst +++ b/peps/pep-0604.rst @@ -6,6 +6,7 @@ BDFL-Delegate: Guido van Rossum Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 28-Aug-2019 Python-Version: 3.10 @@ -23,10 +24,10 @@ writing ``Union[X, Y]`` as ``X | Y``, and allows it to appear in Motivation ========== -PEP 484 and PEP 526 propose a generic syntax to add typing to variables, -parameters and function returns. PEP 585 proposes to `expose +:pep:`484` and :pep:`526` propose a generic syntax to add typing to variables, +parameters and function returns. :pep:`585` proposes to :pep:`expose parameters to generics at runtime -`_. +<585#parameters-to-generics-are-available-at-runtime>`. Mypy [1]_ accepts a syntax which looks like:: annotation: name_type @@ -144,7 +145,7 @@ In some situations, some exceptions will not be raised as expected. If a metaclass implements the ``__or__`` operator, it will override this:: >>> class M(type): - ... def __or__(self, other): return "Hello" + ... def __or__(self, other): return "Hello" ... >>> class C(metaclass=M): pass ... @@ -185,7 +186,7 @@ CONS: 2. Change only PEP 484 (Type hints) to accept the syntax ``type1 | type2`` ? ---------------------------------------------------------------------------- -PEP 563 (Postponed Evaluation of Annotations) is enough to accept this proposition, +:pep:`563` (Postponed Evaluation of Annotations) is enough to accept this proposition, if we accept to not be compatible with the dynamic evaluation of annotations (``eval()``). :: diff --git a/peps/pep-0605-example-release-calendar.png b/peps/pep-0605-example-release-calendar.png new file mode 100644 index 000000000..f7998d240 Binary files /dev/null and b/peps/pep-0605-example-release-calendar.png differ diff --git a/peps/pep-0605-overlapping-support-matrix.png b/peps/pep-0605-overlapping-support-matrix.png new file mode 100644 index 000000000..f01c2c69a Binary files /dev/null and b/peps/pep-0605-overlapping-support-matrix.png differ diff --git a/pep-0605.rst b/peps/pep-0605.rst similarity index 97% rename from pep-0605.rst rename to peps/pep-0605.rst index bd4eb64a7..e76f1a7da 100644 --- a/pep-0605.rst +++ b/peps/pep-0605.rst @@ -9,12 +9,12 @@ Type: Informational Content-Type: text/x-rst Created: 20-Sep-2019 Python-Version: 3.9 -Post-History: 1-Oct-2019, 6-Oct-2019, 20-Oct-2019 +Post-History: 01-Oct-2019, 06-Oct-2019, 20-Oct-2019 Rejection Notice ================ -This PEP was rejected in favour of PEP 602. The potential alpha/beta alternation +This PEP was rejected in favour of :pep:`602`. The potential alpha/beta alternation was deemed too confusing and the two year cadence between releases deemed too long. @@ -24,7 +24,7 @@ Abstract For a long time, CPython's nominal feature release cadence has been "every 18-24 months", and in recent years, has been pretty consistently on the "18 month" -end of that window. PEP 607 provides some common background on the problems +end of that window. :pep:`607` provides some common background on the problems that arise due to that choice of cadence, as well as some of the risks that need to be accounted for when proposing to change it. @@ -155,7 +155,7 @@ and then linked from each of the Python 3.9 alpha and beta announcements. PEP 605: Changes to the pre-release management process ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As detailed in PEP 605, the pre-release management process has been updated to +As detailed in :pep:`605`, the pre-release management process has been updated to produce a rolling series of beta releases that are considered suitable for production use in environments with sufficiently robust integration testing and operational monitoring capabilities. @@ -174,7 +174,7 @@ ignored by post-freeze interpreter builds. The full CPython ABI will be frozen, and the pre-release flag dropped from the ABI flags, in 3.9.0rc1, which is expected to occur 2 months prior to the final -3.9.0 release (refer to the release schedule in PEP 596 for exact target dates). +3.9.0 release (refer to the release schedule in :pep:`596` for exact target dates). For application developers, migrating to the rolling release stream provides the opportunity to be actively involved in the design and development of @@ -336,7 +336,7 @@ application containers, also make it easier to combine an application with a language runtime in a CI pipeline, and then keep them together until the entire container image is later replaced by an updated one. -In light of those changes in the wider environment, PEP 602 has proposed +In light of those changes in the wider environment, :pep:`602` has proposed reducing the feature delivery latency for the Python standard library and CPython reference interpreter by increasing the frequency of CPython feature releases from every 18-24 months to instead occur every 12 months. @@ -361,7 +361,7 @@ more variants in widespread use can make it more difficult to determine when a fault report is an actual error in the project, or an issue in the reporting user's environment. -PEP 602 proposes that affected organisations and projects simply switch to +:pep:`602` proposes that affected organisations and projects simply switch to adopting every second or third CPython release, rather than attempting to adopt every release, but that creates its own set of new problems to be resolved, both practical (e.g. deprecations would need to cover more than one release if we're @@ -370,7 +370,7 @@ number of versions in active use, there is a much higher chance that open source library maintainers will receive bug reports that only occur on Python versions that they're not using themselves). -PEP 598 was an initial attempt by one of the authors of this PEP to propose +:pep:`598` was an initial attempt by one of the authors of this PEP to propose an alternative scheme to reduce feature delivery latency by adopting a semantic versioning style policy that allowed for the incremental delivery of backwards compatible features within a release series, until that series @@ -378,7 +378,7 @@ reached feature complete status. That variant still had the undesirable consequence of imposing visible changes on end users that are happy enough with the current release management model. -This PEP takes the view that both PEP 598 and PEP 602 share a common flaw: they +This PEP takes the view that both :pep:`598` and :pep:`602` share a common flaw: they are attempting to satisfy the needs of two quite distinct audiences within the constraints of a single release model, which results in conflicting design requirements, and the need for awkward trade-offs between those conflicting @@ -418,7 +418,7 @@ for almost all Python users and contributors: * for users of the new incremental feature release stream, targeting the pre-release phase allows for even lower feature delivery latency than the - annual cadence proposed in PEP 602; + annual cadence proposed in :pep:`602`; * for core developers working on new features, increased frequency and adoption of pre-releases should improve pre-release feedback cycles; * for users of the established release stream, the increased adoption and @@ -438,13 +438,13 @@ for almost all Python users and contributors: That said, it is acknowledged that not all the outcomes of this proposal will be beneficial for all members of the wider Python ecosystem: -* for Python library maintainers, both this PEP and PEP 602 would likely +* for Python library maintainers, both this PEP and :pep:`602` would likely result in user pressure to support the faster release cadence. While this PEP attempts to mitigate that by clearly marking which pre-releases include - potentially breaking changes to the full CPython C ABI, and PEP 602 attempts + potentially breaking changes to the full CPython C ABI, and :pep:`602` attempts to mitigate it by keeping the minimum time between full releases at 12 months, it isn't possible to eliminate this downside completely; -* for third party extension module maintainers, both this PEP and PEP 602 would +* for third party extension module maintainers, both this PEP and :pep:`602` would likely result in user pressure to start supporting the stable ABI in order to provide wheel archives that work on the new version as soon as it is available. Whether that's a net negative or not will depend on how the request @@ -508,7 +508,7 @@ full CPython C ABI: built against the full CPython ABI in the preceding pre-release are expected to load correctly, as long as those modules abide by the following additional criteria: - + * the module must not be using any provisional or private C APIs (either from the previous stable release series, or the in development pre-release series) that were removed in this beta release, or were changed in an ABI incompatible @@ -621,7 +621,7 @@ the X.Y.0 release. Changes to management of the CPython stable C ABI ------------------------------------------------- -The CPython stable ABI [5_] makes the commitment that binary extension modules +The CPython stable ABI [5]_ makes the commitment that binary extension modules built against any particular CPython release will continue to work on future CPython releases that support the same stable ABI version (this version is currently ``abi3``). @@ -752,7 +752,7 @@ On Windows systems, the suffix for tagged ``pyd`` files in pre-release builds would include "p" as a pre-release marker immediately after the version number, giving markers like "cp39p-win_amd64". -A proposed reference implementation for this change is available at [4_] (Note: +A proposed reference implementation for this change is available at [4]_ (Note: at time of writing, that implementation had not yet been tested on Windows). @@ -892,11 +892,11 @@ to alternate between traditional stable releases (for 3.8.x, 3.10.x, etc), and release series that used the new rolling release cadence (for 3.9.x, 3.11.x, etc). -This idea suffers from the same core problem as PEP 598 and PEP 602: it imposes +This idea suffers from the same core problem as :pep:`598` and :pep:`602`: it imposes changes on end users that are happy with the status quo without offering them any clear compensating benefit. -It's also affected by one of the main concerns raised against PEP 598: at least +It's also affected by one of the main concerns raised against :pep:`598`: at least some core developers and end users strongly prefer that no particular semantics be assigned to the *value* of any of the numbers in a release version. These community members instead prefer that all the semantic significance be @@ -911,12 +911,12 @@ final release. Why not use Calendar Versioning for the rolling release stream? --------------------------------------------------------------- -Steve Dower's initial write-up of this proposal [1_] suggested the use of +Steve Dower's initial write-up of this proposal [1]_ suggested the use of calendar versioning for the rolling release stream (so the first rolling pre-release after Python 3.8.0 would have been Python 2019.12 rather than 3.9.0b1). -Paul Moore pointed out [2_] two major practical problems with that proposal: +Paul Moore pointed out [2]_ two major practical problems with that proposal: * it isn't going to be clear to users of the calendar-based versions where they stand in relation to the traditionally numbered versions @@ -1114,14 +1114,14 @@ Having a rolling pre-freeze release stream available may also make it more feasi for more CI providers to offer a "CPython beta release" testing option. At the moment, this feature is only available from CI providers that are willing and able to put the necessary time and effort into creating, testing, and publishing -their own builds from the CPython master branch (e.g. [6_]). +their own builds from the CPython master branch (e.g. [6]_). Implications for the proposed Scientific Python ecosystem support period ------------------------------------------------------------------------ Based on discussions at SciPy 2019, NEP (NumPy Enhancement Proposal) 29 has -been drafted [3_] to propose a common convention across the Scientific Python +been drafted [3]_ to propose a common convention across the Scientific Python ecosystem for dropping support for older Python versions. While the exact formulation of that policy is still being discussed, the draft @@ -1209,7 +1209,7 @@ with the specific proposal in this PEP is with Debian, as that has been released in the first half of odd-numbered years since 2005 (roughly 12 months offset from Ubuntu LTS releases). -With the annual release proposal in PEP 602, both Debian and Ubuntu LTS would +With the annual release proposal in :pep:`602`, both Debian and Ubuntu LTS would consistently get a system Python version that is around 6 months old, but would also consistently select different Python versions from each other. @@ -1221,7 +1221,7 @@ releases by the time the Linux distribution ships. If that situation does occur, and is deemed undesirable (but not sufficiently undesirable for *Debian* to choose to adjust their release timing), then that's where the additional complexity of the "incremental feature release" proposal -in PEP 598 may prove worthwhile. +in :pep:`598` may prove worthwhile. (Moving CPython releases to the same half of the year as the Debian and Ubuntu LTS releases would potentially help mitigate the problem, but also creates @@ -1335,7 +1335,7 @@ releases for a longer time without fear of breaking changes. Acknowledgements ================ -Thanks to Ɓukasz Langa for creating PEP 602 and prompting this discussion of +Thanks to Ɓukasz Langa for creating :pep:`602` and prompting this discussion of possible improvements to the CPython release cadence, and to Kyle Stanley and h-vetinari for constructive feedback on the initial draft of this PEP. @@ -1366,12 +1366,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 80 - coding: utf-8 - End: diff --git a/pep-0605/example-release-calendar.odp b/peps/pep-0605/example-release-calendar.odp similarity index 100% rename from pep-0605/example-release-calendar.odp rename to peps/pep-0605/example-release-calendar.odp diff --git a/pep-0605/overlapping-support-matrix.odp b/peps/pep-0605/overlapping-support-matrix.odp similarity index 100% rename from pep-0605/overlapping-support-matrix.odp rename to peps/pep-0605/overlapping-support-matrix.odp diff --git a/pep-0606.rst b/peps/pep-0606.rst similarity index 93% rename from pep-0606.rst rename to peps/pep-0606.rst index 87a9cde23..964c8d2dc 100644 --- a/pep-0606.rst +++ b/peps/pep-0606.rst @@ -76,16 +76,16 @@ The performance overhead of any compatibility code must be low when ``sys.set_python_compat_version()`` is not called. The C API is out of the scope of this PEP: ``Py_LIMITED_API`` macro and -the stable ABI are solving this problem differently, see the `PEP 384: -Defining a Stable ABI `_. +the stable ABI are solving this problem differently, see the :pep:`PEP 384: +Defining a Stable ABI <384>`. Security fixes which break backward compatibility on purpose will not get a compatibility layer; security matters more than compatibility. For example, ``http.client.HTTPSConnection`` was modified in Python 3.4.3 to performs all the necessary certificate and hostname checks by -default. It was a deliberate change motivated by `PEP 476: Enabling +default. It was a deliberate change motivated by :pep:`PEP 476: Enabling certificate verification by default for stdlib http clients -`_ (`bpo-22417 +<476>` (`bpo-22417 `_). The Python language does not provide backward compatibility. @@ -127,8 +127,8 @@ Python 3 is long, and it's getting longer with each Python 3.x release. Cleaning up Python and DeprecationWarning ----------------------------------------- -One of the `Zen of Python (PEP 20) -`_ motto is: +One of the :pep:`Zen of Python (PEP 20) +<20>` motto is: There should be one-- and preferably only one --obvious way to do it. @@ -136,7 +136,7 @@ One of the `Zen of Python (PEP 20) When Python evolves, new ways inevitably emerge. ``DeprecationWarning``\ s are emitted to suggest using the new way, but many developers ignore these warnings, which are silent by default (except in the ``__main__`` -module: see the `PEP 565 `_). +module: see the :pep:`565`). Some developers simply ignore all warnings when there are too many warnings, thus only bother with exceptions when the deprecated code is removed. @@ -393,11 +393,11 @@ means 6 ``.pyc`` files, instead of 3, to support Python 3.8 and Python Temporary moratorium on incompatible changes -------------------------------------------- -In 2009, PEP 3003 "Python Language Moratorium" proposed a +In 2009, :pep:`3003` "Python Language Moratorium" proposed a temporary moratorium (suspension) of all changes to the Python language syntax, semantics, and built-ins for Python 3.1 and Python 3.2. -In May 2018, during the PEP 572 discussions, it was also proposed to slow +In May 2018, during the :pep:`572` discussions, it was also proposed to slow down Python changes: see the python-dev thread `Slow down... `_ @@ -413,8 +413,8 @@ down Python changes: see the python-dev thread `Slow down... PEP 387 ------- -`PEP 387 -- Backwards Compatibility Policy -`_ proposes a process to make +:pep:`PEP 387 -- Backwards Compatibility Policy +<387>` proposes a process to make incompatible changes. The main point is the 4th step of the process: See if there's any feedback. Users not involved in the original @@ -424,11 +424,11 @@ incompatible changes. The main point is the 4th step of the process: PEP 497 ------- -`PEP 497 -- A standard mechanism for backward compatibility -`_ proposes different +:pep:`PEP 497 -- A standard mechanism for backward compatibility +<497>` proposes different solutions to provide backward compatibility. -Except for the ``__past__`` mechanism idea, PEP 497 does not propose +Except for the ``__past__`` mechanism idea, :pep:`497` does not propose concrete solutions: When an incompatible change to core language syntax or semantics is @@ -508,7 +508,7 @@ Examples of Python 3.7 incompatible changes: * ``asyncio`` no longer exports the ``selectors`` and ``_overlapped`` modules as ``asyncio.selectors`` and ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors``. -* PEP 479 is enabled for all code in Python 3.7, meaning that +* :pep:`479` is enabled for all code in Python 3.7, meaning that ``StopIteration`` exceptions raised directly or indirectly in coroutines and generators are transformed into ``RuntimeError`` exceptions. @@ -588,23 +588,23 @@ References Accepted PEPs: -* `PEP 5 -- Guidelines for Language Evolution - `_ -* `PEP 236 -- Back to the __future__ - `_ -* `PEP 411 -- Provisional packages in the Python standard library - `_ -* `PEP 3002 -- Procedure for Backwards-Incompatible Changes - `_ +* :pep:`PEP 5 -- Guidelines for Language Evolution + <5>` +* :pep:`PEP 236 -- Back to the __future__ + <236>` +* :pep:`PEP 411 -- Provisional packages in the Python standard library + <411>` +* :pep:`PEP 3002 -- Procedure for Backwards-Incompatible Changes + <3002>` Draft PEPs: -* `PEP 602 -- Annual Release Cycle for Python - `_ -* `PEP 605 -- A rolling feature release stream for CPython - `_ -* See also withdrawn `PEP 598 -- Introducing incremental feature - releases `_ +* :pep:`PEP 602 -- Annual Release Cycle for Python + <602>` +* :pep:`PEP 605 -- A rolling feature release stream for CPython + <605>` +* See also withdrawn :pep:`PEP 598 -- Introducing incremental feature + releases <598>` Copyright diff --git a/pep-0607.rst b/peps/pep-0607.rst similarity index 85% rename from pep-0607.rst rename to peps/pep-0607.rst index c8c6bc2a0..e1019ddde 100644 --- a/pep-0607.rst +++ b/peps/pep-0607.rst @@ -16,14 +16,14 @@ Post-History: 20-Oct-2019 Abstract ======== -PEP 602 and PEP 605 describe two alternative approaches to delivering smaller +:pep:`602` and :pep:`605` describe two alternative approaches to delivering smaller collections of features to Python's users more frequently (as compared to the current approach of offering new feature releases every 18-24 months, with the first binary alpha release taking place 6-8 months before the final release). Both PEPs also propose moving to a release cadence that results in full releases -occurring at a consistent time of year (every year for PEP 602, every other -year for PEP 605). +occurring at a consistent time of year (every year for :pep:`602`, every other +year for :pep:`605`). This PEP (from the authors of both competing proposals) provides common background on *why* a change in the release cadence is considered desirable, @@ -44,11 +44,11 @@ given that they include larger pieces of relatively untested code. The easiest way to simplify those investigations and reduce the likelihood of users encountering problems is to reduce the size of the batches being shipped. -PEP 602 proposes to address this problem via the straightforward approach of +:pep:`602` proposes to address this problem via the straightforward approach of reducing CPython's typical batch size by 50%, shipping 12 months of changes each time, rather than accumulating 18+ months of changes. -PEP 605 proposes to address it by regularly delivering 2 months worth of changes +:pep:`605` proposes to address it by regularly delivering 2 months worth of changes to a subset of Python's user base that opts in to running a rolling stream of beta releases (similar to running Windows Insider builds instead of the Windows retail release, or running Debian testing instead of Debian stable). @@ -62,10 +62,10 @@ long period of time between stable releases, it creates an incredibly strong temptation for developers to push changes into stable releases before they're really ready for general use. -PEP 602 proposes to address this problem by reducing the period of time +:pep:`602` proposes to address this problem by reducing the period of time between stable releases to 12 months rather than 18 months. -PEP 605 proposes to address it by actively creating a community of +:pep:`605` proposes to address it by actively creating a community of Python users that regularly install and use CPython beta releases, providing an incentive for core developers to start shipping changes earlier in the pre-release cycle, in order to obtain feedback before the feature gets locked @@ -84,10 +84,10 @@ individual volunteers and for corporate contributors, and also complicates alignment with events like PyCon US (typically April/May) and the now-annual core development sprints (typically in September). -PEP 602 proposes to address this problem by publishing a new release in October +:pep:`602` proposes to address this problem by publishing a new release in October every year, and basing the pre-release calendar for each year off that. -PEP 605 proposes to address this problem by alternating between release years +:pep:`605` proposes to address this problem by alternating between release years (where a new stable release is published in August), and non-release years (where only maintenance releases and new rolling beta releases are published). @@ -106,10 +106,10 @@ literally years to correct any design mistakes identified at that point. Marking APIs as provisional nominally offers a way to avoid that constraint, but actually taking advantage of that freedom causes other problems. -PEP 602 proposes to address this problem by starting the alpha period +:pep:`602` proposes to address this problem by starting the alpha period immediately after the previous stable release. -PEP 605 proposes to address this problem by actively promoting adoption of +:pep:`605` proposes to address this problem by actively promoting adoption of CPython pre-releases for running production workloads (not just for library and application compatibility testing), and adjusting the pre-release management process as necessary to make that a reasonable thing to do. @@ -143,12 +143,12 @@ published CPython release series (for example, Debian stable and Ubuntu LTS sometimes skip releases due to the mismatch between their 24-month release cycles and CPython's typically 18-month cycle). -The faster 12-month full release cadence in PEP 602 means that users in this +The faster 12-month full release cadence in :pep:`602` means that users in this category may end up skipping two releases where they would previously have only skipped one. However, the extended notice period for deprecations means that skipping a single release should no longer result in missed deprecation warnings. -The slower 24-month full release cadence in PEP 605 may move some of the users +The slower 24-month full release cadence in :pep:`605` may move some of the users that have historically been in this category into the "update to every stable release" category. @@ -159,7 +159,7 @@ Impact on users and redistributors that update to every release Many of Python's users never install a pre-release, but do update to every stable release series at some point after it is published. -PEP 602 aims to mitigate the potential negative impact on members of this group +:pep:`602` aims to mitigate the potential negative impact on members of this group by keeping the minimum gap between releases to 12 months, and retaining the 18 month full support period for each release. @@ -167,7 +167,7 @@ Keeping the 18-month full support period for each release branch means that the branches will spend roughly the same amount of time in full support and security-fix-only mode as they do now (~18 months and ~42 months, respectively). -PEP 605 aims to mitigate the potential negative impact on members of this group +:pep:`605` aims to mitigate the potential negative impact on members of this group by increasing use during the pre-release period to achieve more stable final releases with wider ecosystem support at launch. @@ -185,8 +185,8 @@ Despite the difficulties of doing so, there are already some users and redistributors that take on the challenge of using or publishing the CPython master branch directly. -Neither PEP 602 nor PEP 605 should directly affect this group, but the rolling -release stream proposal in PEP 605 aims to lower the barriers to more users +Neither :pep:`602` nor :pep:`605` should directly affect this group, but the rolling +release stream proposal in :pep:`605` aims to lower the barriers to more users adopting this style of usage, by allowing them to adopt the tested rolling beta stream, rather than needing to use the master branch directly. @@ -197,10 +197,10 @@ Impact on maintainers of third party libraries For maintainers of third party libraries, the key source of support complexity is the *number* of different Python versions in widespread use. -PEP 602 aims to mitigate the potential negative impact on members of this group +:pep:`602` aims to mitigate the potential negative impact on members of this group by keeping the minimum gap between full releases to 12 months. -PEP 605 aims to mitigate the potential negative impact on members of this group +:pep:`605` aims to mitigate the potential negative impact on members of this group by increasing the gap between full releases to 24 months, retaining the current policy of moving each release branch to security-fix-only mode not long after its successor is released, and retaining the "beta" naming scheme for the new diff --git a/pep-0608.rst b/peps/pep-0608.rst similarity index 95% rename from pep-0608.rst rename to peps/pep-0608.rst index e5acb14b5..43f17afcb 100644 --- a/pep-0608.rst +++ b/peps/pep-0608.rst @@ -59,8 +59,7 @@ before a feature can be removed. In practice, DeprecationWarning warnings are ignored for years in major Python projects. Usually, maintainers explain that there are too many warnings and so they simply ignore warnings. Moreover, DeprecationWarning -is silent by default (except in the ``__main__`` module: `PEP 565 -`_). +is silent by default (except in the ``__main__`` module: :pep:`565`). Even if more and more projects are running their test suite with warnings treated as errors (``-Werror``), Python core developers still @@ -86,10 +85,10 @@ all selected projects is available. Shorter Python release schedule ------------------------------- -The `PEP 602: Annual Release Cycle for Python -`_ and the `PEP 605: A +The :pep:`PEP 602: Annual Release Cycle for Python +<602>` and the :pep:`PEP 605: A rolling feature release stream for CPython -`_ would like to release +<605>` would like to release Python more often to ship new features more quickly. The problem is that each Python ``3.x`` release breaks many projects. @@ -203,8 +202,8 @@ Incompatible changes The definition here is large: any Python change which cause an issue when building or testing a project. -See also the `PEP 606: Python Compatibility Version -`_ for more examples of +See also the :pep:`PEP 606: Python Compatibility Version +<606>` for more examples of incompatible changes. Examples @@ -240,8 +239,8 @@ There are different kinds of incompatible changes: Cleaning up Python and DeprecationWarning ----------------------------------------- -One of the `Zen of Python (PEP 20) -`_ motto is: +One of the :pep:`Zen of Python (PEP 20) +<20>` motto is: There should be one-- and preferably only one --obvious way to do it. diff --git a/pep-0609.rst b/peps/pep-0609.rst similarity index 97% rename from pep-0609.rst rename to peps/pep-0609.rst index 159f2aff7..038807303 100644 --- a/pep-0609.rst +++ b/peps/pep-0609.rst @@ -1,15 +1,13 @@ PEP: 609 -Title: PyPA Governance -Version: $Revision$ -Last-Modified: $Date$ +Title: Python Packaging Authority (PyPA) Governance Author: Dustin Ingram , - Pradyun Gedam + Pradyun Gedam , Sumana Harihareswara Sponsor: Paul Ganssle -BDFL-Delegate: TBD -Discussions-To: https://discuss.python.org/t/pep-609-pypa-governance/ +Discussions-To: https://discuss.python.org/t/pep-609-pypa-governance/2619 Status: Active Type: Process +Topic: Governance, Packaging Content-Type: text/x-rst Created: 05-Nov-2019 Post-History: 05-Nov-2019 @@ -274,14 +272,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0610.rst b/peps/pep-0610.rst similarity index 92% rename from pep-0610.rst rename to peps/pep-0610.rst index f41e4616c..5e20ea55c 100644 --- a/pep-0610.rst +++ b/peps/pep-0610.rst @@ -6,16 +6,21 @@ BDFL-Delegate: Pradyun Gedam Discussions-To: https://discuss.python.org/t/recording-the-source-url-of-an-installed-distribution/1535 Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 21-Apr-2019 Post-History: Resolution: https://discuss.python.org/t/1535/56 + +.. canonical-pypa-spec:: :ref:`packaging:direct-url` + + Abstract ======== -Following PEP 440, a distribution can be identified by a name and either a -version, or a direct URL reference (see `PEP440 Direct References`_). +Following :pep:`440`, a distribution can be identified by a name and either a +version, or a direct URL reference (see :pep:`PEP440 Direct References <440#direct-references>`). After installation, the name and version are captured in the project metadata, but currently there is no way to obtain details of the URL used when the distribution was identified by a direct URL reference. @@ -23,17 +28,8 @@ distribution was identified by a direct URL reference. This proposal defines additional metadata, to be added to the installed distribution by the installation front end, which records the Direct URL Origin for use by -consumers which introspect the database of installed packages (see PEP 376). +consumers which introspect the database of installed packages (see :pep:`376`). -Provisional acceptance -====================== - -In accordance with the PyPA's specification process, this PEP has been -`provisionally accepted `_ -for initial implementation in ``pip`` and other PyPA tools. - -During this time, the specification is still subject to revision based -on real world experience with those initial implementations. Motivation ========== @@ -55,7 +51,7 @@ Python installers such as pip are capable of downloading and installing distributions from package indexes. They are also capable of downloading and installing source code from requirements specifying arbitrary URLs of source archives and Version Control Systems (VCS) repositories, -as standardized in `PEP440 Direct References`_. +as standardized in :pep:`PEP440 Direct References <440#direct-references>`. In other words, two relevant installation modes exist. @@ -70,7 +66,7 @@ In other words, two relevant installation modes exist. (typically a wheel, a source archive or a VCS repository) and installs it. In this mode, installers typically download the source code in a - temporary directory, invoke the PEP 517 build backend to produce a wheel + temporary directory, invoke the :pep:`517` build backend to produce a wheel if needed, install the wheel, and delete the temporary directory. After installation, no trace of the URL the user requested to download the @@ -132,7 +128,7 @@ revision corresponds to a branch, tag or (in the case of Git) a ref. This information can be used when introspecting the Database of Installed Distributions to communicate to users more information about what version was installed (e.g. whether a branch or tag was installed and, if so, the name of the -branch or tag). This also permits one to know whether a PEP 440 direct +branch or tag). This also permits one to know whether a :pep:`440` direct reference URL can be constructed using the tag form, as only tags have the semantics of immutability. @@ -158,7 +154,7 @@ directory, so the information can be recovered when running "freeze". The use of this workaround, although useful, is fragile, creates confusion about the purpose of the editable mode, and works only when the distribution -can be installed with setuptools (i.e. it is not usable with other PEP 517 +can be installed with setuptools (i.e. it is not usable with other :pep:`517` build backends). When this PEP is implemented, it will not be necessary anymore to use @@ -172,8 +168,9 @@ This PEP specifies a new ``direct_url.json`` metadata file in the ``.dist-info`` directory of an installed distribution. The fields specified are sufficient to reproduce the source archive and `VCS -URLs supported by pip`_. They are also sufficient to reproduce `PEP440 Direct -References`_, as well as `Pipfile and Pipfile.lock`_ entries. Finally, they +URLs supported by pip`_. They are also sufficient to reproduce +:pep:`PEP440 Direct References <440#direct-references>`, +as well as `Pipfile and Pipfile.lock`_ entries. Finally, they are sufficient to record the branch, tag, and/or Git ref origin of the installed version that is already available for editable installs by virtue of a VCS checkout being present. @@ -205,7 +202,7 @@ from a requirement specifying a direct URL reference (including a VCS URL). This file MUST NOT be created when installing a distribution from an other type of requirement (i.e. name plus version specifier). -This JSON file MUST be a dictionary, compliant with `RFC 8259`_ and UTF-8 +This JSON file MUST be a dictionary, compliant with :rfc:`8259` and UTF-8 encoded. If present, it MUST contain at least two fields. The first one is ``url``, with @@ -271,7 +268,7 @@ present as a dictionary with the following key: in editable mode, ``false`` otherwise. If absent, default to ``false``. When ``url`` refers to a local directory, it MUST have the ``file`` scheme -and be compliant with `RFC 8089`_. In particular, the path component must +and be compliant with :rfc:`8089`. In particular, the path component must be absolute. Symbolic links SHOULD be preserved when making relative paths absolute. @@ -515,22 +512,22 @@ there are no backwards compatibility implications. Alternatives ============ -PEP426 source_url ------------------ +PEP 426 source_url +------------------ -The now withdrawn PEP 426 specifies a ``source_url`` metadata entry. +The now withdrawn :pep:`426` specifies a ``source_url`` metadata entry. It is also implemented in `distlib`_. It was intended for a slightly different purpose, for use in sdists. This format lacks support for the ``subdirectory`` option of pip requirement -URLs. The same limitation is present in `PEP440 Direct References`_. +URLs. The same limitation is present in :pep:`PEP440 Direct References <440#direct-references>`. It also lacks explicit support for `environment variables in the user:password part of URLs`_. The introduction of a key/value extensibility mechanism and support -for environment variables for user:password in PEP 440, would be necessary +for environment variables for user:password in :pep:`440`, would be necessary for use in this PEP. revision vs ref @@ -545,14 +542,10 @@ References .. _`pip issue #609`: https://github.com/pypa/pip/issues/609 .. _`thread on discuss.python.org`: https://discuss.python.org/t/pip-freeze-vcs-urls-and-pep-517-feat-editable-installs/1473 -.. _PEP440: http://www.python.org/dev/peps/pep-0440 .. _`VCS URLs supported by pip`: https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support -.. _`PEP440 Direct References`: https://www.python.org/dev/peps/pep-0440/#direct-references .. _`Pipfile and Pipfile.lock`: https://github.com/pypa/pipfile .. _distlib: https://distlib.readthedocs.io .. _`environment variables in the user:password part of URLs`: https://pip.pypa.io/en/stable/reference/pip_install/#id10 -.. _`RFC 8259`: https://tools.ietf.org/html/rfc8259 -.. _`RFC 8089`: https://tools.ietf.org/html/rfc8089 .. _`Recording the Direct URL Origin of installed distributions`: https://packaging.python.org/specifications/direct-url Acknowledgements @@ -568,7 +561,8 @@ This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - .. + +.. Local Variables: mode: indented-text indent-tabs-mode: nil diff --git a/pep-0611.rst b/peps/pep-0611.rst similarity index 99% rename from pep-0611.rst rename to peps/pep-0611.rst index f5956bf0e..e267f6a6e 100644 --- a/pep-0611.rst +++ b/peps/pep-0611.rst @@ -38,7 +38,7 @@ It is unsafe as malicious or poorly generated code could cause values to exceed For example, line numbers are represented by 32 bit values internally. This is inefficient, given that modules almost never exceed a few thousand lines. -Despite being inefficient, is still vulnerable to overflow as +Despite being inefficient, it is still vulnerable to overflow as it is easy for an attacker to created a module with billions of newline characters. Memory access is usually a limiting factor in the performance of modern CPUs. diff --git a/pep-0612.rst b/peps/pep-0612.rst similarity index 97% rename from pep-0612.rst rename to peps/pep-0612.rst index 545f314cf..a5c51d3a0 100644 --- a/pep-0612.rst +++ b/peps/pep-0612.rst @@ -3,24 +3,23 @@ Title: Parameter Specification Variables Author: Mark Mendoza Sponsor: Guido van Rossum BDFL-Delegate: Guido van Rossum -Discussions-To: Typing-Sig +Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 18-Dec-2019 Python-Version: 3.10 Post-History: 18-Dec-2019, 13-Jul-2020 -Parameter Specification Variables -================================= Abstract -------- There currently are two ways to specify the type of a callable, the -``Callable[[int, str], bool]`` syntax defined in `PEP 484 -`_\ , and callback protocols from `PEP -544 `_. Neither of +``Callable[[int, str], bool]`` syntax defined in :pep:`484`, +and callback protocols from :pep:`PEP +544 <544#callback-protocols>`. Neither of these support forwarding the parameter types of one callable over to another callable, making it difficult to annotate function decorators. This PEP proposes ``typing.ParamSpec`` and ``typing.Concatenate`` to @@ -56,12 +55,12 @@ function, is an instance of the Python idiom of one function passing all arguments given to it over to another function. This is done through the combination of the ``*args`` and ``**kwargs`` features in both parameters and in arguments. When one defines a function (like ``inner``\ ) that takes ``(*args, -**kwargs)`` and goes on to call another function with ``(*args, **kwargs)``\ -, the wrapping function can only be safely called in all of the ways that the +**kwargs)`` and goes on to call another function with ``(*args, **kwargs)``, +the wrapping function can only be safely called in all of the ways that the wrapped function could be safely called. To type this decorator, we’d like to be able to place a dependency between the parameters of the callable ``f`` and the -parameters of the returned function. `PEP 484 -`_ supports dependencies between +parameters of the returned function. :pep:`484` +supports dependencies between single types, as in ``def append(l: typing.List[T], e: T) -> typing.List[T]: ...``\ , but there is no existing way to do so with a complicated entity like the parameters of a function. @@ -577,7 +576,7 @@ Reference Implementation The `Pyre `_ type checker supports all of the behavior described above. A reference implementation of the runtime components needed for those uses is provided in the ``pyre_extensions`` module. A reference -implementation for CPython can be found +implementation for CPython can be found `here `_. Rejected Alternatives @@ -703,8 +702,6 @@ We decided to go with ``ParamSpec``\ s over this approach for several reasons: In summary, between these two equivalently powerful syntaxes, ``ParamSpec`` fits much more naturally into the status quo. -.. _Concatenating Keyword Parameters: - Concatenating Keyword Parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/pep-0613.rst b/peps/pep-0613.rst similarity index 89% rename from pep-0613.rst rename to peps/pep-0613.rst index f11c56259..3cfdb9f57 100644 --- a/pep-0613.rst +++ b/peps/pep-0613.rst @@ -5,8 +5,10 @@ Sponsor: Guido van Rossum Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/MWRJOBEEEMFVXE7CAKO7B4P46IPM4AN3/ Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 21-Jan-2020 +Python-Version: 3.10 Post-History: 21-Jan-2020 @@ -22,7 +24,7 @@ Motivation ========== Type aliases are declared as top level variable assignments. -In `PEP 484 `_, +In :pep:`PEP 484 <484#type-aliases>`, the distinction between a valid type alias and a global variable was implicitly determined: if a top level assignment is unannotated, and the assigned value is a valid type, then the name being assigned to is a valid type alias. Otherwise, @@ -58,7 +60,7 @@ cannot be used as a return annotation because it is not a valid type. :: - MyType: TypeAlias = “ClassName” + MyType: TypeAlias = "ClassName" def foo() -> MyType: ... Explicit aliases remove ambiguity so neither of the above errors will be @@ -95,6 +97,17 @@ across the codebase can be suppressed. Scope Restrictions: ******************* +:: + + class Foo: + x = ClassName + y: TypeAlias = ClassName + z: Type[ClassName] = ClassName + +Type aliases are valid within class scope, both implicitly (``x``) and +explicitly (``y``). If the line should be interpreted as a class +variable, it must be explicitly annotated (``z``). + :: x = ClassName @@ -103,7 +116,7 @@ Scope Restrictions: The outer ``x`` is a valid type alias, but type checkers must error if the inner ``x`` is ever used as a type because type aliases cannot be defined -inside a nested scope. +inside of a function. This is confusing because the alias declaration rule is not explicit, and because a type error will not be thrown on the location of the inner type alias declaration but rather on every one of its subsequent use cases. @@ -116,10 +129,10 @@ but rather on every one of its subsequent use cases. def bar() -> None: x: TypeAlias = ClassName -With explicit aliases, the outer assignment is still a valid type variable, -and the inner assignment can either be a valid local variable or a clear error, -communicating to the author that type aliases cannot be defined inside a nested -scope. +With explicit aliases, the outer assignment is still a valid type variable. +Inside ``foo``, the inner assignment should be interpreted as ``x: Type[ClassName]``. +Inside ``bar``, the type checker should raise a clear error, communicating +to the author that type aliases cannot be defined inside a function. Specification @@ -153,7 +166,7 @@ Explicit syntax: x: Type[int] = int # typed global expression x: TypeAlias = int # type alias - x: TypeAlias = “MyClass” # type alias + x: TypeAlias = "MyClass" # type alias Note: The examples above illustrate implicit and explicit alias declarations in @@ -200,6 +213,14 @@ appealing because it still sticks with the ``MyType = int`` assignment syntax, and adds some information for the type checker purely as an annotation. +Version History +=============== + +* 2021-11-16 + + * Allow TypeAlias inside class scope + + Copyright ========= diff --git a/pep-0614.rst b/peps/pep-0614.rst similarity index 100% rename from pep-0614.rst rename to peps/pep-0614.rst diff --git a/pep-0615.rst b/peps/pep-0615.rst similarity index 96% rename from pep-0615.rst rename to peps/pep-0615.rst index 18dd4340c..dbac9575f 100644 --- a/pep-0615.rst +++ b/peps/pep-0615.rst @@ -7,7 +7,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 22-Feb-2020 Python-Version: 3.9 -Post-History: 2020-02-25, 2020-03-29 +Post-History: 25-Feb-2020, 29-Mar-2020 Replaces: 431 @@ -54,8 +54,8 @@ time zone database (also called the "tz" database or the Olson database [#tzdb-wiki]_). The time zone database is in the public domain and is widely distributed — it is present by default on many Unix-like operating systems. Great care goes into the stability of the database: there are IETF RFCs both -for the maintenance procedures (RFC 6557 [#rfc6557]_) and for the compiled -binary (TZif) format (RFC 8636 [#rfc8536]_). As such, it is likely that adding +for the maintenance procedures (:rfc:`6557`) and for the compiled +binary (TZif) format (:rfc:`8536`). As such, it is likely that adding support for the compiled outputs of the IANA database will add great value to end users even with the relatively long cadence of standard library releases. @@ -78,8 +78,6 @@ are grouped together by subject. The ``zoneinfo.ZoneInfo`` class ------------------------------- -.. _Constructors: - Constructors ############ @@ -162,7 +160,7 @@ should always be true: This is an alternate constructor that allows the construction of a ``ZoneInfo`` object from any TZif byte stream. This constructor takes an optional parameter, ``key``, which sets the name of the zone, for the purposes of -``__str__`` and ``__repr__`` (see Representations_). +``__str__`` and ``__repr__`` (see `Representations `_). Unlike the primary constructor, this always constructs a new object. There are two reasons that this deviates from the primary constructor's caching behavior: @@ -247,8 +245,6 @@ Manipulation of the cache behavior is expected to be a niche use case; this function is primarily provided to facilitate testing, and to allow users with unusual requirements to tune the cache invalidation behavior to their needs. -.. _Representations: - String representation ##################### @@ -642,7 +638,7 @@ minimal real world effects were it to occur. Including ``tzdata`` in the standard library by default ------------------------------------------------------- -Although PEP 453 [#pep453-ensurepip]_, which introduced the ``ensurepip`` +Although :pep:`453`, which introduced the ``ensurepip`` mechanism to CPython, provides a convenient template for a standard library module maintained on PyPI, a potentially similar ``ensuretzdata`` mechanism is somewhat less necessary, and would be complicated enough that it is considered @@ -673,9 +669,9 @@ is not used by the ``zoneinfo`` module. Using a ``pytz``-like interface ------------------------------- -A ``pytz``-like ([#pytz]_) interface was proposed in PEP 431 [#pep431]_, but +A ``pytz``-like ([#pytz]_) interface was proposed in :pep:`431`, but was ultimately withdrawn / rejected for lack of ambiguous datetime support. -PEP 495 [#pep495]_ added the ``fold`` attribute to address this problem, but +:pep:`495` added the ``fold`` attribute to address this problem, but ``fold`` obviates the need for ``pytz``'s non-standard ``tzinfo`` classes, and so a ``pytz``-like interface is no longer necessary. [#fastest-footgun]_ @@ -738,7 +734,7 @@ There are several other schemes that were considered and rejected: One advantage to this scheme would be that it would add a natural extension point for specifying non-file-based elements on the search path, such as changing the priority of ``tzdata`` if it exists, or if native support for - TZDIST [#rfc7808]_ were to be added to the library in the future. + :rfc:`TZDIST <7808>` were to be added to the library in the future. This was rejected mainly because these sort of special values are not usually found in ``PATH``-like variables and the only currently proposed use @@ -814,7 +810,7 @@ Arguments against this: imported), and it is a perennial source of confusion for end users. This confusing requirement from end-users can be avoided using a - module-level ``__getattr__`` and ``__dir__`` per PEP 562, but this would + module-level ``__getattr__`` and ``__dir__`` per :pep:`562`, but this would add some complexity to the implementation of the ``datetime`` module. This sort of behavior in modules or classes tends to confuse static analysis tools, which may not be desirable for a library as widely used and critical @@ -864,18 +860,6 @@ Footnotes References ========== -.. [#rfc6557] - RFC 6557: Procedures for Maintaining the Time Zone Database - https://tools.ietf.org/html/rfc6557 - -.. [#rfc7808] - RFC 7808: Time Zone Data Distribution Service - https://tools.ietf.org/html/rfc7808 - -.. [#rfc8536] - RFC 8536: The Time Zone Information Format (TZif) - https://tools.ietf.org/html/rfc8536 - .. [#nontransitive_comp] Paul Ganssle: "A curious case of non-transitive datetime comparison" (Published 15 February 2018) @@ -915,18 +899,6 @@ References Davis) https://pyfound.blogspot.com/2019/05/russell-keith-magee-python-on-other.html -.. [#pep453-ensurepip] - PEP 453: Explicit bootstrapping of pip in Python installations - https://www.python.org/dev/peps/pep-0453/ - -.. [#pep431] - PEP 431: Time zone support improvements - https://www.python.org/dev/peps/pep-0431/ - -.. [#pep495] - PEP 495: Local Time Disambiguation - https://www.python.org/dev/peps/pep-0495/ - .. [#tzinfo] ``datetime.tzinfo`` documentation https://docs.python.org/3/library/datetime.html#datetime.tzinfo diff --git a/pep-0616.rst b/peps/pep-0616.rst similarity index 99% rename from pep-0616.rst rename to peps/pep-0616.rst index 3890d9419..73b915100 100644 --- a/pep-0616.rst +++ b/peps/pep-0616.rst @@ -2,7 +2,7 @@ PEP: 616 Title: String methods to remove prefixes and suffixes Author: Dennis Sweeney Sponsor: Eric V. Smith -Status: Accepted +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 19-Mar-2020 @@ -232,7 +232,7 @@ rejected on the following grounds: * This behavior can be surprising or visually confusing, especially when one prefix is empty or is a substring of another prefix, as in - ``'FooBar'.removeprefix(('', 'Foo')) == 'Foo'`` + ``'FooBar'.removeprefix(('', 'Foo')) == 'FooBar'`` or ``'FooBar text'.removeprefix(('Foo', 'FooBar ')) == 'Bar text'``. * The API for ``str.replace()`` only accepts a single pair of diff --git a/pep-0617.rst b/peps/pep-0617.rst similarity index 97% rename from pep-0617.rst rename to peps/pep-0617.rst index 08d074535..35638e492 100644 --- a/pep-0617.rst +++ b/peps/pep-0617.rst @@ -1,11 +1,9 @@ PEP: 617 Title: New PEG parser for CPython -Version: $Revision$ -Last-Modified: $Date$ Author: Guido van Rossum , - Pablo Galindo , - Lysandros Nikolaou -Discussions-To: Python-Dev + Pablo Galindo , + Lysandros Nikolaou +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -13,6 +11,8 @@ Created: 24-Mar-2020 Python-Version: 3.9 Post-History: 02-Apr-2020 +.. highlight:: PEG + ======== Overview ======== @@ -40,7 +40,7 @@ and the *follow sets* of the grammar: * Given a rule, the *first set* is the collection of all terminals that can occur first in a full derivation of that rule. Intuitively, this helps the parser decide among the alternatives in a rule. For - instance, given the rule :: + instance, given the rule:: rule: A | B @@ -163,7 +163,7 @@ an LL(1) parser) several rules are not LL(1) and several workarounds are implemented in the grammar and in other parts of CPython to deal with this. For example, consider the rule for assignment expressions:: - namedexpr_test: NAME [':=' test] + namedexpr_test: [NAME ':='] test This simple rule is not compatible with the Python grammar as *NAME* is among the elements of the *first set* of the rule *test*. To work around this limitation the @@ -298,13 +298,13 @@ and "hidden left-recursion" like:: Syntax ------ -The grammar consists of a sequence of rules of the form: :: +The grammar consists of a sequence of rules of the form:: rule_name: expression Optionally, a type can be included right after the rule name, which specifies the return type of the C or Python function corresponding to -the rule: :: +the rule:: rule_name[return_type]: expression @@ -324,7 +324,7 @@ Python-style comments. Match e1, then match e2. -:: +.. code:: PEG rule_name: first_rule second_rule @@ -339,7 +339,7 @@ The first alternative can also appear on the line after the rule name for formatting purposes. In that case, a \| must be used before the first alternative, like so: -:: +.. code:: PEG rule_name[return_type]: | first_alt @@ -350,14 +350,14 @@ first alternative, like so: Match e. -:: +.. code:: PEG rule_name: (e) A slightly more complex and useful example includes using the grouping operator together with the repeat operators: -:: +.. code:: PEG rule_name: (e1 e2)* @@ -366,14 +366,14 @@ operator together with the repeat operators: Optionally match e. -:: +.. code:: PEG rule_name: [e] A more useful example includes defining that a trailing comma is optional: -:: +.. code:: PEG rule_name: e (',' e)* [','] @@ -384,7 +384,7 @@ optional: Match zero or more occurrences of e. -:: +.. code:: PEG rule_name: (e1 e2)* @@ -395,7 +395,7 @@ Match zero or more occurrences of e. Match one or more occurrences of e. -:: +.. code:: PEG rule_name: (e1 e2)+ @@ -406,7 +406,7 @@ Match one or more occurrences of e, separated by s. The generated parse tree does not include the separator. This is otherwise identical to ``(e (s e)*)``. -:: +.. code:: PEG rule_name: ','.e+ @@ -428,7 +428,7 @@ An example taken from the proposed Python grammar specifies that a primary consists of an atom, which is not followed by a ``.`` or a ``(`` or a ``[``: -:: +.. code:: PEG primary: atom !'.' !'(' !'[' @@ -439,7 +439,7 @@ consists of an atom, which is not followed by a ``.`` or a ``(`` or a Commit to the current alternative, even if it fails to parse. -:: +.. code:: PEG rule_name: '(' ~ some_rule ')' | some_alt @@ -451,7 +451,7 @@ Variables in the Grammar ~~~~~~~~~~~~~~~~~~~~~~~~ A subexpression can be named by preceding it with an identifier and an -``=`` sign. The name can then be used in the action (see below), like this: :: +``=`` sign. The name can then be used in the action (see below), like this:: rule_name[return_type]: '(' a=some_other_rule ')' { a } @@ -469,7 +469,7 @@ files should be written, each one with a different set of actions, keeping everything else apart from said actions identical in both files. As an example of a grammar with Python actions, the piece of the parser generator that parses grammar files is bootstrapped from a meta-grammar file with -Python actions that generate the grammar tree as a result of the parsing. +Python actions that generate the grammar tree as a result of the parsing. In the specific case of the new proposed PEG grammar for Python, having actions allows directly describing how the AST is composed in the grammar @@ -496,7 +496,7 @@ with all the parsed expressions gets returned (this is meant for debugging). The full meta-grammar for the grammars supported by the PEG generator is: -:: +.. code:: PEG start[Grammar]: grammar ENDMARKER { grammar } @@ -589,25 +589,25 @@ As an illustrative example this simple grammar file allows directly generating a full parser that can parse simple arithmetic expressions and that returns a valid C-based Python AST: -:: +.. code:: PEG start[mod_ty]: a=expr_stmt* $ { Module(a, NULL, p->arena) } expr_stmt[stmt_ty]: a=expr NEWLINE { _Py_Expr(a, EXTRA) } - expr[expr_ty]: + expr[expr_ty]: | l=expr '+' r=term { _Py_BinOp(l, Add, r, EXTRA) } | l=expr '-' r=term { _Py_BinOp(l, Sub, r, EXTRA) } | t=term { t } - term[expr_ty]: + term[expr_ty]: | l=term '*' r=factor { _Py_BinOp(l, Mult, r, EXTRA) } | l=term '/' r=factor { _Py_BinOp(l, Div, r, EXTRA) } | f=factor { f } - factor[expr_ty]: + factor[expr_ty]: | '(' e=expr ')' { e } | a=atom { a } - atom[expr_ty]: + atom[expr_ty]: | n=NAME { n } | n=NUMBER { n } | s=STRING { s } @@ -619,7 +619,7 @@ for the parser. A similar grammar written to target Python AST objects: -:: +.. code:: PEG start: expr NEWLINE? ENDMARKER { ast.Expression(expr) } expr: @@ -743,7 +743,7 @@ parser due to encoding issues, sometimes intentional.) - Compiling and throwing away the internal AST took 2.141 seconds. That's 350,040 lines/sec, or 12,899,367 bytes/sec. The max RSS was - 74 MiB (the largest file in the stdlib is much smaller than out + 74 MiB (the largest file in the stdlib is much smaller than our canonical test file). - Compiling to bytecode took 3.290 seconds. That's 227,861 lines/sec, @@ -787,17 +787,16 @@ References ========== .. [1] Ford, Bryan - http://pdos.csail.mit.edu/~baford/packrat/thesis + https://pdos.csail.mit.edu/~baford/packrat/thesis/ .. [2] Medeiros et al. https://arxiv.org/pdf/1207.0443.pdf .. [3] Warth et al. - http://web.cs.ucla.edu/~todd/research/pepm08.pdf + https://web.cs.ucla.edu/~todd/research/pepm08.pdf -.. [#GUIDO_PEG] - Guido's series on PEG parsing - https://medium.com/@gvanrossum_83706/peg-parsing-series-de5d41b2ed60 +[4] Guido's series on PEG parsing +\ https://medium.com/@gvanrossum_83706/peg-parsing-series-de5d41b2ed60 ========= Copyright diff --git a/pep-0618.rst b/peps/pep-0618.rst similarity index 100% rename from pep-0618.rst rename to peps/pep-0618.rst diff --git a/pep-0619.rst b/peps/pep-0619.rst similarity index 74% rename from pep-0619.rst rename to peps/pep-0619.rst index 0c70e3dc4..e60052486 100644 --- a/pep-0619.rst +++ b/peps/pep-0619.rst @@ -1,10 +1,9 @@ PEP: 619 Title: Python 3.10 Release Schedule -Version: $Revision$ -Last-Modified: $Date$ Author: Pablo Galindo Salgado -Status: Draft +Status: Active Type: Informational +Topic: Release Content-Type: text/x-rst Created: 25-May-2020 Python-Version: 3.10 @@ -19,11 +18,10 @@ items. .. Small features may be added up to the first beta release. Bugs may be fixed until the final release, - which is planned for end of October 2021. + which is planned for October 2021. Release Manager and Crew ======================== - - 3.10 Release Manager: Pablo Galindo Salgado - Windows installers: Steve Dower - Mac installers: Ned Deily @@ -37,14 +35,14 @@ Release Schedule --------------- Note: the dates below use a 17-month development period that results -in a 12-month release cadence between major versions, as defined by -PEP 602. +in a 12-month release cadence between feature versions, as defined by +:pep:`602`. Actual: - 3.10 development begins: Monday, 2020-05-18 - 3.10.0 alpha 1: Monday, 2020-10-05 -- 3.10.0 alpha 2: Monday, 2020-11-03 +- 3.10.0 alpha 2: Tuesday, 2020-11-03 - 3.10.0 alpha 3: Monday, 2020-12-07 - 3.10.0 alpha 4: Monday, 2021-01-04 - 3.10.0 alpha 5: Wednesday, 2021-02-03 @@ -64,27 +62,33 @@ Bugfix releases Actual: -Expected: - - 3.10.1: Monday, 2021-12-06 -- 3.10.2: Monday, 2022-02-07 -- 3.10.3: Monday, 2022-04-04 -- 3.10.4: Monday, 2022-06-06 -- 3.10.5: Monday, 2022-08-01 -- 3.10.6: Monday, 2022-10-03 -- 3.10.7: Monday, 2022-12-05 -- 3.10.8: Monday, 2023-02-06 +- 3.10.2: Friday, 2022-01-14 +- 3.10.3: Wednesday, 2022-03-16 +- 3.10.4: Thursday, 2022-03-24 +- 3.10.5: Monday, 2022-06-06 +- 3.10.6: Tuesday, 2022-08-02 +- 3.10.7: Tuesday, 2022-09-06 +- 3.10.8: Tuesday, 2022-10-11 +- 3.10.9: Tuesday, 2022-12-06 +- 3.10.10: Wednesday, 2023-02-08 +- 3.10.11: Wednesday, 2023-04-05 (final regular bugfix release with binary + installers) -Final regular bugfix release with binary installers: +Source-only security fix releases +--------------------------------- -- 3.10.9: Monday, 2023-04-03 +Provided irregularly on an "as-needed" basis until October 2026. + +- 3.10.12: Tuesday, 2023-06-06 +- 3.10.13: Thursday, 2023-08-24 3.10 Lifespan ------------- 3.10 will receive bugfix updates approximately every 2 months for approximately 18 months. Some time after the release of 3.11.0 final, -the ninth and final 3.10 bugfix update will be released. After that, +the 11th and final 3.10 bugfix update will be released. After that, it is expected that security updates (source only) will be released until 5 years after the release of 3.10 final, so until approximately October 2026. @@ -110,13 +114,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 72 - coding: utf-8 - End: diff --git a/pep-0620.rst b/peps/pep-0620.rst similarity index 96% rename from pep-0620.rst rename to peps/pep-0620.rst index d1f7ebbaf..654ac13b8 100644 --- a/pep-0620.rst +++ b/peps/pep-0620.rst @@ -1,11 +1,11 @@ PEP: 620 Title: Hide implementation details from the C API Author: Victor Stinner -Status: Draft +Status: Withdrawn Type: Standards Track Content-Type: text/x-rst Created: 19-Jun-2020 -Python-Version: 3.10 +Python-Version: 3.12 Abstract ======== @@ -27,6 +27,15 @@ changes are already completed. The `Process to reduce the number of broken C extensions`_ dictates the rhythm. +PEP withdrawn +============= + +This PEP was withdrawn by its author since the scope is too broad and the work is +distributed over multiple Python versions, which makes it difficult to make +a decision on the overall PEP. It was split into new PEPs with +narrower and better defined scopes, like :pep:`670`. + + Motivation ========== @@ -36,9 +45,9 @@ The C API blocks CPython evolutions Adding or removing members of C structures is causing multiple backward compatibility issues. -Adding a new member breaks the stable ABI (PEP 384), especially for +Adding a new member breaks the stable ABI (:pep:`384`), especially for types declared statically (e.g. ``static PyTypeObject MyType = -{...};``). In Python 3.4, the PEP 442 "Safe object finalization" added +{...};``). In Python 3.4, the :pep:`442` "Safe object finalization" added the ``tp_finalize`` member at the end of the ``PyTypeObject`` structure. For ABI backward compatibility, a new ``Py_TPFLAGS_HAVE_FINALIZE`` type flag was required to announce if the type structure contains the @@ -163,12 +172,12 @@ Hiding implementation details from the C API has multiple advantages: Relationship with the limited C API ----------------------------------- -The PEP 384 "Defining a Stable ABI" is implemented in Python 3.4. It introduces the +The :pep:`384` "Defining a Stable ABI" is implemented in Python 3.4. It introduces the "limited C API": a subset of the C API. When the limited C API is used, it becomes possible to build a C extension only once and use it on multiple Python versions: that's the stable ABI. -The main limitation of the PEP 384 is that C extensions have to opt-in +The main limitation of the :pep:`384` is that C extensions have to opt-in for the limited C API. Only very few projects made this choice, usually to ease distribution of binaries, especially on Windows. @@ -313,7 +322,7 @@ not only ``PyObject*``, to avoid compiler warnings, since most macros cast their parameters to ``PyObject*``. Python 3.6 requires C compilers to support static inline functions: the -PEP 7 requires a subset of C99. +:pep:`7` requires a subset of C99. **STATUS**: Completed (in Python 3.9) @@ -377,7 +386,7 @@ Making ``PyTypeObject`` structure opaque breaks C extensions declaring types statically (e.g. ``static PyTypeObject MyType = {...};``). C extensions must use ``PyType_FromSpec()`` to allocate types on the heap instead. Using heap types has other advantages like being compatible -with subinterpreters. Combined with PEP 489 "Multi-phase extension +with subinterpreters. Combined with :pep:`489` "Multi-phase extension module initialization", it makes a C extension behavior closer to a Python module, like allowing to create more than one module instance. diff --git a/pep-0621.rst b/peps/pep-0621.rst similarity index 98% rename from pep-0621.rst rename to peps/pep-0621.rst index 6db3c6340..bd0a68bb8 100644 --- a/pep-0621.rst +++ b/peps/pep-0621.rst @@ -10,6 +10,7 @@ Author: Brett Cannon , Discussions-To: https://discuss.python.org/t/pep-621-round-3/5472 Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 22-Jun-2020 Post-History: 22-Jun-2020, @@ -19,6 +20,8 @@ Post-History: 22-Jun-2020, Resolution: https://discuss.python.org/t/pep-621-round-3/5472/109 +.. canonical-pypa-spec:: :ref:`packaging:declaring-project-metadata` + Abstract ======== @@ -282,7 +285,7 @@ meaning is open to interpretation. These fields accept an array of tables with 2 keys: ``name`` and ``email``. Both values must be strings. The ``name`` value MUST be a valid email name (i.e. whatever can be put as a name, before an email, -in `RFC #822`_) and not contain commas. The ``email`` value MUST be a +in :rfc:`822`) and not contain commas. The ``email`` value MUST be a valid email address. Both keys are optional. Using the data to fill in `core metadata`_ is as follows: @@ -466,7 +469,7 @@ Example keywords = ["egg", "bacon", "sausage", "tomatoes", "Lobster Thermidor"] authors = [ {email = "hi@pradyunsg.me"}, - {name = "Tzu-Ping Chung"} + {name = "Tzu-ping Chung"} ] maintainers = [ {name = "Brett Cannon", email = "brett@python.org"} @@ -490,10 +493,10 @@ Example ] [project.urls] - homepage = "example.com" - documentation = "readthedocs.org" - repository = "github.com" - changelog = "github.com/me/spam/blob/master/CHANGELOG.md" + homepage = "https://example.com" + documentation = "https://readthedocs.org" + repository = "https://github.com" + changelog = "https://github.com/me/spam/blob/master/CHANGELOG.md" [project.scripts] spam-cli = "spam:main_cli" @@ -720,7 +723,6 @@ CC0-1.0-Universal license, whichever is more permissive. .. _survey of tools: https://github.com/uranusjr/packaging-metadata-comparisons .. _trove classifiers: https://pypi.org/classifiers/ .. _SPDX: https://spdx.dev/ -.. _RFC #822: https://tools.ietf.org/html/rfc822 .. _entry points specification: https://packaging.python.org/specifications/entry-points/ .. diff --git a/pep-0622.rst b/peps/pep-0622.rst similarity index 96% rename from pep-0622.rst rename to peps/pep-0622.rst index ed9582b77..0b68a93d1 100644 --- a/pep-0622.rst +++ b/peps/pep-0622.rst @@ -1,7 +1,5 @@ PEP: 622 Title: Structural Pattern Matching -Version: $Revision$ -Last-Modified: $Date$ Author: Brandt Bucher , Daniel F Moisset , Tobias Kohn , @@ -9,13 +7,13 @@ Author: Brandt Bucher , Guido van Rossum , Talin BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Superseded Type: Standards Track Content-Type: text/x-rst Created: 23-Jun-2020 Python-Version: 3.10 -Post-History: 23-Jun-2020, 8-Jul-2020 +Post-History: 23-Jun-2020, 08-Jul-2020 Superseded-By: 634 @@ -77,8 +75,6 @@ Finally, we discuss some possible extensions that might be considered in the future, once the community has ample experience with the currently proposed syntax and semantics. -.. _overview: - Overview ======== @@ -200,7 +196,7 @@ required component values, which may be nested several objects deep. Pattern matching as present in many other languages provides an elegant solution to this problem. These range from statically compiled functional languages like F# and Haskell, via mixed-paradigm languages -like Scala [4]_ and Rust [3]_, to dynamic languages like Elixir and +like Scala_ and Rust_, to dynamic languages like Elixir and Ruby, and is under consideration for JavaScript. We are indebted to these languages for guiding the way to Pythonic pattern matching, as Python is indebted to so many other languages for many of its @@ -313,13 +309,14 @@ Although this will work, it's not necessarily what the proposal is focused on, and the new syntax has been designed to best support the destructuring scenarios. -See the `syntax`_ sections below for a more detailed specification. +See the `syntax `_ sections below +for a more detailed specification. We propose that destructuring objects can be customized by a new special ``__match_args__`` attribute. As part of this PEP we specify the general API and its implementation for some standard library -classes (including named tuples and dataclasses). See the `runtime`_ -section below. +classes (including named tuples and dataclasses). See the `runtime +`_ section below. Finally, we aim to provide comprehensive support for static type checkers and similar tools. For this purpose, we propose to introduce @@ -327,12 +324,10 @@ a ``@typing.sealed`` class decorator that will be a no-op at runtime but will indicate to static tools that all sub-classes of this class must be defined in the same module. This will allow effective static exhaustiveness checks, and together with dataclasses, will provide -basic support for algebraic data types [2]_. See the `static checkers`_ -section for more details. +basic support for `algebraic data types`_. See the `static checkers +`_ section for more details. -.. _syntax: - Syntax and Semantics ==================== @@ -420,15 +415,16 @@ A simplified, approximate grammar for the proposed syntax is:: | mapping_pattern | class_pattern -See `Appendix A`_ for the full, unabridged grammar. The simplified grammars in -this section are there for helping the reader, not as a full specification. +See `Appendix A `_ for the full, unabridged grammar. +The simplified grammars in this section are there for helping the reader, +not as a full specification. We propose that the match operation should be a statement, not an expression. Although in many languages it is an expression, being a statement better suits the general logic of Python syntax. See `rejected ideas`_ for more discussion. -The allowed patterns are described in detail below in the `patterns`_ -subsection. +The allowed patterns are described in detail below in the `patterns +`_ subsection. The ``match`` and ``case`` keywords are proposed to be soft keywords, so that they are recognized as keywords at the beginning of a match @@ -460,8 +456,8 @@ statements. Note that unlike for the previously proposed ``switch`` statement, the pre-computed dispatch dictionary semantics does not apply here. There is no ``default`` or ``else`` case - instead the special wildcard -``_`` can be used (see the section on `capture_pattern`_) as a final -'catch-all' pattern. +``_`` can be used (see the section on `capture_pattern `_) +as a final 'catch-all' pattern. Name bindings made during a successful pattern match outlive the executed suite and can be used after the match statement. This follows the logic of other @@ -498,8 +494,6 @@ We introduce the proposed syntax gradually. Here we start from the main building blocks. The following patterns are supported: -.. _literal_pattern: - Literal Patterns ~~~~~~~~~~~~~~~~ @@ -551,8 +545,6 @@ are supported. F-strings are not allowed (since in general they are not really literals). -.. _capture_pattern: - Capture Patterns ~~~~~~~~~~~~~~~~ @@ -599,8 +591,6 @@ as a `wildcard pattern`_. Reminder: ``None``, ``False`` and ``True`` are keywords denoting literals, not names. -.. _wildcard_pattern: - Wildcard Pattern ~~~~~~~~~~~~~~~~ @@ -619,8 +609,6 @@ matches but *never* binds:: Given that no binding is made, it can be used as many times as desired, unlike capture patterns. -.. _constant_value_pattern: - Constant Value Patterns ~~~~~~~~~~~~~~~~~~~~~~~ @@ -652,8 +640,6 @@ patterns (they always denote variables to be captured). See considered for constant value patterns. -.. _sequence_pattern: - Sequence Patterns ~~~~~~~~~~~~~~~~~ @@ -691,8 +677,6 @@ example: ``"a"`` and ends with ``"z"``. -.. _mapping_pattern: - Mapping Patterns ~~~~~~~~~~~~~~~~ @@ -735,8 +719,6 @@ on-the-fly by ``__missing__`` or ``__getitem__``. For example, were already present when the ``match`` block was entered. -.. _class_pattern: - Class Patterns ~~~~~~~~~~~~~~ @@ -766,8 +748,8 @@ example:: Whether a match succeeds or not is determined by the equivalent of an ``isinstance`` call. If the subject (``shape``, in the example) is not an instance of the named class (``Point`` or ``Rectangle``), the match -fails. Otherwise, it continues (see details in the `runtime`_ -section). +fails. Otherwise, it continues (see details in the `runtime +`_ section). The named class must inherit from ``type``. It may be a single name or a dotted name (e.g. ``some_mod.SomeClass`` or ``mod.pkg.Class``). @@ -817,8 +799,6 @@ the same set of variables (excluding ``_``). For example:: ... -.. _guards: - Guards ------ @@ -879,14 +859,12 @@ the match suite, or after the match statement. However, the name will Technically, most such examples can be rewritten using guards and/or nested match statements, but this will be less readable and/or will produce less -efficient code. Essentially, most of the arguments in PEP 572 apply here +efficient code. Essentially, most of the arguments in :pep:`572` apply here equally. The wildcard ``_`` is not a valid name here. -.. _runtime: - Runtime specification ===================== @@ -1016,8 +994,6 @@ existing standard library classes and adding ``__match_args__`` where it looks beneficial. -.. _static checkers: - Static checkers specification ============================= @@ -1036,8 +1012,8 @@ thus forcing people to add safety asserts like this:: else: assert False, "should never get here" -PEP 484 specifies that static type checkers should support exhaustiveness in -conditional checks with respect to enum values. PEP 586 later generalized this +:pep:`484` specifies that static type checkers should support exhaustiveness in +conditional checks with respect to enum values. :pep:`586` later generalized this requirement to literal types. This PEP further generalizes this requirement to @@ -1081,7 +1057,7 @@ and enum values are combined:: ... # Type-checking error: basic user unhandled -Obviously, no ``Matchable`` protocol (in terms of PEP 544) is needed, since +Obviously, no ``Matchable`` protocol (in terms of :pep:`544`) is needed, since every class is matchable and therefore is subject to the checks specified above. @@ -1093,10 +1069,10 @@ Quite often it is desirable to apply exhaustiveness to a set of classes without defining ad-hoc union types, which is itself fragile if a class is missing in the union definition. A design pattern where a group of record-like classes is combined into a union is popular in other languages that support pattern -matching and is known under a name of algebraic data types [2]_. +matching and is known under a name of `algebraic data types`_. -We propose to add a special decorator class ``@sealed`` to the ``typing`` -module [6]_, that will have no effect at runtime, but will indicate to static +We propose to add a special decorator class ``@sealed`` to the :py:mod:`typing` +module, that will have no effect at runtime, but will indicate to static type checkers that all subclasses (direct and indirect) of this class should be defined in the same module as the base class. @@ -1168,7 +1144,7 @@ match:: Note that the above snippet actually fails at runtime with the current implementation of generic classes in the ``typing`` module, as well as -with builtin generic classes in the recently accepted PEP 585, because +with builtin generic classes in the recently accepted :pep:`585`, because they prohibit ``isinstance`` checks. To clarify, generic classes are not prohibited in general from participating @@ -1224,7 +1200,7 @@ Precise type checking of star matches Type checkers should perform precise type checking of star items in pattern matching giving them either a heterogeneous ``list[T]`` type, or -a ``TypedDict`` type as specified by PEP 589. For example:: +a ``TypedDict`` type as specified by :pep:`589`. For example:: stuff: Tuple[int, str, str, float] @@ -1277,7 +1253,7 @@ desire to break or deprecate. The difference between hard and soft keywords is that hard keywords are *always* reserved words, even in positions where they make no sense (e.g. ``x = class + 1``), while soft keywords only get a special -meaning in context. Since PEP 617 the parser backtracks, that means that on +meaning in context. Since :pep:`617` the parser backtracks, that means that on different attempts to parse a code fragment it could interpret a soft keyword differently. @@ -1315,7 +1291,7 @@ easy, just an addition to a table, or perhaps modification of a regular expression. **Deep** parsers understand the complete syntax of Python. An example of this -is the auto-formatter Black [9]_. A particular requirement with these kinds of +is the auto-formatter Black_. A particular requirement with these kinds of tools is that they not only need to understand the syntax of the current version of Python, but older versions of Python as well. @@ -1326,7 +1302,7 @@ time with the new syntax. It has been noted that a number of these third-party tools leverage common parsing libraries (Black for example uses a fork of the lib2to3 parser). It may be helpful -to identify widely used parsing libraries (such as parso [10]_ and libCST [11]_) +to identify widely used parsing libraries (such as parso_ and libCST_) and upgrade them to be PEG compatible. However, since this work would need to be done not only for the match statement, @@ -1344,7 +1320,7 @@ GitHub. An `interactive playground `_ -based on the above implementation was created using Binder [12]_ and Jupyter [13]_. +based on the above implementation was created using Binder_ and Jupyter_. Example Code ============ @@ -1354,8 +1330,6 @@ A small `collection of example code available on GitHub. -.. _rejected ideas: - Rejected Ideas ============== @@ -1372,7 +1346,7 @@ believe the proposed syntax significantly improves readability for a wide range of code patterns, by allowing to express *what* one wants to do, rather than *how* to do it. We hope the few real code snippets we included in the PEP above illustrate this comparison well enough. For more real code examples -and their translations see Ref. [7]_. +and their translations see Ref. [1]_. Don't do this, use existing method dispatching mechanisms @@ -1554,7 +1528,7 @@ ambiguous with capture patterns. Five other alternatives were considered: case side: ... # Assigns side = entree[-1]. This works well with the recommendations for naming constants from - PEP 8. The main objection is that there's no other part of core + :pep:`8`. The main objection is that there's no other part of core Python where the case of a name is semantically significant. In addition, Python allows identifiers to use different scripts, many of which (e.g. CJK) don't have a case distinction. @@ -1716,7 +1690,7 @@ A negation of a match pattern using the operator ``!`` as a prefix would match exactly if the pattern itself does not match. For instance, ``!(3 | 4)`` would match anything except ``3`` or ``4``. -This was rejected because there is documented evidence [8]_ that this feature +This was rejected because there is `documented evidence`_ that this feature is rarely useful (in languages which support it) or used as double negation ``!!`` to control variable scopes and prevent variable bindings (which does not apply to Python). It can also be simulated using guard conditions. @@ -1932,8 +1906,6 @@ We decided not to add an ``else`` clause for several reasons. new functionality. -.. _deferred ideas: - Deferred Ideas ============== @@ -2107,7 +2079,7 @@ the fact that, because of the way names are bound, there are no real constants in Python. It also meant that the ``__match__`` method would have to re-implement much of the logic of matching which would otherwise be implemented in C code in the Python VM. As a result, this option would -perform poorly compared to an equilvalent ``if``-statement. +perform poorly compared to an equivalent ``if``-statement. The simpler protocol suffered from the fact that although it was more performant, it was much less flexible, and did not allow for many of @@ -2210,7 +2182,8 @@ Version History - Drop the ``__match__`` protocol (moved to `deferred ideas`_) - Drop ``ImpossibleMatchError`` exception - Drop leading dot for loads (moved to `deferred ideas`_) - - Reworked the initial sections (everything before `syntax`_) + - Reworked the initial sections (everything before `syntax + `_) - Added an overview of all the types of patterns before the detailed description - Added simplified syntax next to the description of each pattern @@ -2220,47 +2193,18 @@ Version History References ========== -.. [1] - https://en.wikipedia.org/wiki/Pattern_matching +.. [1] https://github.com/gvanrossum/patma/blob/master/EXAMPLES.md -.. [2] - https://en.wikipedia.org/wiki/Algebraic_data_type +.. _algebraic data types: https://en.wikipedia.org/wiki/Algebraic_data_type +.. _Rust: https://doc.rust-lang.org/reference/patterns.html +.. _Scala: https://docs.scala-lang.org/tour/pattern-matching.html +.. _documented evidence: https://dl.acm.org/doi/abs/10.1145/2480360.2384582 +.. _Black: https://black.readthedocs.io/en/stable/ +.. _parso: https://github.com/davidhalter/parso +.. _LibCST: https://github.com/Instagram/LibCST +.. _Binder: https://mybinder.org +.. _Jupyter: https://jupyter.org -.. [3] - https://doc.rust-lang.org/reference/patterns.html - -.. [4] - https://docs.scala-lang.org/tour/pattern-matching.html - -.. [5] - https://docs.python.org/3/library/dataclasses.html - -.. [6] - https://docs.python.org/3/library/typing.html - -.. [7] - https://github.com/gvanrossum/patma/blob/master/EXAMPLES.md - -.. [8] - https://dl.acm.org/doi/abs/10.1145/2480360.2384582 - -.. [9] - https://black.readthedocs.io/en/stable/ - -.. [10] - https://github.com/davidhalter/parso - -.. [11] - https://github.com/Instagram/LibCST - -.. [12] - https://mybinder.org - -.. [13] - https://jupyter.org - - -.. _Appendix A: Appendix A -- Full Grammar ========================== @@ -2277,7 +2221,7 @@ Other notation used beyond standard EBNF: - ``SEP.RULE+`` is shorthand for ``RULE (SEP RULE)*`` - ``!RULE`` is a negative lookahead assertion -:: +.. code:: text match_expr: | star_named_expression ',' star_named_expressions? @@ -2332,14 +2276,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0623.rst b/peps/pep-0623.rst similarity index 88% rename from pep-0623.rst rename to peps/pep-0623.rst index c1d263547..0d182add1 100644 --- a/pep-0623.rst +++ b/peps/pep-0623.rst @@ -2,19 +2,21 @@ PEP: 623 Title: Remove wstr from Unicode Author: Inada Naoki BDFL-Delegate: Victor Stinner -Status: Accepted +Discussions-To: https://mail.python.org/archives/list/python-dev@python.org/thread/BO2TQHSXWL2RJMINWQQRBF5LANDDJNHH/ +Status: Final Type: Standards Track Content-Type: text/x-rst Created: 25-Jun-2020 Python-Version: 3.10 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/VQKDIZLZ6HF2MLTNCUFURK2IFTXVQEYA/ Abstract ======== -PEP 393 deprecated some unicode APIs, and introduced ``wchar_t *wstr``, +:pep:`393` deprecated some unicode APIs, and introduced ``wchar_t *wstr``, and ``Py_ssize_t wstr_length`` in the Unicode structure to support -these deprecated APIs. [1]_ +these deprecated APIs. This PEP is planning removal of ``wstr``, and ``wstr_length`` with deprecated APIs using these members by Python 3.12. @@ -59,14 +61,14 @@ Rationale Python 4.0 is not scheduled yet ------------------------------- -PEP 393 introduced efficient internal representation of Unicode and +:pep:`393` introduced efficient internal representation of Unicode and removed border between "narrow" and "wide" build of Python. -PEP 393 was implemented in Python 3.3 which is released in 2012. Old +:pep:`393` was implemented in Python 3.3 which is released in 2012. Old APIs were deprecated since then, and the removal was scheduled in Python 4.0. -Python 4.0 was expected as next version of Python 3.9 when PEP 393 +Python 4.0 was expected as next version of Python 3.9 when :pep:`393` was accepted. But the next version of Python 3.9 is Python 3.10, not 4.0. This is why this PEP schedule the removal plan again. @@ -74,7 +76,7 @@ not 4.0. This is why this PEP schedule the removal plan again. Python 2 reached EOL -------------------- -Since Python 2 didn't have PEP 393 Unicode implementation, legacy +Since Python 2 didn't have :pep:`393` Unicode implementation, legacy APIs might help C extension modules supporting both of Python 2 and 3. But Python 2 reached the EOL in 2020. We can remove legacy APIs kept @@ -183,9 +185,6 @@ References They no longer cache the ``wchar_t*`` representation of string objects. -.. [1] PEP 393 -- Flexible String Representation - (https://www.python.org/dev/peps/pep-0393/) - Copyright ========= diff --git a/pep-0624.rst b/peps/pep-0624.rst similarity index 99% rename from pep-0624.rst rename to peps/pep-0624.rst index 18545fe09..30b6fb1ba 100644 --- a/pep-0624.rst +++ b/peps/pep-0624.rst @@ -30,7 +30,7 @@ This PEP proposes to remove deprecated ``Py_UNICODE`` encoder APIs in Python 3.1 .. note:: - `PEP 623 `_ propose to remove + :pep:`623` propose to remove Unicode object APIs relating to ``Py_UNICODE``. On the other hand, this PEP is not relating to Unicode object. These PEPs are split because they have different motivations and need different discussions. diff --git a/peps/pep-0625.rst b/peps/pep-0625.rst new file mode 100644 index 000000000..a49d08488 --- /dev/null +++ b/peps/pep-0625.rst @@ -0,0 +1,209 @@ +PEP: 625 +Title: Filename of a Source Distribution +Author: Tzu-ping Chung , + Paul Moore +PEP-Delegate: Pradyun Gedam +Discussions-To: https://discuss.python.org/t/draft-pep-file-name-of-a-source-distribution/4686 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 08-Jul-2020 +Post-History: 08-Jul-2020 +Resolution: https://discuss.python.org/t/pep-625-file-name-of-a-source-distribution/4686/159 + +Abstract +======== + +This PEP describes a standard naming scheme for a Source Distribution, also +known as an *sdist*. An sdist is distinct from an arbitrary archive file +containing source code of Python packages, and can be used to communicate +information about the distribution to packaging tools. + +A standard sdist specified here is a gzipped tar file with a specially +formatted filename and the usual ``.tar.gz`` suffix. This PEP does not specify +the contents of the tarball, as that is covered in other specifications. + +Motivation +========== + +An sdist is a Python package distribution that contains "source code" of the +Python package, and requires a build step to be turned into a wheel on +installation. This format is often considered as an unbuilt counterpart of a +:pep:`427` wheel, and given special treatments in various parts of the +packaging ecosystem. + +The content of an sdist is specified in :pep:`517` and :pep:`643`, but currently +the filename of the sdist is incompletely specified, meaning that consumers +of the format must download and process the sdist to confirm the name and +version of the distribution included within. + +Installers currently rely on heuristics to infer the name and/or version from +the filename, to help the installation process. pip, for example, parses the +filename of an sdist from a :pep:`503` index, to obtain the distribution's +project name and version for dependency resolution purposes. But due to the +lack of specification, the installer does not have any guarantee as to the +correctness of the inferred data, and must verify it at some point by locally +building the distribution metadata. + +This build step is awkward for a certain class of operations, when the user +does not expect the build process to occur. `pypa/pip#8387`_ describes an +example. The command ``pip download --no-deps --no-binary=numpy numpy`` is +expected to only download an sdist for numpy, since we do not need to check +for dependencies, and both the name and version are available by introspecting +the downloaded filename. pip, however, cannot assume the downloaded archive +follows the convention, and must build and check the metadata. For a :pep:`518` +project, this means running the ``prepare_metadata_for_build_wheel`` hook +specified in :pep:`517`, which incurs significant overhead. + + +Rationale +========= + +By creating a special filename scheme for the sdist format, this PEP frees up +tools from the time-consuming metadata verification step when they only need +the metadata available in the filename. + +This PEP also serves as the formal specification to the long-standing +filename convention used by the current sdist implementations. The filename +contains the distribution name and version, to aid tools identifying a +distribution without needing to download, unarchive the file, and perform +costly metadata generation for introspection, if all the information they need +is available in the filename. + + +Specification +============= + +The name of an sdist should be ``{distribution}-{version}.tar.gz``. + +* ``distribution`` is the name of the distribution as defined in :pep:`345`, + and normalised as described in `the wheel spec`_ e.g. ``'pip'``, + ``'flit_core'``. +* ``version`` is the version of the distribution as defined in :pep:`440`, + e.g. ``20.2``, and normalised according to the rules in that PEP. + +An sdist must be a gzipped tar archive in pax format, that is able to be +extracted by the standard library ``tarfile`` module with the open flag +``'r:gz'``. + +Code that produces an sdist file MUST give the file a name that matches this +specification. The specification of the ``build_sdist`` hook from :pep:`517` is +extended to require this naming convention. + +Code that processes sdist files MAY determine the distribution name and version +by simply parsing the filename, and is not required to verify that information +by generating or reading the metadata from the sdist contents. + +Conforming sdist files can be recognised by the presence of the ``.tar.gz`` +suffix and a *single* hyphen in the filename. Note that some legacy files may +also match these criteria, but this is not expected to be an issue in practice. +See the "Backwards Compatibility" section of this document for more details. + + +Backwards Compatibility +======================= + +The new filename scheme is a subset of the current informal naming +convention for sdist files, so tools that create or publish files conforming +to this standard will be readable by older tools that only understand the +previous naming conventions. + +Tools that consume sdist filenames would technically not be able to determine +whether a file is using the new standard or a legacy form. However, a review +of the filenames on PyPI determined that 37% of files are obviously legacy +(because they contain multiple or no hyphens) and of the remainder, parsing +according to this PEP gives the correct answer in all but 0.004% of cases. + +Currently, tools that consume sdists should, if they are to be fully correct, +treat the name and version parsed from the filename as provisional, and verify +them by downloading the file and generating the actual metadata (or reading it, +if the sdist conforms to :pep:`643`). Tools supporting this specification can +treat the name and version from the filename as definitive. In theory, this +could risk mistakes if a legacy filename is assumed to conform to this PEP, +but in practice the chance of this appears to be vanishingly small. + + +Rejected Ideas +============== + +Rely on the specification for sdist metadata +-------------------------------------------- + +Since this PEP was first written, :pep:`643` has been accepted, defining a +trustworthy, standard sdist metadata format. This allows distribution metadata +(and in particular name and version) to be determined statically. + +This is not considered sufficient, however, as in a number of significant +cases (for example, reading filenames from a package index) the application +only has access to the filename, and reading metadata would involve a +potentially costly download. + +Use a dedicated file extension +------------------------------ + +The original version of this PEP proposed a filename of +``{distribution}-{version}.sdist``. This has the advantage of being explicit, +as well as allowing a future change to the storage format without needing a +further change of the file naming convention. + +However, there are significant compatibility issues with a new extension. Index +servers may currently disallow unknown extensions, and if we introduced a new +one, it is not clear how to handle cases like a legacy index trying to mirror an +index that hosts new-style sdists. Is it acceptable to only partially mirror, +omitting sdists for newer versions of projects? Also, build backends that produce +the new format would be incompaible with index servers that only accept the old +format, and as there is often no way for a user to request an older version of a +backend when doing a build, this could make it impossible to build and upload +sdists. + +Augment a currently common sdist naming scheme +---------------------------------------------- + +A scheme ``{distribution}-{version}.sdist.tar.gz`` was raised during the +initial discussion. This was abandoned due to backwards compatibility issues +with currently available installation tools. pip 20.1, for example, would +parse ``distribution-1.0.sdist.tar.gz`` as project ``distribution`` with +version ``1.0.sdist``. This would cause the sdist to be downloaded, but fail to +install due to inconsistent metadata. + +The main advantage of this proposal was that it is easier for tools to +recognise the new-style naming. But this is not a particularly significant +benefit, given that all sdists with a single hyphen in the name are parsed +the same way under the old and new rules. + + +Open Issues +=========== + +The contents of an sdist are required to contain a single top-level directory +named ``{name}-{version}``. Currently no normalisation rules are required +for the components of this name. Should this PEP require that the same normalisation +rules are applied here as for the filename? Note that in practice, it is likely +that tools will create the two names using the same code, so normalisation is +likely to happen naturally, even if it is not explicitly required. + + +References +========== + +.. _`pypa/pip#8387`: https://github.com/pypa/pip/issues/8387 +.. _`the wheel spec`: https://packaging.python.org/en/latest/specifications/binary-distribution-format/ + + +Copyright +========= + +This document is placed in the public domain or under the CC0-1.0-Universal +license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0626.rst b/peps/pep-0626.rst similarity index 100% rename from pep-0626.rst rename to peps/pep-0626.rst diff --git a/pep-0627.rst b/peps/pep-0627.rst similarity index 93% rename from pep-0627.rst rename to peps/pep-0627.rst index 5a1a3e595..81a13d72d 100644 --- a/pep-0627.rst +++ b/peps/pep-0627.rst @@ -3,13 +3,17 @@ Title: Recording installed projects Author: Petr Viktorin BDFL-Delegate: Paul Moore Discussions-To: https://discuss.python.org/t/pep-627/4126 -Status: Accepted -Type: Informational +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 15-Jul-2020 Resolution: https://discuss.python.org/t/pep-627/4126/42 +.. canonical-pypa-spec:: :ref:`packaging:recording-installed-packages` + + Abstract ======== @@ -20,7 +24,7 @@ Packaging Authority (PyPA) standards repository, and sets up guidelines for changing it. Two files in installed ``.dist-info`` directories are made optional: -``RECORD`` (which PEP 376 lists as mandatory, but suggests it can be left out +``RECORD`` (which :pep:`376` lists as mandatory, but suggests it can be left out for "system packages"), and ``INSTALLER``. @@ -30,11 +34,11 @@ Motivation Python packaging is moving from relying on specific tools (Setuptools and pip) toward an ecosystem of tools and tool-agnostic interoperability standards. -PEP 376 is not written as an interoperability standard. +:pep:`376` is not written as an interoperability standard. It describes implementation details of specific tools and libraries, and is underspecified, leaving much room for implementation-defined behavior. -This is a proposal to “distill” the standard from PEP 376, clarify it, +This is a proposal to “distill” the standard from :pep:`376`, clarify it, and rewrite it to be tool-agnostic. The aim of this PEP is to have a better standard, not necessarily a perfect one. @@ -44,7 +48,7 @@ Some issues are left to later clarification. Rationale Change ================ -PEP 376's rationale focuses on two problems: +:pep:`376`'s rationale focuses on two problems: * There are too many ways to install projects and this makes interoperation difficult. * There is no API to get information on installed distributions. @@ -86,7 +90,7 @@ to *Recording installed projects*. While putting files in known locations on disk may be thought of as a “database”, it's not what most people think about when they hear the term. -The PyPA links to PEP 376 under the heading *Recording installed distributions*. +The PyPA links to :pep:`376` under the heading *Recording installed distributions*. The PyPA glossary defines “Distribution” (or, “Distribution Package” to prevent confusion with e.g. Linux distributions) as “A versioned archive file [
]”. @@ -134,9 +138,9 @@ The “base” of relative paths in ``RECORD`` is specified relative to the Both *hash* and *size* fields are now optional (for any file, not just ``.pyc``, ``.pyo`` and ``RECORD``). Leavng them out is discouraged, except for ``*.pyc`` and ``RECORD`` itself. -(Note that PEP 376 is unclear on what was optional; when taken literally, +(Note that :pep:`376` is unclear on what was optional; when taken literally, its text and examples contradict. Despite that, “both fields are optional“ is a -reasonable interpretation of PEP 376. +reasonable interpretation of :pep:`376`. The alternative would be to mandate—rather than recommend—which files can be recorded without hash and size, and to update that list over time as new use cases come up.) @@ -156,7 +160,7 @@ Tools must not uninstall/remove projects that lack a ``RECORD`` file managers of Linux distros). On Windows, files in ``RECORD`` may be separated by either ``/`` or ``\``. -PEP 376 was unclear on this: it mandates forward slashes in one place, but +:pep:`376` was unclear on this: it mandates forward slashes in one place, but shows backslackes in a Windows-specific example. diff --git a/pep-0628.txt b/peps/pep-0628.rst similarity index 95% rename from pep-0628.txt rename to peps/pep-0628.rst index e99de632b..43088ac2b 100644 --- a/pep-0628.txt +++ b/peps/pep-0628.rst @@ -9,7 +9,6 @@ Content-Type: text/x-rst Created: 28-Jun-2011 Python-Version: 3.6 Post-History: 28-Jun-2011 -Resolution: https://bugs.python.org/issue12345 Abstract @@ -27,12 +26,13 @@ of assigning a name to the value ``2 * pi`` (``2π``). PEP Acceptance ============== -This PEP is now accepted and math.tau will be a part of Python 3.6. +This PEP is now `accepted`_ and ``math.tau`` will be a part of Python 3.6. Happy birthday Nick! The idea in this PEP has been implemented in the auspiciously named `issue 12345`_. +.. _accepted: https://bugs.python.org/issue12345#msg272287 .. _issue 12345: http://bugs.python.org/issue12345 diff --git a/pep-0629.rst b/peps/pep-0629.rst similarity index 97% rename from pep-0629.rst rename to peps/pep-0629.rst index 537fdf33d..2d0f99f5c 100644 --- a/pep-0629.rst +++ b/peps/pep-0629.rst @@ -5,6 +5,7 @@ BDFL-Delegate: Brett Cannon Discussions-To: https://discuss.python.org/t/pep-629-versioning-pypis-simple-api/4720 Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 16-Jul-2020 Post-History: 16-Jul-2020 @@ -53,9 +54,9 @@ Overview This PEP proposes the inclusion of a meta tag on the responses of every successful request to a simple API page, which contains a name attribute -of "pypi:repository-version", and a content that is a PEP 440 compatible +of "pypi:repository-version", and a content that is a :pep:`440` compatible version number, which is further constrained to ONLY be Major.Minor, and -none of the additional features supported by PEP 440. +none of the additional features supported by :pep:`440`. This would end up looking like:: diff --git a/pep-0630.rst b/peps/pep-0630.rst similarity index 53% rename from pep-0630.rst rename to peps/pep-0630.rst index 10416182f..9dfdef56b 100644 --- a/pep-0630.rst +++ b/peps/pep-0630.rst @@ -2,54 +2,63 @@ PEP: 630 Title: Isolating Extension Modules Author: Petr Viktorin Discussions-To: capi-sig@python.org -Status: Active +Status: Final Type: Informational Content-Type: text/x-rst Created: 25-Aug-2020 Post-History: 16-Jul-2020 -Isolating Extension Modules -=========================== +.. highlight:: c + +.. canonical-doc:: `Isolating Extension Modules HOWTO `_ Abstract --------- +======== -Traditionally, state of Python extension modules was kept in C +Traditionally, state belonging to Python extension modules was kept in C ``static`` variables, which have process-wide scope. This document describes problems of such per-process state and efforts to make -per-module state, a better default, possible and easy to use. +per-module state—a better default—possible and easy to use. The document also describes how to switch to per-module state where -possible. The switch involves allocating space for that state, switching -from static types to heap types, and—perhaps most importantly—accessing -per-module state from code. +possible. This transition involves allocating space for that state, potentially +switching from static types to heap types, and—perhaps most +importantly—accessing per-module state from code. -About this document -------------------- -As an `informational PEP `__, -this document does not introduce any changes: those should be done in +About This Document +=================== + +As an :pep:`informational PEP <1#pep-types>`, +this document does not introduce any changes; those should be done in their own PEPs (or issues, if small enough). Rather, it covers the motivation behind an effort that spans multiple releases, and instructs early adopters on how to use the finished features. -Once support is reasonably complete, the text can be moved to Python's -documentation as a HOWTO. Meanwhile, in the spirit of documentation-driven -development, gaps identified in this text can show where to focus -the effort, and the text can be updated as new features are implemented +Once support is reasonably complete, this content can be moved to Python's +documentation as a `HOWTO `__. +Meanwhile, in the spirit of documentation-driven development, +gaps identified in this PEP can show where to focus the effort, +and it can be updated as new features are implemented. Whenever this PEP mentions *extension modules*, the advice also -applies to *built-in* modules, such as the C parts of the standard -library. The standard library is expected to switch to per-module state -early. +applies to *built-in* modules. + +.. note:: + This PEP contains generic advice. When following it, always take into + account the specifics of your project. + + For example, while much of this advice applies to the C parts of + Python's standard library, the PEP does not factor in stdlib specifics + (unusual backward compatibility issues, access to private API, etc.). PEPs related to this effort are: -- PEP 384 -- *Defining a Stable ABI*, which added C API for creating +- :pep:`384` -- *Defining a Stable ABI*, which added a C API for creating heap types -- PEP 489 -- *Multi-phase extension module initialization* -- PEP 573 -- *Module State Access from C Extension Methods* +- :pep:`489` -- *Multi-phase extension module initialization* +- :pep:`573` -- *Module State Access from C Extension Methods* This document is concerned with Python's public C API, which is not offered by all implementations of Python. However, nothing in this PEP is @@ -58,8 +67,9 @@ specific to CPython. As with any Informational PEP, this text does not necessarily represent a Python community consensus or recommendation. + Motivation ----------- +========== An *interpreter* is the context in which Python code runs. It contains configuration (e.g. the import path) and runtime state (e.g. the set of @@ -70,13 +80,13 @@ two cases to think about—users may run interpreters: - in sequence, with several ``Py_InitializeEx``/``Py_FinalizeEx`` cycles, and -- in parallel, managing “sub-interpreters” using +- in parallel, managing "sub-interpreters" using ``Py_NewInterpreter``/``Py_EndInterpreter``. Both cases (and combinations of them) would be most useful when embedding Python within a library. Libraries generally shouldn't make assumptions about the application that uses them, which includes -assumptions about a process-wide “main Python interpreter”. +assuming a process-wide "main Python interpreter". Currently, CPython doesn't handle this use case well. Many extension modules (and even some stdlib modules) use *per-process* global state, @@ -84,34 +94,36 @@ because C ``static`` variables are extremely easy to use. Thus, data that should be specific to an interpreter ends up being shared between interpreters. Unless the extension developer is careful, it is very easy to introduce edge cases that lead to crashes when a module is loaded in -more than one interpreter. +more than one interpreter in the same process. -Unfortunately, *per-interpreter* state is not easy to achieve: extension +Unfortunately, *per-interpreter* state is not easy to achieve—extension authors tend to not keep multiple interpreters in mind when developing, and it is currently cumbersome to test the behavior. + Rationale for Per-module State ------------------------------- +============================== Instead of focusing on per-interpreter state, Python's C API is evolving to better support the more granular *per-module* state. By default, C-level data will be attached to a *module object*. Each interpreter -will then create its own module object, keeping data separate. For +will then create its own module object, keeping the data separate. For testing the isolation, multiple module objects corresponding to a single extension can even be loaded in a single interpreter. Per-module state provides an easy way to think about lifetime and -resource ownership: the extension module author will set up when a +resource ownership: the extension module will initialize when a module object is created, and clean up when it's freed. In this regard, -a module is just like any other ``PyObject *``; there are no “on -interpreter shutdown” hooks to think about—or forget about. +a module is just like any other ``PyObject *``; there are no "on +interpreter shutdown" hooks to think—or forget—about. -Goal: Easy-to-use Module State -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Goal: Easy-to-Use Module State +------------------------------ It is currently cumbersome or impossible to do everything the C API -offers while keeping modules isolated. Enabled by PEP 384, changes in -PEPs 489 and 573 (and future planned ones) aim to first make it +offers while keeping modules isolated. Enabled by :pep:`384`, changes in +:pep:`489` and :pep:`573` (and future planned ones) aim to first make it *possible* to build modules this way, and then to make it *easy* to write new modules this way and to convert old ones, so that it can become a natural default. @@ -122,20 +134,22 @@ per-thread or per-task state. The goal is to treat these as exceptional cases: they should be possible, but extension authors will need to think more carefully about them. + Non-goals: Speedups and the GIL -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- There is some effort to speed up CPython on multi-core CPUs by making the GIL per-interpreter. While isolating interpreters helps that effort, defaulting to per-module state will be beneficial even if no speedup is achieved, as it makes supporting multiple interpreters safer by default. -How to make modules safe with multiple interpreters ---------------------------------------------------- + +Making Modules Safe with Multiple Interpreters +============================================== There are many ways to correctly support multiple interpreters in extension modules. The rest of this text describes the preferred way to -write such a module, or to convert an existing module. +write such a module, or to convert an existing one. Note that support is a work in progress; the API for some features your module needs might not yet be ready. @@ -143,15 +157,17 @@ module needs might not yet be ready. A full example module is available as `xxlimited `__. -This section assumes that “*you*” are an extension module author. +This section assumes that "*you*" are an extension module author. Isolated Module Objects -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- The key point to keep in mind when developing an extension module is that several module objects can be created from a single shared library. -For example:: +For example: + +.. code-block:: pycon >>> import sys >>> import binascii @@ -165,7 +181,7 @@ As a rule of thumb, the two modules should be completely independent. All objects and state specific to the module should be encapsulated within the module object, not shared with other module objects, and cleaned up when the module object is deallocated. Exceptions are -possible (see “Managing global state” below), but they will need more +possible (see `Managing Global State`_), but they will need more thought and attention to edge cases than code that follows this rule of thumb. @@ -173,14 +189,18 @@ While some modules could do with less stringent restrictions, isolated modules make it easier to set clear expectations (and guidelines) that work across a variety of use cases. + Surprising Edge Cases -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Note that isolated modules do create some surprising edge cases. Most notably, each module object will typically not share its classes and -exceptions with other similar modules. Continuing from the example -above, note that ``old_binascii.Error`` and ``binascii.Error`` are -separate objects. In the following code, the exception is *not* caught:: +exceptions with other similar modules. Continuing from the +`example above `__, +note that ``old_binascii.Error`` and ``binascii.Error`` are +separate objects. In the following code, the exception is *not* caught: + +.. code-block:: pycon >>> old_binascii.Error == binascii.Error False @@ -188,7 +208,7 @@ separate objects. In the following code, the exception is *not* caught:: ... old_binascii.unhexlify(b'qwertyuiop') ... except binascii.Error: ... print('boo') - ... + ... Traceback (most recent call last): File "", line 2, in binascii.Error: Non-hexadecimal digit found @@ -197,14 +217,15 @@ This is expected. Notice that pure-Python modules behave the same way: it is a part of how Python works. The goal is to make extension modules safe at the C level, not to make -hacks behave intuitively. Mutating ``sys.modules`` “manually” counts +hacks behave intuitively. Mutating ``sys.modules`` "manually" counts as a hack. + Managing Global State -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Sometimes, state of a Python module is not specific to that module, but -to the entire process (or something else “more global” than a module). +to the entire process (or something else "more global" than a module). For example: - The ``readline`` module manages *the* terminal. @@ -220,23 +241,25 @@ If that is not possible, consider explicit locking. If it is necessary to use process-global state, the simplest way to avoid issues with multiple interpreters is to explicitly prevent a -module from being loaded more than once per process—see “Opt-Out: -Limiting to One Module Object per Process” below. +module from being loaded more than once per process—see +`Opt-Out: Limiting to One Module Object per Process`_. + Managing Per-Module State -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- -To use per-module state, use `multi-phase extension module -initialization `__ -introduced in PEP 489. This signals that your module supports multiple +To use per-module state, use `multi-phase extension module initialization +`__ +introduced in :pep:`489`. This signals that your module supports multiple interpreters correctly. Set ``PyModuleDef.m_size`` to a positive number to request that many bytes of storage local to the module. Usually, this will be set to the size of some module-specific ``struct``, which can store all of the module's C-level state. In particular, it is where you should put -pointers to classes (including exceptions) and settings (e.g. ``csv``'s -`field_size_limit `__) +pointers to classes (including exceptions, but excluding static types) +and settings (e.g. ``csv``'s `field_size_limit +`__) which the C code needs to function. .. note:: @@ -246,9 +269,9 @@ which the C code needs to function. which is easy to get wrong and hard to test sufficiently. If the module state includes ``PyObject`` pointers, the module object -must hold references to those objects and implement module-level hooks -``m_traverse``, ``m_clear``, ``m_free``. These work like -``tp_traverse``, ``tp_clear``, ``tp_free`` of a class. Adding them will +must hold references to those objects and implement the module-level hooks +``m_traverse``, ``m_clear`` and ``m_free``. These work like +``tp_traverse``, ``tp_clear`` and ``tp_free`` of a class. Adding them will require some work and make the code longer; this is the price for modules which can be unloaded cleanly. @@ -258,7 +281,7 @@ example module initialization shown at the bottom of the file. Opt-Out: Limiting to One Module Object per Process -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------- A non-negative ``PyModuleDef.m_size`` signals that a module supports multiple interpreters correctly. If this is not yet the case for your @@ -279,12 +302,13 @@ process. For example:: // ... rest of initialization } + Module State Access from Functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- Accessing the state from module-level functions is straightforward. Functions get the module object as their first argument; for extracting -the state there is ``PyModule_GetState``:: +the state, you can use ``PyModule_GetState``:: static PyObject * func(PyObject *module, PyObject *args) @@ -296,15 +320,17 @@ the state there is ``PyModule_GetState``:: // ... rest of logic } -(Note that ``PyModule_GetState`` may return NULL without setting an -exception if there is no module state, i.e. ``PyModuleDef.m_size`` was -zero. In your own module, you're in control of ``m_size``, so this is -easy to prevent.) +.. note:: + ``PyModule_GetState`` may return NULL without setting an + exception if there is no module state, i.e. ``PyModuleDef.m_size`` was + zero. In your own module, you're in control of ``m_size``, so this is + easy to prevent. -Heap types -~~~~~~~~~~ -Traditionally, types defined in C code were *static*, that is, +Heap Types +========== + +Traditionally, types defined in C code are *static*; that is, ``static PyTypeObject`` structures defined directly in code and initialized using ``PyType_Ready()``. @@ -315,18 +341,35 @@ the Python level: for example, you can't set ``str.myattribute = 123``. .. note:: Sharing truly immutable objects between interpreters is fine, - as long as they don't provide access to mutable objects. But, every - Python object has a mutable implementation detail: the reference - count. Changes to the refcount are guarded by the GIL. Thus, code - that shares any Python objects across interpreters implicitly depends - on CPython's current, process-wide GIL. + as long as they don't provide access to mutable objects. + However, in CPython, every Python object has a mutable implementation + detail: the reference count. Changes to the refcount are guarded by the GIL. + Thus, code that shares any Python objects across interpreters implicitly + depends on CPython's current, process-wide GIL. -An alternative to static types is *heap-allocated types*, or heap types -for short. These correspond more closely to classes created by Python’s +Because they are immutable and process-global, static types cannot access +"their" module state. +If any method of such a type requires access to module state, +the type must be converted to a *heap-allocated type*, or *heap type* +for short. These correspond more closely to classes created by Python's ``class`` statement. +For new modules, using heap types by default is a good rule of thumb. + +Static types can be converted to heap types, but note that +the heap type API was not designed for "lossless" conversion +from static types -- that is, creating a type that works exactly like a given +static type. Unlike static types, heap type objects are mutable by default. +Also, when rewriting the class definition in a new API, +you are likely to unintentionally change a few details (e.g. pickle-ability +or inherited slots). Always test the details that are important to you. + + +Defining Heap Types +------------------- + Heap types can be created by filling a ``PyType_Spec`` structure, a -description or “blueprint” of a class, and calling +description or "blueprint" of a class, and calling ``PyType_FromModuleAndSpec()`` to construct a new class object. .. note:: @@ -338,31 +381,76 @@ The class should generally be stored in *both* the module state (for safe access from C) and the module's ``__dict__`` (for access from Python code). + +Garbage Collection Protocol +--------------------------- + +Instances of heap types hold a reference to their type. +This ensures that the type isn't destroyed before all its instances are, +but may result in reference cycles that need to be broken by the +garbage collector. + +To avoid memory leaks, instances of heap types must implement the +garbage collection protocol. +That is, heap types should: + +- Have the ``Py_TPFLAGS_HAVE_GC`` flag. +- Define a traverse function using ``Py_tp_traverse``, which + visits the type (e.g. using ``Py_VISIT(Py_TYPE(self));``). + +Please refer to the `documentation +`__ of `Py_TPFLAGS_HAVE_GC +`__ and +`tp_traverse +` +for additional considerations. + +If your traverse function delegates to the ``tp_traverse`` of its base class +(or another type), ensure that ``Py_TYPE(self)`` is visited only once. +Note that only heap type are expected to visit the type in ``tp_traverse``. + +For example, if your traverse function includes:: + + base->tp_traverse(self, visit, arg) + +...and ``base`` may be a static type, then it should also include:: + + if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // a heap type's tp_traverse already visited Py_TYPE(self) + } else { + Py_VISIT(Py_TYPE(self)); + } + +It is not necessary to handle the type's reference count in ``tp_new`` +and ``tp_clear``. + + Module State Access from Classes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- If you have a type object defined with ``PyType_FromModuleAndSpec()``, -you can call ``PyType_GetModule`` to get the associated module, then +you can call ``PyType_GetModule`` to get the associated module, and then ``PyModule_GetState`` to get the module's state. To save a some tedious error-handling boilerplate code, you can combine these two steps with ``PyType_GetModuleState``, resulting in:: - my_struct *state = (my_struct*)PyType_GetModuleState(type); - if (state === NULL) { - return NULL; - } + my_struct *state = (my_struct*)PyType_GetModuleState(type); + if (state === NULL) { + return NULL; + } + Module State Access from Regular Methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------------- -Accessing the module-level state from methods of a class is somewhat -more complicated, but possible thanks to changes introduced in PEP 573. +Accessing the module-level state from methods of a class is somewhat more +complicated, but is possible thanks to the changes introduced in :pep:`573`. To get the state, you need to first get the *defining class*, and then get the module state from it. The largest roadblock is getting *the class a method was defined in*, or -that method's “defining class” for short. The defining class can have a +that method's "defining class" for short. The defining class can have a reference to the module it is part of. Do not confuse the defining class with ``Py_TYPE(self)``. If the method @@ -370,9 +458,11 @@ is called on a *subclass* of your type, ``Py_TYPE(self)`` will refer to that subclass, which may be defined in different module than yours. .. note:: - The following Python code. can illustrate the concept. + The following Python code can illustrate the concept. ``Base.get_defining_class`` returns ``Base`` even - if ``type(self) == Sub``:: + if ``type(self) == Sub``: + + .. code-block:: python class Base: def get_defining_class(self): @@ -381,12 +471,11 @@ that subclass, which may be defined in different module than yours. class Sub(Base): pass - -For a method to get its “defining class”, it must use the -``METH_METHOD | METH_FASTCALL | METH_KEYWORDS`` `calling -convention `__ -and the corresponding `PyCMethod -signature `__:: +For a method to get its "defining class", it must use the +``METH_METHOD | METH_FASTCALL | METH_KEYWORDS`` `calling convention +`__ +and the corresponding `PyCMethod signature +`__:: PyObject *PyCMethod( PyObject *self, // object the method was called on @@ -424,41 +513,99 @@ For example:: {NULL}, } + +Module State Access from Slot Methods, Getters and Setters +---------------------------------------------------------- + +.. note:: + + This is new in Python 3.11. + + .. After adding to limited API: + + If you use the `limited API __, + you must update ``Py_LIMITED_API`` to ``0x030b0000``, losing ABI + compatibility with earlier versions. + +Slot methods -- the fast C equivalents for special methods, such as `nb_add +`__ +for ``__add__`` or `tp_new +`__ +for initialization -- have a very simple API that doesn't allow +passing in the defining class, unlike with ``PyCMethod``. +The same goes for getters and setters defined with +`PyGetSetDef `__. + +To access the module state in these cases, use the `PyType_GetModuleByDef +`__ +function, and pass in the module definition. +Once you have the module, call `PyModule_GetState +`__ +to get the state:: + + PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &module_def); + my_struct *state = (my_struct*)PyModule_GetState(module); + if (state === NULL) { + return NULL; + } + +``PyType_GetModuleByDef`` works by searching the `MRO +`__ +(i.e. all superclasses) for the first superclass that has a corresponding +module. + +.. note:: + + In very exotic cases (inheritance chains spanning multiple modules + created from the same definition), ``PyType_GetModuleByDef`` might not + return the module of the true defining class. However, it will always + return a module with the same definition, ensuring a compatible + C memory layout. + + +Lifetime of the Module State +---------------------------- + +When a module object is garbage-collected, its module state is freed. +For each pointer to (a part of) the module state, you must hold a reference +to the module object. + +Usually this is not an issue, because types created with +``PyType_FromModuleAndSpec``, and their instances, hold a reference +to the module. +However, you must be careful in reference counting when you reference +module state from other places, such as callbacks for external +libraries. + + Open Issues ------------ +=========== Several issues around per-module state and heap types are still open. Discussions about improving the situation are best held on the `capi-sig mailing list `__. -Module State Access from Slot Methods, Getters and Setters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Currently (as of Python 3.9), there is no API to access the module state -from: - -- slot methods (meaning type slots, such as ``tp_new``, ``nb_add`` or - ``tp_iternext``) -- getters and setters defined with ``tp_getset`` Type Checking -~~~~~~~~~~~~~ +------------- -Currently (as of Python 3.9), heap types have no good API to write +Currently (as of Python 3.10), heap types have no good API to write ``Py*_Check`` functions (like ``PyUnicode_Check`` exists for ``str``, a -static type), and so it is not easy to ensure whether instances have a +static type), and so it is not easy to ensure that instances have a particular C layout. -Metaclasses -~~~~~~~~~~~ -Currently (as of Python 3.9), there is no good API to specify the -*metaclass* of a heap type, that is, the ``ob_type`` field of the type +Metaclasses +----------- + +Currently (as of Python 3.10), there is no good API to specify the +*metaclass* of a heap type; that is, the ``ob_type`` field of the type object. -Per-Class scope -~~~~~~~~~~~~~~~ + +Per-Class Scope +--------------- It is also not possible to attach state to *types*. While ``PyHeapTypeObject`` is a variable-size object (``PyVarObject``), @@ -466,17 +613,18 @@ its variable-size storage is currently consumed by slots. Fixing this is complicated by the fact that several classes in an inheritance hierarchy may need to reserve some state. + +Lossless Conversion to Heap Types +--------------------------------- + +The heap type API was not designed for "lossless" conversion from static types; +that is, creating a type that works exactly like a given static type. +The best way to address it would probably be to write a guide that covers +known "gotchas". + + Copyright ---------- +========= This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0631.rst b/peps/pep-0631.rst similarity index 94% rename from pep-0631.rst rename to peps/pep-0631.rst index 4820d6e94..0324f7a44 100644 --- a/pep-0631.rst +++ b/peps/pep-0631.rst @@ -3,11 +3,13 @@ Title: Dependency specification in pyproject.toml based on PEP 508 Author: Ofek Lev Sponsor: Paul Ganssle Discussions-To: https://discuss.python.org/t/5018 -Status: Accepted +Status: Superseded Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 20-Aug-2020 Post-History: 20-Aug-2020 +Superseded-By: 621 Resolution: https://discuss.python.org/t/how-to-specify-dependencies-pep-508-strings-or-a-table-in-toml/5243/38 Abstract @@ -15,16 +17,15 @@ Abstract This PEP specifies how to write a project's dependencies in a ``pyproject.toml`` file for packaging-related tools to consume -using the `fields defined in PEP 621`_. +using the :pep:`fields defined in PEP 621 <621#dependencies-optional-dependencies>`. .. note:: - This PEP has been accepted and is expected to be merged into - :pep:`621`. + This PEP has been accepted and was merged into :pep:`621`. Entries ======= -All dependency entries MUST be valid `PEP 508 strings`_. +All dependency entries MUST be valid :pep:`PEP 508 strings <508>`. Build backends SHOULD abort at load time for any parsing errors. @@ -222,8 +223,6 @@ This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. -.. _fields defined in PEP 621: https://www.python.org/dev/peps/pep-0621/#dependencies-optional-dependencies -.. _PEP 508 strings: https://www.python.org/dev/peps/pep-0508/ .. _Requires-Dist: https://packaging.python.org/specifications/core-metadata/#requires-dist-multiple-use .. _Provides-Extra: https://packaging.python.org/specifications/core-metadata/#provides-extra-multiple-use .. _docker-compose: https://github.com/docker/compose/blob/789bfb0e8b2e61f15f423d371508b698c64b057f/setup.py#L28-L61 diff --git a/pep-0632.rst b/peps/pep-0632.rst similarity index 98% rename from pep-0632.rst rename to peps/pep-0632.rst index 3f225d4e7..efd40ec6e 100644 --- a/pep-0632.rst +++ b/peps/pep-0632.rst @@ -58,7 +58,7 @@ Deprecation and removal will make it obvious that issues should be fixed in the setuptools project, and will reduce a source of bug reports and unnecessary test maintenance. It will also help promote the development of alternative build backends, which can now be -supported more easily thanks to PEP 517. [4]_ +supported more easily thanks to :pep:`517`. Specification @@ -119,7 +119,7 @@ Code that imports distutils will no longer work from Python 3.12. The suggested migration path is to use the equivalent (though not identical) imports from setuptools (see [5]_), or to migrate to an -alternative build backend (see PEP 517 [4]_). +alternative build backend (see :pep:`517`). Code already exists in setuptools to transparently switch ``setup.py`` files using distutils onto their equivalents, and so most working @@ -247,9 +247,6 @@ References .. [3] setuptools Issue #417 - Adopt distutils (https://github.com/pypa/setuptools/issues/417) -.. [4] PEP 517 - A build-system independent format for source trees - (https://www.python.org/dev/peps/pep-0517/) - .. [5] Porting from Distutils (https://setuptools.readthedocs.io/en/latest/deprecated/distutils-legacy.html) diff --git a/pep-0633.rst b/peps/pep-0633.rst similarity index 97% rename from pep-0633.rst rename to peps/pep-0633.rst index 8dbcd3964..5e673b3c8 100644 --- a/pep-0633.rst +++ b/peps/pep-0633.rst @@ -6,9 +6,10 @@ Sponsor: Brett Cannon Discussions-To: https://discuss.python.org/t/dependency-specification-in-pyproject-toml-using-an-exploded-toml-table/5123/ Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 02-Sep-2020 -Post-History: 2020-09-02 +Post-History: 02-Sep-2020 Resolution: https://discuss.python.org/t/how-to-specify-dependencies-pep-508-strings-or-a-table-in-toml/5243/38 @@ -64,7 +65,7 @@ In the specification of multiple requirements with the same distribution name (where environment markers choose the appropriate dependency), the chosen solution is similar to `Poetry`_'s, where an array of requirements is allowed. -The direct-reference keys closely align with and utilise pep:`610` and +The direct-reference keys closely align with and utilise :pep:`610` and :pep:`440` as to reduce differences in the packaging ecosystem and rely on previous work in specification. @@ -86,8 +87,8 @@ To reduce confusion with this document being a specification for specifying dependencies, the word "requirement" is used to mean a :pep:`508` dependency specification. -The following tables are added to the added to the ``project`` table specified -in :pep:`621`. +The following tables are added to the ``project`` table specified in +:pep:`621`. .. _TOML: https://toml.io/ @@ -108,15 +109,14 @@ values can have one of the following types: - array: an array of requirement tables. It is an error to specify an empty array ``[]`` as a value. -.. _requirement-spec: - Requirement table ^^^^^^^^^^^^^^^^^ The keys of the requirement table are as follows (all are optional): -- ``version`` (string): a :pep:`440` version specifier, which is a comma- - delimited list of version specifier clauses. The string MUST be non-empty. +- ``version`` (string): a :pep:`440` version specifier, which is a + comma-delimited list of version specifier clauses. The string MUST be + non-empty. - ``extras`` (array of strings): a list of :pep:`508` extras declarations for the distribution. The list MUST be non-empty. @@ -164,7 +164,7 @@ The values can have one of the following types: - array: an array of requirement tables. These requirement tables have -`the same specification as above <#requirement-spec>`_, with the addition of +`the same specification as above `_, with the addition of the following required key: - ``for-extra`` (string): the name of the :pep:`508` extra that this @@ -740,12 +740,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0634.rst b/peps/pep-0634.rst similarity index 96% rename from pep-0634.rst rename to peps/pep-0634.rst index 8067a05c9..20ff9ff50 100644 --- a/pep-0634.rst +++ b/peps/pep-0634.rst @@ -5,7 +5,7 @@ Last-Modified: $Date$ Author: Brandt Bucher , Guido van Rossum BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Accepted Type: Standards Track Content-Type: text/x-rst @@ -20,22 +20,22 @@ Abstract ======== This PEP provides the technical specification for the match -statement. It replaces PEP 622, which is hereby split in three parts: +statement. It replaces :pep:`622`, which is hereby split in three parts: -- PEP 634: Specification -- PEP 635: Motivation and Rationale -- PEP 636: Tutorial +- :pep:`634`: Specification +- :pep:`635`: Motivation and Rationale +- :pep:`636`: Tutorial This PEP is intentionally devoid of commentary; the motivation and all -explanations of our design choices are in PEP 635. First-time readers -are encouraged to start with PEP 636, which provides a gentler +explanations of our design choices are in :pep:`635`. First-time readers +are encouraged to start with :pep:`636`, which provides a gentler introduction to the concepts, syntax and semantics of patterns. Syntax and Semantics ==================== -See `Appendix A`_ for the complete grammar. +See `Appendix A <634-appendix-a_>`_ for the complete grammar. Overview and Terminology ------------------------ @@ -134,7 +134,7 @@ The precise pattern binding rules vary per pattern type and are specified below. -.. _guards: +.. _634-guards: Guards ^^^^^^ @@ -169,7 +169,7 @@ A match statement may have at most one irrefutable case block, and it must be last. -.. _patterns: +.. _634-patterns: Patterns -------- @@ -225,7 +225,7 @@ until one succeeds. The OR pattern is then deemed to succeed. If none of the subpatterns succeed the OR pattern fails. -.. _literal_pattern: +.. _634-literal_pattern: Literal Patterns ^^^^^^^^^^^^^^^^ @@ -261,7 +261,7 @@ value expressed by the literal, using the following comparisons rules: using the ``is`` operator. -.. _capture_pattern: +.. _634-capture_pattern: Capture Patterns ^^^^^^^^^^^^^^^^ @@ -275,7 +275,7 @@ The single underscore (``_``) is not a capture pattern (this is what A capture pattern always succeeds. It binds the subject value to the name using the scoping rules for name binding established for the -walrus operator in PEP 572. (Summary: the name becomes a local +walrus operator in :pep:`572`. (Summary: the name becomes a local variable in the closest containing function scope unless there's an applicable ``nonlocal`` or ``global`` statement.) @@ -284,7 +284,7 @@ disallows for example ``case x, x: ...`` but allows ``case [x] | x: ...``. -.. _wildcard_pattern: +.. _634-wildcard_pattern: Wildcard Pattern ^^^^^^^^^^^^^^^^ @@ -331,7 +331,7 @@ A parenthesized pattern has no additional syntax. It allows users to add parentheses around patterns to emphasize the intended grouping. -.. _sequence_pattern: +.. _634-sequence_pattern: Sequence Patterns ^^^^^^^^^^^^^^^^^ @@ -411,7 +411,7 @@ then matched to the corresponding subject items, as for a fixed-length sequence. -.. _mapping_pattern: +.. _634-mapping_pattern: Mapping Patterns ^^^^^^^^^^^^^^^^ @@ -466,7 +466,7 @@ with keys that were already present when the match statement was entered. -.. _class_pattern: +.. _634-class_pattern: Class Patterns ^^^^^^^^^^^^^^ @@ -592,7 +592,7 @@ existing standard library classes and adding ``__match_args__`` where it looks beneficial. -.. _Appendix A: +.. _634-appendix-a: Appendix A -- Full Grammar ========================== @@ -682,13 +682,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0635.rst b/peps/pep-0635.rst similarity index 98% rename from pep-0635.rst rename to peps/pep-0635.rst index edb6d4aa8..9e25e8f20 100644 --- a/pep-0635.rst +++ b/peps/pep-0635.rst @@ -5,7 +5,7 @@ Last-Modified: $Date$ Author: Tobias Kohn , Guido van Rossum BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -18,9 +18,9 @@ Resolution: https://mail.python.org/archives/list/python-committers@python.org/m Abstract ======== -This PEP provides the motivation and rationale for PEP 634 +This PEP provides the motivation and rationale for :pep:`634` ("Structural Pattern Matching: Specification"). First-time readers -are encouraged to start with PEP 636, which provides a gentler +are encouraged to start with :pep:`636`, which provides a gentler introduction to the concepts, syntax and semantics of patterns. @@ -97,8 +97,8 @@ but also on the value of some class attributes, like the ``BinOp`` example above. The Visitor pattern is insufficiently flexible for this: it can only select based on the class. -For a complete example, see -https://github.com/gvanrossum/patma/blob/master/examples/expr.py#L231 +See a `complete example +`_. Like the Visitor pattern, pattern matching allows for a strict separation of concerns: specific actions or data processing is independent of the @@ -155,7 +155,7 @@ Rationale This section provides the rationale for individual design decisions. It takes the place of "Rejected ideas" in the standard PEP format. -It is organized in sections corresponding to the specification (PEP 634). +It is organized in sections corresponding to the specification (:pep:`634`). Overview and Terminology @@ -397,7 +397,7 @@ and not for individual patterns. return a + [p] + b -.. _patterns: +.. _635-patterns: Patterns -------- @@ -586,7 +586,7 @@ without adding a significant benefit. It can always be added later. return expr -.. _literal_pattern: +.. _635-literal_pattern: Literal Patterns ~~~~~~~~~~~~~~~~ @@ -675,7 +675,7 @@ being. return expr -.. _capture_pattern: +.. _635-capture_pattern: Capture Patterns ~~~~~~~~~~~~~~~~ @@ -734,7 +734,7 @@ especially given that we expect capture patterns to be very common. return sum(a) / len(a) -.. _wildcard_pattern: +.. _635-wildcard_pattern: Wildcard Pattern ~~~~~~~~~~~~~~~~ @@ -775,7 +775,7 @@ ultimate "wildcard", it does not convey the desired semantics. An alternative that does not suggest an arbitrary number of items would be ``?``. This is even being proposed independently from -pattern matching in PEP 640. We feel however that using ``?`` as a +pattern matching in :pep:`640`. We feel however that using ``?`` as a special "assignment" target is likely more confusing to Python users than using ``_``. It violates Python's (admittedly vague) principle of using punctuation characters only in ways similar to how they are @@ -790,7 +790,7 @@ globbing, "maybe" in regular expressions, "conditional expression" in C and many C-derived languages, "predicate function" in Scheme, "modify error handling" in Rust, "optional argument" and "optional chaining" in TypeScript (the latter meaning has also been proposed for -Python by PEP 505). An as yet unnamed PEP proposes it to mark +Python by :pep:`505`). An as yet unnamed PEP proposes it to mark optional types, e.g. ``int?``. Another common use of ``?`` in programming systems is "help", for @@ -901,7 +901,7 @@ Allowing users to explicitly specify the grouping is particularly helpful in case of OR patterns. -.. _sequence_pattern: +.. _635-sequence_pattern: Sequence Patterns ~~~~~~~~~~~~~~~~~ @@ -961,7 +961,7 @@ enumerate all other types that may be used to represent bytes (e.g. some, but not all, instances of ``memoryview`` and ``array.array``). -.. _mapping_pattern: +.. _635-mapping_pattern: Mapping Patterns ~~~~~~~~~~~~~~~~ @@ -1008,7 +1008,7 @@ unexpected side effect. change_red_to_blue(child) -.. _class_pattern: +.. _635-class_pattern: Class Patterns ~~~~~~~~~~~~~~ @@ -1235,7 +1235,7 @@ an instance of ``Node`` but only extract the value of the ``right`` attribute. Backwards Compatibility ======================= -Through its use of "soft keywords" and the new PEG parser (PEP 617), +Through its use of "soft keywords" and the new PEG parser (:pep:`617`), the proposal remains fully backwards compatible. However, 3rd party tooling that uses a LL(1) parser to parse Python source code may be forced to switch parser technology to be able to support those same @@ -1278,13 +1278,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0636.rst b/peps/pep-0636.rst similarity index 97% rename from pep-0636.rst rename to peps/pep-0636.rst index ccec6ed05..3f9353aac 100644 --- a/pep-0636.rst +++ b/peps/pep-0636.rst @@ -5,7 +5,7 @@ Last-Modified: $Date$ Author: Daniel F Moisset Sponsor: Guido van Rossum BDFL-Delegate: -Discussions-To: Python-Dev +Discussions-To: python-dev@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -18,20 +18,20 @@ Resolution: https://mail.python.org/archives/list/python-committers@python.org/m Abstract ======== -This PEP is a tutorial for the pattern matching introduced by PEP 634. +This PEP is a tutorial for the pattern matching introduced by :pep:`634`. -PEP 622 proposed syntax for pattern matching, which received detailed discussion +:pep:`622` proposed syntax for pattern matching, which received detailed discussion both from the community and the Steering Council. A frequent concern was about how easy it would be to explain (and learn) this feature. This PEP addresses that concern providing the kind of document which developers could use to learn about pattern matching in Python. -This is considered supporting material for PEP 634 (the technical specification -for pattern matching) and PEP 635 (the motivation and rationale for having pattern +This is considered supporting material for :pep:`634` (the technical specification +for pattern matching) and :pep:`635` (the motivation and rationale for having pattern matching and design considerations). For readers who are looking more for a quick review than for a tutorial, -see `Appendix A`_. +see `Appendix A `_. Tutorial ======== @@ -435,7 +435,7 @@ Any class is a valid match target, and that includes built-in classes like ``boo ``str`` or ``int``. That allows us to combine the code above with a class pattern. So instead of writing ``{"text": message, "color": c}`` we can use ``{"text": str() as message, "color": str() as c}`` to ensure that ``message`` and ``c`` -are both strings. For many builtin classes (see PEP-634 for the whole list), you can +are both strings. For many builtin classes (see :pep:`634` for the whole list), you can use a positional parameter as a shorthand, writing ``str(c)`` rather than ``str() as c``. The fully rewritten version looks like this:: @@ -453,7 +453,7 @@ The fully rewritten version looks like this:: -.. _Appendix A: +.. _PEP 636 Appendix A: Appendix A -- Quick Intro ========================= @@ -614,13 +614,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0637.rst b/peps/pep-0637.rst similarity index 97% rename from pep-0637.rst rename to peps/pep-0637.rst index 50cd30d63..100fe9f78 100644 --- a/pep-0637.rst +++ b/peps/pep-0637.rst @@ -42,7 +42,7 @@ arguments is also provided:: >>> val = x[*(1, 2), **{a=3, b=4}] # Equivalent to above. -This PEP is a successor to PEP 472, which was rejected due to lack of +This PEP is a successor to :pep:`472`, which was rejected due to lack of interest in 2019. Since then there's been renewed interest in the feature. Overview @@ -51,7 +51,7 @@ Overview Background ---------- -PEP 472 was opened in 2014. The PEP detailed various use cases and was created by +:pep:`472` was opened in 2014. The PEP detailed various use cases and was created by extracting implementation strategies from a broad discussion on the python-ideas mailing list, although no clear consensus was reached on which strategy should be used. Many corner cases have been examined more closely and felt @@ -60,7 +60,7 @@ awkward, backward incompatible or both. The PEP was eventually rejected in 2019 [#rejection]_ mostly due to lack of interest for the feature despite its 5 years of existence. -However, with the introduction of type hints in PEP 484 [#pep-0484]_ the +However, with the introduction of type hints in :pep:`484` the square bracket notation has been used consistently to enrich the typing annotations, e.g. to specify a list of integers as Sequence[int]. Additionally, there has been an expanded growth of packages for data analysis such as pandas @@ -160,7 +160,7 @@ specification would improve notation and provide additional value: >>> # The call is now shorter, more mnemonic, and looks+works like typing >>> trio.run[name="func"](func, value1, param2=value2) -7. Availability of star arguments would benefit PEP-646 Variadic Generics [#pep-0646]_, +7. Availability of star arguments would benefit :pep:`646` Variadic Generics, especially in the forms ``a[*x]`` and ``a[*x, *y, p, q, *z]``. The PEP details exactly this notation in its "Unpacking: Star Operator" section. @@ -480,7 +480,7 @@ The successful implementation of this PEP will result in the following behavior: obj[] # Invalid. 12. The same semantics given above must be extended to ``__class__getitem__``: - Since PEP 560, type hints are dispatched so that for ``x[y]``, if no + Since :pep:`560`, type hints are dispatched so that for ``x[y]``, if no ``__getitem__`` method is found, and ``x`` is a type (class) object, and ``x`` has a class method ``__class_getitem__``, that method is called. The same changes should be applied to this method as well, @@ -689,9 +689,9 @@ A reference implementation is currently being developed here [#reference-impl]_. Workarounds =========== -Every PEP that changes the Python language should "clearly explain why +Every PEP that changes the Python language should :pep:`"clearly explain why the existing language specification is inadequate to address the -problem that the PEP solves." [#pep-0001]_ +problem that the PEP solves" <1#what-belongs-in-a-successful-pep>`. Some rough equivalents to the proposed extension, which we call work-arounds, are already possible. The work-arounds provide an alternative to enabling the @@ -755,7 +755,7 @@ Rejected Ideas Previous PEP 472 solutions -------------------------- -PEP 472 presents a good amount of ideas that are now all to be considered +:pep:`472` presents a good amount of ideas that are now all to be considered Rejected. A personal email from D'Aprano to the author specifically said: I have now carefully read through PEP 472 in full, and I am afraid I @@ -765,7 +765,7 @@ We agree that those options are inferior to the currently presented, for one reason or another. To keep this document compact, we will not present here the objections for -all options presented in PEP 472. Suffice to say that they were discussed, +all options presented in :pep:`472`. Suffice to say that they were discussed, and each proposed alternative had one or few dealbreakers. Adding new dunders @@ -842,7 +842,7 @@ Has problems similar to the above. create a new "kwslice" object ----------------------------- -This proposal has already been explored in "New arguments contents" P4 in PEP 472:: +This proposal has already been explored in "New arguments contents" P4 in :pep:`472`:: obj[a, b:c, x=1] # calls type(obj).__getitem__(obj, a, slice(b, c), key(x=1)) @@ -1096,18 +1096,12 @@ References .. [#rejection] "Rejection of PEP 472" (https://mail.python.org/pipermail/python-dev/2019-March/156693.html) -.. [#pep-0484] "PEP 484 -- Type hints" - (https://www.python.org/dev/peps/pep-0484) .. [#request-1] "Allow kwargs in __{get|set|del}item__" (https://mail.python.org/archives/list/python-ideas@python.org/thread/EUGDRTRFIY36K4RM3QRR52CKCI7MIR2M/) .. [#request-2] "PEP 472 -- Support for indexing with keyword arguments" (https://mail.python.org/archives/list/python-ideas@python.org/thread/6OGAFDWCXT5QVV23OZWKBY4TXGZBVYZS/) -.. [#pep-0001] "PEP 1 -- PEP Purpose and Guidelines" - (https://www.python.org/dev/peps/pep-0001/#what-belongs-in-a-successful-pep) .. [#trio-run] "trio.run() should take \*\*kwargs in addition to \*args" (https://github.com/python-trio/trio/issues/470) -.. [#pep-0646] "PEP 646 -- Variadic Generics" - (https://www.python.org/dev/peps/pep-0646) .. [#numpy-ml] "[Numpy-discussion] Request for comments on PEP 637 - Support for indexing with keyword arguments" (http://numpy-discussion.10968.n7.nabble.com/Request-for-comments-on-PEP-637-Support-for-indexing-with-keyword-arguments-td48489.html) .. [#reference-impl] "Reference implementation" diff --git a/pep-0638.rst b/peps/pep-0638.rst similarity index 94% rename from pep-0638.rst rename to peps/pep-0638.rst index 2e1884248..7498d3200 100644 --- a/pep-0638.rst +++ b/peps/pep-0638.rst @@ -1,10 +1,12 @@ PEP: 638 Title: Syntactic Macros Author: Mark Shannon +Discussions-To: https://mail.python.org/archives/list/python-dev@python.org/thread/U4C4XHNRC4SHS3TPZWCTY4SN4QU3TT6V/ Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 24-Sep-2020 +Post-History: 26-Sep-2020 Abstract ======== @@ -169,7 +171,7 @@ Compilation Upon encountering a ``macro`` during translation to bytecode, the code generator will look up the macro processor registered for the macro, -and pass the AST, rooted at the macro to the processor function. +and pass the AST rooted at the macro to the processor function. The returned AST will then be substituted for the original tree. For macros with multiple names, @@ -219,14 +221,14 @@ Defining macro processors ~~~~~~~~~~~~~~~~~~~~~~~~~ A macro processor is defined by a four-tuple, consisting of -``(func, kind, version, additional_names)`` +``(func, kind, version, additional_names)``: * ``func`` must be a callable that takes ``len(additional_names)+1`` arguments, all of which are abstract syntax trees, and returns a single abstract syntax tree. * ``kind`` must be one of the following: - * ``macros.STMT_MACRO`` A statement macro where the body of the macro is indented. This is the only form allowed to have additional names. - * ``macros.SIBLING_MACRO`` A statement macro where the body of the macro is the next statement is the same block. The following statement is moved into the macro as its body. - * ``macros.EXPR_MACRO`` An expression macro. + * ``macros.STMT_MACRO``: A statement macro where the body of the macro is indented. This is the only form allowed to have additional names. + * ``macros.SIBLING_MACRO``: A statement macro where the body of the macro is the next statement in the same block. The following statement is moved into the macro as its body. + * ``macros.EXPR_MACRO``: An expression macro. * ``version`` is used to track versions of macros, so that generated bytecodes can be correctly cached. It must be an integer. * ``additional_names`` are the names of the additional parts of the macro, and must be a tuple of strings. @@ -278,22 +280,19 @@ Two new AST nodes will be needed to express macros, ``macro_stmt`` and ``macro_e :: class macro_stmt(_ast.stmt): - _fields = "name", "args", "importname", "asname", "body" class macro_expr(_ast.expr): - _fields = "name", "args" -In addition, macro processors will needs a means to express control flow or side-effecting code, that produces a value. -To support this, a new ast node, called ``stmt_expr``, that combines a statement and expression will be added. +In addition, macro processors will need a means to express control flow or side-effecting code, that produces a value. +A new AST node called ``stmt_expr`` will be added, combining a statement and an expression. This new ast node will be a subtype of ``expr``, but include a statement to allow side effects. It will be compiled to bytecode by compiling the statement, then compiling the value. :: class stmt_expr(_ast.expr): - _fields = "stmt", "value" Hygiene and debugging @@ -433,7 +432,7 @@ Which could be converted to: Zero-cost markers and annotations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Annotations, either decorators or PEP 3107 function annotations, have a runtime cost +Annotations, either decorators or :pep:`3107` function annotations, have a runtime cost even if they serve only as markers for checkers or as documentation. :: @@ -450,8 +449,8 @@ can be replaced with the zero-cost macro: def foo(...): ... -Protyping language extensions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Prototyping language extensions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Although macros would be most valuable for domain-specific extensions, it is possible to demonstrate possible language extensions using macros. diff --git a/peps/pep-0639.rst b/peps/pep-0639.rst new file mode 100644 index 000000000..b655733ae --- /dev/null +++ b/peps/pep-0639.rst @@ -0,0 +1,2788 @@ +PEP: 639 +Title: Improving License Clarity with Better Package Metadata +Author: Philippe Ombredanne , + C.A.M. Gerlach , +PEP-Delegate: Brett Cannon +Discussions-To: https://discuss.python.org/t/12622 +Status: Draft +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 15-Aug-2019 +Post-History: `15-Aug-2019 `__, + `17-Dec-2021 `__, + + +.. _639-abstract: + +Abstract +======== + +This PEP defines a specification for how licenses are documented in the +`core metadata `__, with +:ref:`license expression strings <639-spec-field-license-expression>` using +`SPDX identifiers `__ in a new ``License-Expression`` field. +This will make license declarations simpler and less ambiguous for +package authors to create, end users to read and understand, and +tools to programmatically process. + +The PEP also: + +- :ref:`Formally specifies <639-spec-field-license-file>` + a new ``License-File`` field, and defines how license files should be + :ref:`included in distributions <639-spec-project-formats>`, + as already used by the Wheel and Setuptools projects. + +- Deprecates the legacy ``License`` :ref:`field <639-spec-field-license>` + and ``license ::`` :ref:`classifiers <639-spec-field-classifier>`. + +- :ref:`Adds and deprecates <639-spec-source-metadata>` the corresponding keys + in the ``pyproject.toml`` ``[project]`` table. + +- :ref:`Provides clear guidance <639-spec-converting-metadata>` for authors and + tools converting legacy license metadata, adding license files and + validating license expressions. + +- Describes a :ref:`reference implementation <639-reference-implementation>`, + analyzes numerous :ref:`potential alternatives <639-rejected-ideas>`, + includes :ref:`detailed examples <639-examples>`, + explains :ref:`user scenarios <639-user-scenarios>` and + surveys license documentation + :ref:`in Python packaging <639-license-doc-python>` and + :ref:`other ecosystems <639-license-doc-other-projects>`. + +The changes in this PEP will update the +`core metadata `__ to version 2.4, modify the +`project (source) metadata specification `__, +and make minor additions to the `source distribution (sdist) `__, +`built distribution (wheel) `__ and +`installed project `__ standards. + + +.. _639-goals: + +Goals +===== + +This PEP's scope is limited to covering new mechanisms for documenting +the license of a distribution package, specifically defining: + +- A means of specifying a SPDX license expression. +- A method of including license texts in distributions and installed projects. + +The changes to the core metadata specification that this PEP requires have been +designed to minimize impact and maximize backward compatibility. +This specification builds off of existing ways to document licenses that are +already in use in popular tools (e.g. adding support to core metadata for the +``License-File`` field :ref:`already used <639-license-doc-setuptools-wheel>` +in the Wheel and Setuptools projects) and by some package authors +(e.g. storing an SPDX license expression in the existing ``License`` field). + +In addition to these proposed changes, this PEP contains guidance for tools +handling and converting these metadata, a tutorial for package authors +covering various common use cases, detailed examples of them in use, +and a comprehensive survey of license documentation in Python and other +languages. + +It is the intent of the PEP authors to work closely with tool maintainers to +implement the recommendations for validation and warnings specified here. + + +.. _639-non-goals: + +Non-Goals +========= + +This PEP is neutral regarding the choice of license by any particular +package author. This PEP makes no recommendation for specific licenses, +and does not require the use of a particular license documentation convention. + +Rather, the SPDX license expression syntax proposed in this PEP provides a +simpler and more expressive mechanism to accurately document any kind of +license that applies to a Python package, whether it is open source, +free/libre, proprietary, or a combination of such. + +This PEP also does not impose any additional restrictions when uploading to +PyPI, unless projects choose to make use of the new fields. + +Instead, it is intended to document best practices already in use, extend them +to use a new formally-specified and supported mechanism, and provide guidance +for packaging tools on how to hand the transition and inform users accordingly. + +This PEP also is not about license documentation in files inside projects, +though this is a :ref:`surveyed topic <639-license-doc-source-files>` +in an appendix, and nor does it intend to cover cases where the source and +binary distribution packages don't have :ref:`the same licenses +<639-rejected-ideas-difference-license-source-binary>`. + + +.. _639-motivation: + +Motivation +========== + +Software must be licensed in order for anyone other than its creator to +download, use, share and modify it, so providing accurate license information +to Python package users is an important matter. +Today, there are multiple fields where +licenses are documented in core metadata, and there are limitations to what +can be expressed in each of them. This often leads to confusion and a lack of +clarity, both for package authors and end users. + +Many package authors have expressed difficulty and frustrations due to the +limited capabilities to express licensing in project metadata, and this +creates further trouble for Linux and BSD distribution re-packagers. +This has triggered a number of license-related discussions and issues, +including on `outdated and ambiguous PyPI classifiers `__, +`license interoperability with other ecosystems `__, +`too many confusing license metadata options `__, +`limited support for license files in the Wheel project `__, and +`the lack of clear, precise and standardized license metadata `__. + +The current license classifiers address some common cases, and could +be extended to include the full range of current SPDX identifiers +while deprecating the many ambiguous classifiers +(including some popular and problematic ones, +such as ``License :: OSI Approved :: BSD License``). +However, this requires a substantial amount of effort +to duplicate the SPDX license list and keep it in sync. +Furthermore, it is effectively a hard break in backward compatibility, +forcing a huge proportion of package authors to immediately update to new +classifiers (in most cases, with many possible choices that require closely +examining the project's license) immediately when PyPI deprecates the old ones. + +Furthermore, this only covers simple packages entirely under a single license; +it doesn't address the substantial fraction of common projects that vendor +dependencies (e.g. Setuptools), offer a choice of licenses (e.g. Packaging) +or were relicensed, adapt code from other projects or contain fonts, images, +examples, binaries or other assets under other licenses. It also requires +both authors and tools understand and implement the PyPI-specific bespoke +classifier system, rather than using short, easy to add and standardized +SPDX identifiers in a simple text field, as increasingly widely adopted by +most other packaging systems to reduce the overall burden on the ecosystem. +Finally, this does not provide as clear an indicator that a package +has adopted the new system, and should be treated accordingly. + +On average, Python packages tend to have more ambiguous and missing license +information than other common ecosystems (such as npm, Maven or +Gem). This is supported by the `statistics page `__ of the +`ClearlyDefined project `__, an +`Open Source Initiative `__ incubated effort to help +improve licensing clarity of other FOSS projects, covering all packages +from PyPI, Maven, npm and Rubygems. + + +.. _639-rationale: + +Rationale +========= + +A survey of existing license metadata definitions in use in the Python +ecosystem today is provided in +:ref:`an appendix <639-license-doc-python>` of this PEP, +and license documentation in a variety of other packaging systems, +Linux distros, languages ecosystems and applications is surveyed in +:ref:`another appendix <639-license-doc-other-projects>`. + +There are a few takeaways from the survey, which have guided the design +and recommendations of this PEP: + +- Most package formats use a single ``License`` field. + +- Many modern package systems use some form of license expression syntax to + optionally combine more than one license identifier together. + SPDX and SPDX-like syntaxes are the most popular in use. + +- SPDX license identifiers are becoming the de facto way to reference common + licenses everywhere, whether or not a full license expression syntax is used. + +- Several package formats support documenting both a license expression and the + paths of the corresponding files that contain the license text. Most Free and + Open Source Software licenses require package authors to include their full + text in a distribution. + +The use of a new ``License-Expression`` field will provide an intuitive, +structured and unambiguous way to express the license of a +package using a well-defined syntax and well-known license identifiers. +Similarly, a formally-specified ``License-File`` field offers a standardized +way to ensure that the full text of the license(s) are included with the +package when distributed, as legally required, and allows other tools consuming +the core metadata to unambiguously locate a distribution's license files. + +While dramatically simplifying and improving the present Python license +metadata story, this specification standardizes and builds upon +existing practice in the `Setuptools `__ and +`Wheel `__ projects. +Furthermore, an up-to-date version of the current draft of this PEP is +`already successfully implemented `__ in the popular +PyPA `Hatch `__ packaging tool, and an earlier draft of the +license files portion is `implemented in Setuptools `__. + +Over time, encouraging the use of these fields and deprecating the ambiguous, +duplicative and confusing legacy alternatives will help Python software +publishers improve the clarity, accuracy and portability of their licensing +practices, to the benefit of package authors, consumers and redistributors +alike. + + +.. _639-terminology: + +Terminology +=========== + +This PEP seeks to clearly define the terms it uses, given that some have +multiple established meanings (e.g. import vs. distribution package, +wheel *format* vs. Wheel *project*); are related and often used +interchangeably, but have critical distinctions in meaning +(e.g. ``[project]`` *key* vs. core metadata *field*); are existing concepts +that don't have formal terms/definitions (e.g. project/source metadata vs. +distribution/built metadata, build vs. publishing tools), or are new concepts +introduced here (e.g. license expression/identifier). + +This PEP also uses terms defined in the +`PyPA PyPUG Glossary `__ +(specifically *built/binary distribution*, *distribution package*, +*project* and *source distribution*), and by the `SPDX Project `__ +(*license identifier*, *license expression*). + +The keywords "MUST", "MUST NOT", "REQUIRED", +"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" +in this document are to be interpreted as described in :rfc:`2119`. + +Terms are listed here in their full versions; +related words (``Rel:``) are in parenthesis, +including short forms (``Short:``), sub-terms (``Sub:``) and common synonyms +for the purposes of this PEP (``Syn:``). + +**Core Metadata** *(Syn: Package Metadata, Sub: Distribution Metadata)* + The `PyPA specification `__ and the set of metadata fields + it defines that describe key static attributes of distribution packages + and installed projects. + + The **distribution metadata** refers to, more specifically, the concrete form + core metadata takes when included inside a distribution archive + (``PKG-INFO`` in a sdist and ``METADATA`` in a wheel) or installed project + (``METADATA``). + +**Core Metadata Field** *(Short: Metadata Field/Field)* + A single key-value pair, or sequence of such with the same key, as defined + by the `core metadata specification `__. + Notably, distinct from a ``pyproject.toml`` ``[project]`` table *key*. + +**Distribution Package** *(Sub: Package, Distribution Archive)* + (`See PyPUG `__) + In this PEP, **package** is used to refer to the abstract concept of a + distributable form of a Python project, while **distribution** more + specifically references the physical **distribution archive**. + +**License Classifier** + A `PyPI Trove classifier `__ + (as `described in the core metadata specification + `__) + which begins with ``License ::``, currently used to indicate + a project's license status by including it as a ``Classifier`` + in the core metadata. + +**License Expression** *(Syn: SPDX Expression)* + A string with valid `SPDX license expression syntax `__ + including any SPDX license identifiers as defined here, which describes + a project's license(s) and how they relate to one another. Examples: + ``GPL-3.0-or-later``, ``MIT AND (Apache-2.0 OR BSD-2-clause)`` + +**License Identifier** *(Syn: License ID/SPDX Identifier)* + A valid `SPDX short-form license identifier `__, as described in the + :ref:`639-spec-field-license-expression` section of this PEP; briefly, + this includes all valid SPDX identifiers and the ``LicenseRef-Public-Domain`` + and ``LicenseRef-Proprietary`` strings. Examples: ``MIT``, ``GPL-3.0-only`` + +**Project** *(Sub: Project Source Tree, Installed Project)* + (`See PyPUG `__) + Here, a **project source tree** refers to the on-disk format of + a project used for development, while an **installed project** is the form a + project takes once installed from a distribution, as + `specified by PyPA `__. + +**Project Source Metadata** *(Sub: Project Table Metadata, Key, Subkey)* + Core metadata defined by the package author in the project source tree, + as top-level keys in the ``[project]`` table of a ``pyproject.toml`` file, + in the ``[metadata]`` table of ``setup.cfg``, or the equivalent for other + build tools. + + The **Project Table Metadata**, or ``pyproject.toml`` ``[project]`` metadata, + refers specifically to the former, as defined by the + `PyPA Declaring Project Metadata specification `__ + and originally specified in :pep:`621`. + A **Project Table Key**, or an unqualified *key* refers specifically to + a top-level ``[project]`` key + (notably, distinct from a core metadata *field*), + while a **subkey** refers to a second-level key in a table-valued + ``[project]`` key. + +**Root License Directory** *(Short: License Directory)* + The directory under which license files are stored in a project/distribution + and the root directory that their paths, as recorded under the + ``License-File`` core metadata fields, are relative to. + Defined here to be the project root directory for source trees and source + distributions, and a subdirectory named ``licenses`` of the directory + containing the core metadata (i.e., the ``.dist-info/licenses`` + directory) for built distributions and installed projects. + +**Tool** *(Sub: Packaging Tool, Build Tool, Install Tool, Publishing Tool)* + A program, script or service executed by the user or automatically that + seeks to conform to the specification defined in this PEP. + + A **packaging tool** refers to a tool used to build, publish, + install, or otherwise directly interact with Python packages. + + A **build tool** is a packaging tool used to generate a source or built + distribution from a project source tree or sdist, when directly invoked + as such (as opposed to by end-user-facing install tools). + Examples: Wheel project, :pep:`517` backends via ``build`` or other + package-developer-facing frontends, calling ``setup.py`` directly. + + An **install tool** is a packaging tool used to install a source or built + distribution in a target environment. Examples include the PyPA pip and + ``installer`` projects. + + A **publishing tool** is a packaging tool used to upload distribution + archives to a package index, such as Twine for PyPI. + +**Wheel** *(Short: wheel, Rel: wheel format, Wheel project)* + Here, **wheel**, the standard built distribution format introduced in + :pep:`427` and `specified by the PyPA `__, will be referred to in + lowercase, while the `Wheel project `__, its reference + implementation, will be referred to as such with **Wheel** in Title Case. + + +.. _639-specification: + +Specification +============= + +The changes necessary to implement the improved license handling outlined in +this PEP include those in both +:ref:`distribution package metadata <639-spec-core-metadata>`, +as defined in the `core metadata specification `__, and +:ref:`author-provided project source metadata <639-spec-source-metadata>`, +as defined in the `project source metadata specification <_pep621spec>`__ +(and originally introduced in :pep:`621`). + +Further, :ref:`minor additions <639-spec-project-formats>` to the +source distribution (sdist), built distribution (wheel) and installed project +specifications will help document and clarify the already allowed, +now formally standardized behavior in these respects. +Finally, :ref:`guidance is established <639-spec-converting-metadata>` +for tools handling and converting legacy license metadata to license +expressions, to ensure the results are consistent, correct and unambiguous. + +Note that the guidance on errors and warnings is for tools' default behavior; +they MAY operate more strictly if users explicitly configure them to do so, +such as by a CLI flag or a configuration option. + + +.. _639-spec-core-metadata: + +Core metadata +------------- + +The `PyPA Core Metadata specification `__ defines the names +and semantics of each of the supported fields in the distribution metadata of +Python distribution packages and installed projects. + +This PEP :ref:`adds <639-spec-field-license-expression>` the +``License-Expression`` field, +:ref:`adds <639-spec-field-license-file>` the ``License-File`` field, +:ref:`deprecates <639-spec-field-license>` the ``License`` field, +and :ref:`deprecates <639-spec-field-classifier>` the license classifiers +in the ``Classifier`` field. + +The error and warning guidance in this section applies to build and +publishing tools; end-user-facing install tools MAY be more lenient than +mentioned here when encountering malformed metadata +that does not conform to this specification. + +As it adds new fields, this PEP updates the core metadata to version 2.4. + + +.. _639-spec-field-license-expression: + +Add ``License-Expression`` field +'''''''''''''''''''''''''''''''' + +The ``License-Expression`` optional field is specified to contain a text string +that is a valid SPDX license expression, as defined herein. + +Publishing tools SHOULD issue an informational warning if this field is +missing, and MAY raise an error. Build tools MAY issue a similar warning, +but MUST NOT raise an error. + +.. _639-license-expression-definition: + +A license expression is a string using the SPDX license expression syntax as +documented in the `SPDX specification `__, either +Version 2.2 or a later compatible version. + +When used in the ``License-Expression`` field and as a specialization of +the SPDX license expression definition, a license expression can use the +following license identifiers: + +- Any SPDX-listed license short-form identifiers that are published in the + `SPDX License List `__, version 3.17 or any later compatible + version. Note that the SPDX working group never removes any license + identifiers; instead, they may choose to mark an identifier as "deprecated". + +- The ``LicenseRef-Public-Domain`` and ``LicenseRef-Proprietary`` strings, to + identify licenses that are not included in the SPDX license list. + +When processing the ``License-Expression`` field to determine if it contains +a valid license expression, build and publishing tools: + +- SHOULD halt execution and raise an error if: + + - The field does not contain a valid license expression + + - One or more license identifiers are not valid + (as :ref:`defined above <639-license-expression-definition>`) + +- SHOULD report an informational warning, and publishing tools MAY raise an + error, if one or more license identifiers have been marked as deprecated in + the `SPDX License List `__. + +- MUST store a case-normalized version of the ``License-Expression`` field + using the reference case for each SPDX license identifier and + uppercase for the ``AND``, ``OR`` and ``WITH`` keywords. + +- SHOULD report an informational warning, and MAY raise an error if + the normalization process results in changes to the + ``License-Expression`` field contents. + +For all newly-upload distributions that include a +``License-Expression`` field, the `Python Package Index (PyPI) `__ MUST +validate that it contains a valid, case-normalized license expression with +valid identifiers (as defined here) and MUST reject uploads that do not. +PyPI MAY reject an upload for using a deprecated license identifier, +so long as it was deprecated as of the above-mentioned SPDX License List +version. + + +.. _639-spec-field-license-file: + +Add ``License-File`` field +'''''''''''''''''''''''''' + +Each instance of the ``License-File`` optional field is specified to contain +the string representation of the path in the project source tree, relative to +the project root directory, of a license-related file. +It is a multi-use field that may appear zero or +more times, each instance listing the path to one such file. Files specified +under this field could include license text, author/attribution information, +or other legal notices that need to be distributed with the package. + +As :ref:`specified by this PEP <639-spec-project-formats>`, its value +is also that file's path relative to the root license directory in both +installed projects and the standardized distribution package types. +In other legacy, non-standard or new distribution package formats and +mechanisms of accessing and storing core metadata, the value MAY correspond +to the license file path relative to a format-defined root license directory. +Alternatively, it MAY be treated as a unique abstract key to access the +license file contents by another means, as specified by the format. + +If a ``License-File`` is listed in a source or built distribution's core +metadata, that file MUST be included in the distribution at the specified path +relative to the root license directory, and MUST be installed with the +distribution at that same relative path. + +The specified relative path MUST be consistent between project source trees, +source distributions (sdists), built distributions (wheels) and installed +projects. Therefore, inside the root license directory, packaging tools +MUST reproduce the directory structure under which the +source license files are located relative to the project root. + +Path delimiters MUST be the forward slash character (``/``), +and parent directory indicators (``..``) MUST NOT be used. +License file content MUST be UTF-8 encoded text. + +Build tools MAY and publishing tools SHOULD produce an informative warning +if a built distribution's metadata contains no ``License-File`` entries, +and publishing tools MAY but build tools MUST NOT raise an error. + +For all newly-uploaded distribution packages that include one or more +``License-File`` fields and declare a ``Metadata-Version`` of ``2.4`` or +higher, PyPI SHOULD validate that the specified files are present in all +uploaded distributions, and MUST reject uploads that do not validate. + + +.. _639-spec-field-license: + +Deprecate ``License`` field +''''''''''''''''''''''''''' + +The legacy unstructured-text ``License`` field is deprecated and replaced by +the new ``License-Expression`` field. Build and publishing tools MUST raise +an error if both these fields are present and their values are not identical, +including capitalization and excluding leading and trailing whitespace. + +If only the ``License`` field is present, such tools SHOULD issue a warning +informing users it is deprecated and recommending ``License-Expression`` +instead. + +For all newly-uploaded distributions that include a +``License-Expression`` field, the `Python Package Index (PyPI) `__ MUST +reject any that specify a ``License`` field and the text of which is not +identical to that of ``License-Expression``, as defined in this section. + +Along with license classifiers, the ``License`` field may be removed from a +new version of the specification in a future PEP. + + +.. _639-spec-field-classifier: + +Deprecate license classifiers +''''''''''''''''''''''''''''' + +Using license `classifiers `__ in the ``Classifier`` field +(`described in the core metadata specification `__) +is deprecated and replaced by the more precise ``License-Expression`` field. + +If the ``License-Expression`` field is present, build tools SHOULD and +publishing tools MUST raise an error if one or more license classifiers +is included in a ``Classifier`` field, and MUST NOT add +such classifiers themselves. + +Otherwise, if this field contains a license classifier, build tools MAY +and publishing tools SHOULD issue a warning informing users such classifiers +are deprecated, and recommending ``License-Expression`` instead. +For compatibility with existing publishing and installation processes, +the presence of license classifiers SHOULD NOT raise an error unless +``License-Expression`` is also provided. + +For all newly-uploaded distributions that include a +``License-Expression`` field, the `Python Package Index (PyPI) `__ MUST +reject any that also specify any license classifiers. + +New license classifiers MUST NOT be `added to PyPI `__; +users needing them SHOULD use the ``License-Expression`` field instead. +Along with the ``License`` field, license classifiers may be removed from a +new version of the specification in a future PEP. + + +.. _639-spec-source-metadata: + +Project source metadata +----------------------- + +As originally introduced in :pep:`621`, the +`PyPA Declaring Project Metadata specification `__ +defines how to declare a project's source +metadata under a ``[project]`` table in the ``pyproject.toml`` file for +build tools to consume and output distribution core metadata. + +This PEP :ref:`adds <639-spec-key-license-expression>` +a top-level string value for the ``license`` key, +:ref:`adds <639-spec-key-license-files>` the new ``license-files`` key +and :ref:`deprecates <639-spec-key-license>` +the table value for the ``license`` key +along with its corresponding table subkeys, ``text`` and ``file``. + + +.. _639-spec-key-license-expression: + +Add string value to ``license`` key +''''''''''''''''''''''''''''''''''' + +A top-level string value is defined +for the ``license`` key in the ``[project]`` table, +which is specified to be a valid SPDX license expression, +as :ref:`defined previously <639-license-expression-definition>`. +Its value maps to the ``License-Expression`` field in the core metadata. + +Build tools SHOULD validate the expression as described in the +:ref:`639-spec-field-license-expression` section, +outputting an error or warning as specified. +When generating the core metadata, tools MUST perform case normalization. + +If a top-level string value for the ``license`` key is present and valid, +for purposes of backward compatibility +tools MAY back-fill the ``License`` core metadata field +with the normalized value of the ``license`` key. + + +.. _639-spec-key-license-files: + +Add ``license-files`` key +''''''''''''''''''''''''' + +A new ``license-files`` key is added to the ``[project]`` table for specifying +paths in the project source tree relative to ``pyproject.toml`` to file(s) +containing licenses and other legal notices to be distributed with the package. +It corresponds to the ``License-File`` fields in the core metadata. + +Its value is a table, which if present MUST contain one of two optional, +mutually exclusive subkeys, ``paths`` and ``globs``; if both are specified, +tools MUST raise an error. Both are arrays of strings; the ``paths`` subkey +contains verbatim file paths, and the ``globs`` subkey valid glob patterns, +which MUST be parsable by the ``glob`` `module `__ in the +Python standard library. + +**Note**: To avoid ambiguity, confusion and (per :pep:`20`, the Zen of Python) +"more than one (obvious) way to do it", allowing a flat array of strings +as the value for the ``license-files`` key has been +:ref:`left out for now <639-license-files-allow-flat-array>`. + +Path delimiters MUST be the forward slash character (``/``), +and parent directory indicators (``..``) MUST NOT be used. +Tools MUST assume that license file content is valid UTF-8 encoded text, +and SHOULD validate this and raise an error if it is not. + +If the ``paths`` subkey is a non-empty array, build tools: + +- MUST treat each value as a verbatim, literal file path, and + MUST NOT treat them as glob patterns. + +- MUST include each listed file in all distribution archives. + +- MUST NOT match any additional license files beyond those explicitly + statically specified by the user under the ``paths`` subkey. + +- MUST list each file path under a ``License-File`` field in the core metadata. + +- MUST raise an error if one or more paths do not correspond to a valid file + in the project source that can be copied into the distribution archive. + +If the ``globs`` subkey is a non-empty array, build tools: + +- MUST treat each value as a glob pattern, and MUST raise an error if the + pattern contains invalid glob syntax. + +- MUST include all files matched by at least one listed pattern in all + distribution archives. + +- MAY exclude files matched by glob patterns that can be unambiguously + determined to be backup, temporary, hidden, OS-generated or VCS-ignored. + +- MUST list each matched file path under a ``License-File`` field in the + core metadata. + +- SHOULD issue a warning and MAY raise an error if no files are matched. + +- MAY issue a warning if any individual user-specified pattern + does not match at least one file. + +If the ``license-files`` key is present, and the ``paths`` or ``globs`` subkey +is set to a value of an empty array, then tools MUST NOT include any +license files and MUST NOT raise an error. + +.. _639-default-patterns: + +If the ``license-files`` key is not present and not explicitly marked as +``dynamic``, tools MUST assume a default value of the following: + +.. code-block:: toml + + license-files.globs = ["LICEN[CS]E*", "COPYING*", "NOTICE*", "AUTHORS*"] + +In this case, tools MAY issue a warning if no license files are matched, +but MUST NOT raise an error. + +If the ``license-files`` key is marked as ``dynamic`` (and not present), +to preserve consistent behavior with current tools and help ensure the packages +they create are legally distributable, build tools SHOULD default to +including at least the license files matching the above patterns, unless the +user has explicitly specified their own. + + +.. _639-spec-key-license: + +Deprecate ``license`` key table subkeys +''''''''''''''''''''''''''''''''''''''' + +Table values for the ``license`` key in the ``[project]`` table, +including the ``text`` and ``file`` table subkeys, are now deprecated. +If the new ``license-files`` key is present, +build tools MUST raise an error if the ``license`` key is defined +and has a value other than a single top-level string. + +If the new ``license-files`` key is not present +and the ``text`` subkey is present in a ``license`` table, +tools SHOULD issue a warning informing users it is deprecated +and recommending a license expression as a top-level string key instead. + +Likewise, if the new ``license-files`` key is not present +and the ``file`` subkey is present in the ``license`` table, +tools SHOULD issue a warning informing users it is deprecated and recommending +the ``license-files`` key instead. + +If the specified license ``file`` is present in the source tree, +build tools SHOULD use it to fill the ``License-File`` field +in the core metadata, and MUST include the specified file +as if it were specified in a ``license-file.paths`` field. +If the file does not exist at the specified path, +tools MUST raise an informative error as previously specified. +However, tools MUST also still assume the +:ref:`specified default value <639-default-patterns>` +for the ``license-files`` key and also include, +in addition to a license file specified under the ``license.file`` subkey, +any license files that match the specified list of patterns. + +Table values for the ``license`` key MAY be removed +from a new version of the specification in a future PEP. + + +.. _639-spec-project-formats: + +License files in project formats +-------------------------------- + +A few minor additions will be made to the relevant existing specifications +to document, standardize and clarify what is already currently supported, +allowed and implemented behavior, as well as explicitly mention the root +license directory the license files are located in and relative to for +each format, per the :ref:`639-spec-field-license-file` section. + +**Project source trees** + As described in the :ref:`639-spec-source-metadata` section, the + `Declaring Project Metadata specification `__ + will be updated to reflect that license file paths MUST be relative to the + project root directory; i.e. the directory containing the ``pyproject.toml`` + (or equivalently, other legacy project configuration, + e.g. ``setup.py``, ``setup.cfg``, etc). + +**Source distributions** *(sdists)* + The `sdist specification `__ will be updated to reflect that for + ``Metadata-Version`` is ``2.4`` or greater, the sdist MUST contain any + license files specified by ``License-File`` in the ``PKG-INFO`` at their + respective paths relative to the top-level directory of the sdist + (containing the ``pyproject.toml`` and the ``PKG-INFO`` core metadata). + +**Built distributions** *(wheels)* + The `wheel specification `__ will be updated to reflect that if + the ``Metadata-Version`` is ``2.4`` or greater and one or more + ``License-File`` fields is specified, the ``.dist-info`` directory MUST + contain a ``licenses`` subdirectory, which MUST contain the files listed + in the ``License-File`` fields in the ``METADATA`` file at their respective + paths relative to the ``licenses`` directory. + +**Installed projects** + The `Recording Installed Projects specification `__ will be + updated to reflect that if the ``Metadata-Version`` is ``2.4`` or greater + and one or more ``License-File`` fields is specified, the ``.dist-info`` + directory MUST contain a ``licenses`` subdirectory which MUST contain + the files listed in the ``License-File`` fields in the ``METADATA`` file + at their respective paths relative to the ``licenses`` directory, + and that any files in this directory MUST be copied from wheels + by install tools. + + +.. _639-spec-converting-metadata: + +Converting legacy metadata +-------------------------- + +Tools MUST NOT use the contents of the ``license.text`` ``[project]`` key +(or equivalent tool-specific format), +license classifiers or the value of the core metadata ``License`` field +to fill the top-level string value of the ``license`` key +or the core metadata ``License-Expression`` field +without informing the user and requiring unambiguous, affirmative user action +to select and confirm the desired license expression value before proceeding. + + +.. _639-spec-mapping-classifiers-identifiers: + +Mapping license classifiers to SPDX identifiers +''''''''''''''''''''''''''''''''''''''''''''''' + +Most single license classifiers (namely, all those not mentioned below) +map to a single valid SPDX license identifier, +allowing tools to infer the SPDX license identifier they correspond to, +both for use when analyzing and auditing packages, +and providing a semi-automated mechanism of filling the ``license`` key +or the ``License-Expression`` field +following the :ref:`specification above <639-spec-converting-metadata>`. + +Some legacy license classifiers intend to specify a particular license, +but do not specify the particular version or variant, leading to a +`critical ambiguity `__ +as to their terms, compatibility and acceptability. +Tools MUST NOT attempt to automatically infer a ``License-Expression`` +when one of these classifiers is used without affirmative user action: + +- ``License :: OSI Approved :: Academic Free License (AFL)`` +- ``License :: OSI Approved :: Apache Software License`` +- ``License :: OSI Approved :: Apple Public Source License`` +- ``License :: OSI Approved :: Artistic License`` +- ``License :: OSI Approved :: BSD License`` +- ``License :: OSI Approved :: GNU Affero General Public License v3`` +- ``License :: OSI Approved :: GNU Free Documentation License (FDL)`` +- ``License :: OSI Approved :: GNU General Public License (GPL)`` +- ``License :: OSI Approved :: GNU General Public License v2 (GPLv2)`` +- ``License :: OSI Approved :: GNU General Public License v3 (GPLv3)`` +- ``License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)`` +- ``License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)`` +- ``License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)`` +- ``License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)`` + +A comprehensive mapping of these classifiers to their possible specific +identifiers was `assembled by Dustin Ingram `__, which tools +MAY use as a reference for the identifier selection options to offer users +when prompting the user to explicitly select the license identifier +they intended for their project. + +.. note:: + + Several additional classifiers, namely the "or later" variants of + the AGPLv3, GPLv2, GPLv3 and LGPLv3, are also listed in the aforementioned + mapping, but unambiguously map to their respective licenses, + and so are not listed here. + However, LGPLv2 is included above, as it could ambiguously + refer to either the distinct v2.0 or v2.1 variants of that license. + +In addition, for the various special cases, the following mappings are +considered canonical and normative for the purposes of this specification: + +- Classifier ``License :: Public Domain`` MAY be mapped to the generic + ``License-Expression: LicenseRef-Public-Domain``. + If tools do so, they SHOULD issue an informational warning encouraging + the use of more explicit and legally portable license identifiers, + such as those for the `CC0 1.0 license `__ (``CC0-1.0``), + the `Unlicense `__ (``Unlicense``), + or the `MIT license `__ (``MIT``), + since the meaning associated with the term "public domain" is thoroughly + dependent on the specific legal jurisdiction involved, + some of which lack the concept entirely. + Alternatively, tools MAY choose to treat these classifiers as ambiguous. + +- The generic and sometimes ambiguous classifiers: + + - ``License :: Free For Educational Use`` + - ``License :: Free For Home Use`` + - ``License :: Free for non-commercial use`` + - ``License :: Freely Distributable`` + - ``License :: Free To Use But Restricted`` + - ``License :: Freeware`` + - ``License :: Other/Proprietary License`` + + MAY be mapped to the generic + ``License-Expression: LicenseRef-Proprietary``, + but tools MUST issue a prominent, informative warning if they do so. + Alternatively, tools MAY choose to treat these classifiers as ambiguous. + +- The generic and ambiguous classifiers ``License :: OSI Approved`` and + ``License :: DFSG approved`` do not map to any license expression, + and thus tools SHOULD treat them as ambiguous, or if not MUST ignore them. + +- The classifiers ``License :: GUST Font License 1.0`` and + ``License :: GUST Font License 2006-09-30`` have no mapping to SPDX license + identifiers, and no PyPI package uses them as of 2022-07-09. + +When multiple license classifiers are used, their relationship is ambiguous, +and it is typically not possible to determine if all the licenses apply or if +there is a choice that is possible among the licenses, +In this case, tools MUST NOT automatically infer a license expression, +unless one license classifier is a parent of the other, +i.e. the child contains all ``::``-delineated components of the parent, +in which case tools MAY ignore the parent classifier +but SHOULD issue an informative warning when doing so. + + +.. _639-backwards-compatibility: + +Backwards Compatibility +======================= + +Adding a new, dedicated ``License-Expression`` core metadata field +and a top-level string value for the ``license`` key reserved for this purpose +in the ``pyproject.toml`` ``[project]`` table +unambiguously signals support for the specification in this PEP. +This avoids the risk of new tooling +misinterpreting a license expression as a free-form license description +or vice versa, and raises an error if and only if the user affirmatively +upgrades to the latest metadata version and adds the new field/key. + +The legacy ``License`` core metadata field +and the ``license`` key table subkeys (``text`` and ``file``) +in the ``pyproject.toml`` ``[project]`` table +will be deprecated along with the license classifiers, +retaining backwards compatibility while gently preparing users for their +future removal. Such a removal would follow a suitable transition period, and +be left to a future PEP and a new version of the core metadata specification. + +Formally specifying the new ``License-File`` core metadata field and the +inclusion of the listed files in the distribution merely codifies and +refines the existing practices in popular packaging tools, including the Wheel +and Setuptools projects, and is designed to be largely backwards-compatible +with their existing use of that field. Likewise, the new ``license-files`` +key in the ``[project]`` table of ``pyproject.toml`` +standardizes statically specifying the files to include, +as well as the default behavior, and allows other tools to make use of them, +while only having an effect once users and tools expressly adopt it. + +Due to requiring license files not be flattened into ``.dist-info`` and +specifying that they should be placed in a dedicated ``licenses`` subdir, +wheels produced following this change will have differently-located +licenses relative to those produced via the previous unspecified, +installer-specific behavior, but as until this PEP there was no way of +discovering these files or accessing them programmatically, and this will +be further discriminated by a new metadata version, there aren't any foreseen +mechanism for this to pose a practical issue. + +Furthermore, this resolves existing compatibility issues with the current +ad hoc behavior, namely license files being silently clobbered if they have +the same names as others at different paths, unknowingly rendering the wheel +undistributable, and conflicting with the names of other metadata files in +the same directory. Formally specifying otherwise would in fact block full +forward compatibility with additional standard or installer-specified files +and directories added to ``.dist-info``, as they too could conflict with +the names of existing licenses. + +While minor additions will be made to the source distribution (sdist), +built distribution (wheel) and installed project specifications, all of these +are merely documenting, clarifying and formally specifying behaviors explicitly +allowed under their current respective specifications, and already implemented +in practice, and gating them behind the explicit presence of both the new +metadata versions and the new fields. In particular, sdists may contain +arbitrary files following the project source tree layout, and formally +mentioning that these must include the license files listed in the metadata +merely documents and codifies existing Setuptools practice. Likewise, arbitrary +installer-specific files are allowed in the ``.dist-info`` directory of wheels +and copied to installed projects, and again this PEP just formally clarifies +and standardizes what is already being done. + +Finally, while this PEP does propose PyPI implement validation of the new +``License-Expression`` and ``License-File`` fields, this has no effect on +existing packages, nor any effect on any new distributions uploaded unless they +explicitly choose to opt in to using these new fields while not +following the requirements in the specification. Therefore, this does not have +a backward compatibility impact, and in fact ensures forward compatibility with +any future changes by ensuring all distributions uploaded to PyPI with the new +fields are valid and conform to the specification. + + +.. _639-security-implications: + +Security Implications +===================== + +This PEP has no foreseen security implications: the ``License-Expression`` +field is a plain string and the ``License-File`` fields are file paths. +Neither introduces any known new security concerns. + + +.. _639-how-to-teach-this: + +How to Teach This +================= + +The simple cases are simple: a single license identifier is a valid license +expression, and a large majority of packages use a single license. + +The plan to teach users of packaging tools how to express their package's +license with a valid license expression is to have tools issue informative +messages when they detect invalid license expressions, or when the deprecated +``License`` field or license classifiers are used. + +An immediate, descriptive error message if an invalid ``License-Expression`` +is used will help users understand they need to use SPDX identifiers in +this field, and catch them if they make a mistake. +For authors still using the now-deprecated, less precise and more redundant +``License`` field or license classifiers, packaging tools will warn +them and inform them of the modern replacement, ``License-Expression``. +Finally, for users who may have forgotten or not be aware they need to do so, +publishing tools will gently guide them toward including ``license`` +and ``license-files`` in their project source metadata. + +Tools may also help with the conversion and suggest a license expression in +many, if not most common cases: + +- The section :ref:`639-spec-mapping-classifiers-identifiers` provides + tool authors with guidelines on how to suggest a license expression produced + from legacy classifiers. + +- Tools may also be able to infer and suggest how to update + an existing ``License`` value in project source metadata + and convert that to a license expression, + as also :ref:`specified in this PEP <639-spec-converting-metadata>`. + For instance, a tool may suggest converting a value of ``MIT`` + in the ``license.text`` key in ``[project]`` + (or the equivalent in tool-specific formats) + to a top-level string value of the ``license`` key (or equivalent). + Likewise, a tool could suggest converting from a ``License`` of ``Apache2`` + (which is not a valid license expression + as :ref:`defined in this PEP <639-spec-field-license-expression>`) + to a ``License-Expression`` of ``Apache-2.0`` + (the equivalent valid license expression using an SPDX license identifier). + + +.. _639-reference-implementation: + +Reference Implementation +======================== + +Tools will need to support parsing and validating license expressions in the +``License-Expression`` field. + +The `license-expression library `__ is a reference Python +implementation that handles license expressions including parsing, +formatting and validation, using flexible lists of license symbols +(including SPDX license IDs and any extra identifiers included here). +It is licensed under Apache-2.0 and is already used in several projects, +including the `SPDX Python Tools `__, +the `ScanCode toolkit `__ +and the Free Software Foundation Europe (FSFE) `REUSE project `__. + + +.. _639-rejected-ideas: + +Rejected Ideas +============== + +Core metadata fields +-------------------- + +Potential alternatives to the structure, content and deprecation of the +core metadata fields specified in this PEP. + + +Re-use the ``License`` field +'''''''''''''''''''''''''''' + +Following `initial discussion `__, earlier versions of this +PEP proposed re-using the existing ``License`` field, which tools would +attempt to parse as a SPDX license expression with a fallback to free text. +Initially, this would merely cause a warning (or even pass silently), +but would eventually be treated as an error by modern tooling. + +This offered the potential benefit of greater backwards-compatibility, +easing the community into using SPDX license expressions while taking advantage +of packages that already have them (either intentionally or coincidentally), +and avoided adding yet another license-related field. + +However, following substantial discussion, consensus was reached that a +dedicated ``License-Expression`` field was the preferred overall approach. +The presence of this field is an unambiguous signal that a package +intends it to be interpreted as a valid SPDX identifier, without the need +for complex and potentially erroneous heuristics, and allows tools to +easily and unambiguously detect invalid content. + +This avoids both false positive (``License`` values that a package author +didn't explicitly intend as an explicit SPDX identifier, but that happen +to validate as one), and false negatives (expressions the author intended +to be valid SPDX, but due to a typo or mistake are not), which are otherwise +not clearly distinguishable from true positives and negatives, an ambiguity +at odds with the goals of this PEP. + +Furthermore, it allows both the existing ``License`` field and +the license classifiers to be more easily deprecated, +with tools able to cleanly distinguish between packages intending to +affirmatively conform to the updated specification in this PEP or not, +and adapt their behavior (warnings, errors, etc) accordingly. +Otherwise, tools would either have to allow duplicative and potentially +conflicting ``License`` fields and classifiers, or warn/error on the +substantial number of existing packages that have SPDX identifiers as the +value for the ``License`` field, intentionally or otherwise (e.g. ``MIT``). + +Finally, it avoids changing the behavior of an existing metadata field, +and avoids tools having to guess the ``Metadata-Version`` and field behavior +based on its value rather than merely its presence. + +While this would mean the subset of existing distributions containing +``License`` fields valid as SPDX license expressions wouldn't automatically be +recognized as such, this only requires appending a few characters to the key +name in the project's source metadata, and this PEP provides extensive +guidance on how this can be done automatically by tooling. + +Given all this, it was decided to proceed with defining a new, +purpose-created field, ``License-Expression``. + + +Re-Use the ``License`` field with a value prefix +'''''''''''''''''''''''''''''''''''''''''''''''' + +As an alternative to the previous, prefixing SPDX license expressions with, +e.g. ``spdx:`` was suggested to reduce the ambiguity inherent in re-using +the ``License`` field. However, this effectively amounted to creating +a field within a field, and doesn't address all the downsides of +keeping the ``License`` field. Namely, it still changes the behavior of an +existing metadata field, requires tools to parse its value +to determine how to handle its content, and makes the specification and +deprecation process more complex and less clean. + +Yet, it still shares a same main potential downside as just creating a new +field: projects currently using valid SPDX identifiers in the ``License`` +field, intentionally or not, won't be automatically recognized, and requires +about the same amount of effort to fix, namely changing a line in the +project's source metadata. Therefore, it was rejected in favor of a new field. + + +Don't make ``License-Expression`` mutually exclusive +'''''''''''''''''''''''''''''''''''''''''''''''''''' + +For backwards compatibility, the ``License`` field and/or the license +classifiers could still be allowed together with the new +``License-Expression`` field, presumably with a warning. However, this +could easily lead to inconsistent, and at the very least duplicative +license metadata in no less than *three* different fields, which is +squarely contrary to the goals of this PEP of making the licensing story +simpler and unambiguous. Therefore, and in concert with clear community +consensus otherwise, this idea was soundly rejected. + + +Don't deprecate existing ``License`` field and classifiers +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +Several community members were initially concerned that deprecating the +existing ``License`` field and classifiers would result in +excessive churn for existing package authors and raise the barrier to +entry for new ones, particularly everyday Python developers seeking to +package and publish their personal projects without necessarily caring +too much about the legal technicalities or being a "license lawyer". +Indeed, every deprecation comes with some non-zero short-term cost, +and should be carefully considered relative to the overall long-term +net benefit. And at the minimum, this change shouldn't make it more +difficult for the average Python developer to share their work under +a license of their choice, and ideally improve the situation. + +Following many rounds of proposals, discussion and refinement, +the general consensus was clearly in favor of deprecating the legacy +means of specifying a license, in favor of "one obvious way to do it", +to improve the currently complex and fragmented story around license +documentation. Not doing so would leave three different un-deprecated ways of +specifying a license for a package, two of them ambiguous, less than +clear/obvious how to use, inconsistently documented and out of date. +This is more complex for all tools in the ecosystem to support +indefinitely (rather than simply installers supporting older packages +implementing previous frozen metadata versions), resulting in a non-trivial +and unbounded maintenance cost. + +Furthermore, it leads to a more complex and confusing landscape for users with +three similar but distinct options to choose from, particularly with older +documentation, answers and articles floating around suggesting different ones. +Of the three, ``License-Expression`` is the simplest and clearest to use +correctly; users just paste in their desired license identifier, or select it +via a tool, and they're done; no need to learn about Trove classifiers and +dig through the list to figure out which one(s) apply (and be confused +by many ambiguous options), or figure out on their own what should go +in the ``license`` key (anything from nothing, to the license text, +to a free-form description, to the same SPDX identifier they would be +entering in the ``license`` key anyway, assuming they can +easily find documentation at all about it). In fact, this can be +made even easier thanks to the new field. For example, GitHub's popular +`ChooseALicense.com `__ links to how to add SPDX license +identifiers to the project source metadata of various languages that support +them right in the sidebar of every license page; the SPDX support in this +PEP enables adding Python to that list. + +For current package maintainers who have specified a ``License`` or license +classifiers, this PEP only recommends warnings and prohibits errors for +all but publishing tools, which are allowed to error if their intended +distribution platform(s) so requires. Once maintainers are ready to +upgrade, for those already using SPDX license expressions (accidentally or not) +this only requires appending a few characters to the key name in the +project's source metadata, and for those with license classifiers that +map to a single unambiguous license, or another defined case (public domain, +proprietary), they merely need to drop the classifier and paste in the +corresponding license identifier. This PEP provides extensive guidance and +examples, as will other resources, as well as explicit instructions for +automated tooling to take care of this with no human changes needed. +More complex cases where license metadata is currently specified may +need a bit of human intervention, but in most cases tools will be able +to provide a list of options following the mappings in this PEP, and +these are typically the projects most likely to be constrained by the +limitations of the existing license metadata, and thus most benefited +by the new fields in this PEP. + +Finally, for unmaintained packages, those using tools supporting older +metadata versions, or those who choose not to provide license metadata, +no changes are required regardless of the deprecation. + + +Don't mandate validating new fields on PyPI +''''''''''''''''''''''''''''''''''''''''''' + +Previously, while this PEP did include normative guidelines for packaging +publishing tools (such as Twine), it did not provide specific guidance +for PyPI (or other package indices) as to whether and how they +should validate the ``License-Expression`` or ``License-File`` fields, +nor how they should handle using them in combination with the deprecated +``License`` field or license classifiers. This simplifies the specification +and either defers implementation on PyPI to a later PEP, or gives +discretion to PyPI to enforce the stated invariants, to minimize +disruption to package authors. + +However, this had been left unstated from before the ``License-Expression`` +field was separate from the existing ``License``, which would make +validation much more challenging and backwards-incompatible, breaking +existing packages. With that change, there was a clear consensus that +the new field should be validated from the start, guaranteeing that all +distributions uploaded to PyPI that declare core metadata version 2.4 +or higher and have the ``License-Expression`` field will have a valid +expression, such that PyPI and consumers of its packages and metadata +can rely upon to follow the specification here. + +The same can be extended to the new ``License-File`` field as well, +to ensure that it is valid and the legally required license files are +present, and thus it is lawful for PyPI, users and downstream consumers +to distribute the package. (Of course, this makes no *guarantee* of such +as it is ultimately reliant on authors to declare them, but it improves +assurance of this and allows doing so in the future if the community so +decides.) To be clear, this would not require that any uploaded distribution +have such metadata, only that if they choose to declare it per the new +specification in this PEP, it is assured to be valid. + + +Source metadata ``license`` key +------------------------------- + +Alternate possibilities related to the ``license`` key in the +``pyproject.toml`` project source metadata. + + +Add ``expression`` and ``files`` subkeys to table +''''''''''''''''''''''''''''''''''''''''''''''''' + +A previous working draft of this PEP added ``expression`` and ``files`` subkeys +to the existing ``license`` table in the project source metadata, to parallel +the existing ``file`` and ``text`` subkeys. While this seemed perhaps the +most obvious approach at first glance, it had several serious drawbacks +relative to that ultimately taken here. + +Most saliently, this means two very different types of metadata are being +specified under the same top-level key that require very different handling, +and furthermore, unlike the previous arrangement, the subkeys were not mutually +exclusive and can both be specified at once, and with some subkeys potentially +being dynamic and others static, and mapping to different core metadata fields. + +Furthermore, this leads to a conflict with marking the key as ``dynamic`` +(assuming that is intended to specify the ``[project]`` table keys, +as that PEP seems to imprecisely imply, +rather than core metadata fields), as either or both would have +to be treated as ``dynamic``. +Grouping both license expressions and license files under the same key +forces an "all or nothing" approach, and creates ambiguity as to user intent. + +There are further downsides to this as well. Both users and tools would need to +keep track of which fields are mutually exclusive with which of the others, +greatly increasing cognitive and code complexity, and in turn the probability +of errors. Conceptually, juxtaposing so many different fields under the +same key is rather jarring, and leads to a much more complex mapping between +``[project]`` keys and core metadata fields, not in keeping with :pep:`621`. +This causes the ``[project]`` table naming and structure to diverge further +from both the core metadata and native formats of the various popular packaging +tools that use it. Finally, this results in the spec being significantly more +complex and convoluted to understand and implement than the alternatives. + +The approach this PEP now takes, using the reserved top-level string value +of the ``license`` key, adding a new ``license-files`` key +and deprecating the ``license`` table subkeys (``text`` and ``file``), +avoids most of the issues identified above, +and results in a much clearer and cleaner design overall. +It allows ``license`` and ``license-files`` to be tagged +``dynamic`` independently, separates two independent types of metadata +(syntactically and semantically), restores a closer to 1:1 mapping of +``[project]`` table keys to core metadata fields, +and reduces nesting by a level for both. +Other than adding one extra key to the file, there was no significant +apparent downside to this latter approach, so it was adopted for this PEP. + + +Add an ``expression`` subkey instead of a string value +'''''''''''''''''''''''''''''''''''''''''''''''''''''' + +Adding just an ``expression`` subkey to the ``license`` table, +instead of using the reserved top-level string value, +would be more explicit for readers and writers, +in line with this PEP's goals. +However, it still has the downsides listed above +that are not specific to the inclusion of the ``files`` key. + +Relative to a flat string value, +it adds verbosity, complexity and an extra level of nesting, +and requires users and tools to remember and handle +the mutual exclusivity of the subkeys +and remember which are deprecated and which are not, +instead of cleanly deprecating the table subkeys as a whole. +Furthermore, it is less clearly the "default" choice for modern use, +given users tend to gravitate toward the simplest and most obvious option. +Finally, it seems reasonable to follow the suggested guidance in :pep:`621`, +given the top-level string value was specifically reserved for this purpose. + + +Define a new top-level ``license-expression`` key +''''''''''''''''''''''''''''''''''''''''''''''''' + +An earlier version of this PEP defined a new, top-level ``license-expression`` +under the ``[project]`` table, +rather than using the reserved string value of the ``license`` key. +This was seen as clearer and more explicit for readers and writers, +in line with the goals of this PEP. + +Additionally, while differences from existing tool formats (and core metadata +field names) have precedent in :pep:`621`, +using a key with an identical name as in most/all current tools +to mean something different (and map to a different core metadata field), +with distinct and incompatible syntax and semantics, does not, +and could cause confusion and ambiguity for readers and authors. + +Also, per the `project source metadata spec `__, +this would allow separately marking the ``[project]`` keys +corresponding to the ``License`` and ``License-Expression`` metadata fields +as ``dynamic``, +avoiding a potential concern with back-filling the ``License`` field +from the ``License-Expression`` field as this PEP currently allows +without it as ``license`` as dynamic +(which would not be possible, since they both map to the same top-level key). + +However, community consensus favored using +the top-level string value of the existing ``license`` key, +as :pep:`reserved for this purpose by PEP 621 <621#license>`: + + A practical string value for the license key has been purposefully left + out to allow for a future PEP to specify support for SPDX expressions + (the same logic applies to any sort of "type" field specifying what + license the file or text represents). + +This is shorter and simpler for users to remember and type, +avoids adding a new top-level key while taking advantage of an existing one, +guides users toward using a license expression as the default, +and follows what was envisioned in the original :pep:`621`. + +Additionally, this allows cleanly deprecating the table values +without deprecating the key itself, +and makes them inherently mutually exclusive without users having to remember +and tools having to enforce it. + +Finally, consistency with other tool formats and the underlying core metadata +was not considered a sufficient priority +to override the advantages of using the existing key, +and the ``dynamic`` concerns were mostly mitigated by +not specifying legacy license to license expression conversion at build time, +explicitly specifying backfilling the ``License`` field when not ``dynamic``, +and the fact that both fields are mutually exclusive, +so there is little practical need to distinguish which is dynamic. + +Therefore, a top-level string value for ``license`` was adopted for this PEP, +as an earlier working draft had temporarily specified. + + +Add a ``type`` key to treat ``text`` as expression +'''''''''''''''''''''''''''''''''''''''''''''''''' + +Instead of using the reserved top-level string value +of the ``license`` key in the ``[project]`` table, +one could add a ``type`` subkey to the ``license`` table +to control whether ``text`` (or a string value) +is interpreted as free-text or a license expression. This could make +backward compatibility a little more seamless, as older tools could ignore +it and always treat ``text`` as ``license``, while newer tools would +know to treat it as a license expression, if ``type`` was set appropriately. +Indeed, :pep:`621` seems to suggest something of this sort as a possible +alternative way that SPDX license expressions could be implemented. + +However, all the same downsides as in the previous item apply here, +including greater complexity, a more complex mapping between the project +source metadata and core metadata and inconsistency between the presentation +in tool config, project source metadata and core metadata, +a much less clean deprecation, further bikeshedding over what to name it, +and inability to mark one but not the other as dynamic, among others. + +In addition, while theoretically potentially a little easier in the short +term, in the long term it would mean users would always have to remember +to specify the correct ``type`` to ensure their license expression is +interpreted correctly, which adds work and potential for error; we could +never safety change the default while being confident that users +understand that what they are entering is unambiguously a license expression, +with all the false positive and false negative issues as above. + +Therefore, for these as well as the same reasons this approach was rejected +for the core metadata in favor of a distinct ``License-Expression`` field, +we similarly reject this here in favor of +the reserved string value of the ``license`` key. + + +Must be marked dynamic to back-fill +''''''''''''''''''''''''''''''''''' + +The ``license`` key in the ``pyproject.toml`` could be required to be +explicitly set to dynamic in order for the ``License`` core metadata field +to be automatically back-filled from +the top-level string value of the ``license`` key. +This would be more explicit that the filling will be done, +as strictly speaking the ``license`` key is not (and cannot be) specified in +``pyproject.toml``, and satisfies a stricter interpretation of the letter +of the previous :pep:`621` specification that this PEP revises. + +However, this doesn't seem to be necessary, because it is simply using the +static, verbatim literal value of the ``license`` key, as specified +strictly in this PEP. Therefore, any conforming tool can trivially, +deterministically and unambiguously derive this using only the static data +in the ``pyproject.toml`` file itself. + +Furthermore, this actually adds significant ambiguity, as it means the value +could get filled arbitrarily by other tools, which would in turn compromise +and conflict with the value of the new ``License-Expression`` field, which is +why such is explicitly prohibited by this PEP. Therefore, not marking it as +``dynamic`` will ensure it is only handled in accordance with this PEP's +requirements. + +Finally, users explicitly being told to mark it as ``dynamic``, or not, to +control filling behavior seems to be a bit of a mis-use of the ``dynamic`` +field as apparently intended, and prevents tools from adapting to best +practices (fill, don't fill, etc) as they develop and evolve over time. + + +Source metadata ``license-files`` key +------------------------------------- + +Alternatives considered for the ``license-files`` key in the +``pyproject.toml`` ``[project]`` table, primarily related to the +path/glob type handling. + + +Add a ``type`` subkey to ``license-files`` +'''''''''''''''''''''''''''''''''''''''''' + +Instead of defining mutually exclusive ``paths`` and ``globs`` subkeys +of the ``license-files`` ``[project]`` table key, we could +achieve the same effect with a ``files`` subkey for the list and +a ``type`` subkey for how to interpret it. However, the latter offers no +real advantage over the former, in exchange for requiring more keystrokes, +verbosity and complexity, as well as less flexibility in allowing both, +or another additional subkey in the future, as well as the need to bikeshed +over the subkey name. Therefore, it was summarily rejected. + + +Only accept verbatim paths +'''''''''''''''''''''''''' + +Globs could be disallowed completely as values to the ``license-files`` +key in ``pyproject.toml`` and only verbatim literal paths allowed. +This would ensure that all license files are explicitly specified, all +specified license files are found and included, and the source metadata +is completely static in the strictest sense of the term, without tools +having to inspect the rest of the project source files to determine exactly +what license files will be included and what the ``License-File`` values +will be. This would also modestly simplify the spec and tool implementation. + +However, practicality once again beats purity here. Globs are supported and +used by many existing tools for finding license files, and explicitly +specifying the full path to every license file would be unnecessarily tedious +for more complex projects with vendored code and dependencies. More +critically, it would make it much easier to accidentally miss a required +legal file, silently rendering the package illegal to distribute. + +Tools can still statically and consistently determine the files to be included, +based only on those glob patterns the user explicitly specified and the +filenames in the package, without installing it, executing its code or even +examining its files. Furthermore, tools are still explicitly allowed to warn +if specified glob patterns (including full paths) don't match any files. +And, of course, sdists, wheels and others will have the full static list +of files specified in their distribution metadata. + +Perhaps most importantly, this would also preclude the currently specified +default value, as widely used by the current most popular tools, and thus +be a major break to backward compatibility, tool consistency, and safe +and sane default functionality to avoid unintentional license violations. +And of course, authors are welcome and encouraged to specify their license +files explicitly via the ``paths`` table subkey, once they are aware of it and +if it is suitable for their project and workflow. + + +Only accept glob patterns +''''''''''''''''''''''''' + +Conversely, all ``license-files`` strings could be treated as glob patterns. +This would slightly simplify the spec and implementation, avoid an extra level +of nesting, and more closely match the configuration format of existing tools. + +However, for the cost of a few characters, it ensures users are aware +whether they are entering globs or verbatim paths. Furthermore, allowing +license files to be specified as literal paths avoids edge cases, such as those +containing glob characters (or those confusingly or even maliciously similar +to them, as described in :pep:`672`). + +Including an explicit ``paths`` value ensures that the resulting +``License-File`` metadata is correct, complete and purely static in the +strictest sense of the term, with all license paths explicitly specified +in the ``pyproject.toml`` file, guaranteed to be included and with an early +error should any be missing. This is not practical to do, at least without +serious limitations for many workflows, if we must assume the items +are glob patterns rather than literal paths. + +This allows tools to locate them and know the exact values of the +``License-File`` core metadata fields without having to traverse the +source tree of the project and match globs, potentially allowing easier, +more efficient and reliable programmatic inspection and processing. + +Therefore, given the relatively small cost and the significant benefits, +this approach was not adopted. + + +Infer whether paths or globs +'''''''''''''''''''''''''''' + +It was considered whether to simply allow specifying an array of strings +directly for the ``license-files`` key, rather than making it a table with +explicit ``paths`` and ``globs``. This would be somewhat simpler and avoid +an extra level of nesting, and more closely match the configuration format +of existing tools. However, it was ultimately rejected in favor of separate, +mutually exclusive ``paths`` and ``globs`` table subkeys. + +In practice, it only saves six extra characters in the ``pyproject.toml`` +(``license-files = [...]`` vs ``license-files.globs = [...]``), but allows +the user to more explicitly declare their intent, ensures they understand how +the values are going to be interpreted, and serves as an unambiguous indicator +for tools to parse them as globs rather than verbatim path literals. + +This, in turn, allows for more appropriate, clearly specified tool +behaviors for each case, many of which would be unreliable or impossible +without it, to avoid common traps, provide more helpful feedback and +behave more sensibly and intuitively overall. These include, with ``paths``, +guaranteeing that each and every specified file is included and immediately +raising an error if one is missing, and with ``globs``, checking glob syntax, +excluding unwanted backup, temporary, or other such files (as current tools +already do), and optionally warning if a glob doesn't match any files. +This also avoids edge cases (e.g. paths that contain glob characters) and +reliance on heuristics to determine interpretation—the very thing this PEP +seeks to avoid. + + +.. _639-license-files-allow-flat-array: + +Also allow a flat array value +''''''''''''''''''''''''''''' + +Initially, after deciding to define ``license-files`` as a table of ``paths`` +and ``globs``, thought was given to making a top-level string array under the +``license-files`` key mean one or the other (probably ``globs``, to match most +current tools). This is slightly shorter and simpler, would allow gently +nudging users toward a preferred one, and allow a slightly cleaner handling of +the empty case (which, at present, is treated identically for either). + +However, this again only saves six characters in the best case, and there +isn't an obvious choice; whether from a perspective of preference (both had +clear use cases and benefits), nor as to which one users would naturally +assume. + +Flat may be better than nested, but in the face of ambiguity, users +may not resist the temptation to guess. Requiring users to explicitly specify +one or the other ensures they are aware of how their inputs will be handled, +and is more readable for others, both human and machine alike. It also makes +the spec and tool implementation slightly more complicated, and it can always +be added in the future, but not removed without breaking backward +compatibility. And finally, for the "preferred" option, it means there is +more than one obvious way to do it. + +Therefore, per :pep:`20`, the Zen of Python, this approach is hereby rejected. + + +Allow both ``paths`` and ``globs`` subkeys +'''''''''''''''''''''''''''''''''''''''''' + +Allowing both ``paths`` and ``globs`` subkeys to be specified under the +``license-files`` table was considered, as it could potentially allow +more flexible handling for particularly complex projects, and specify on a +per-pattern rather than overall basis whether ``license-files`` entries +should be treated as ``paths`` or ``globs``. + +However, given the existing proposed approach already matches or exceeds the +power and capabilities of those offered in tools' config files, there isn't +clear demand for this and few likely cases that would benefit, it adds a large +amount of complexity for relatively minimal gain, in terms of the +specification, in tool implementations and in ``pyproject.toml`` itself. + +There would be many more edge cases to deal with, such as how to handle files +matched by both lists, and it conflicts in multiple places with the current +specification for how tools should behave with one or the other, such as when +no files match, guarantees of all files being included and of the file paths +being explicitly, statically specified, and others. + +Like the previous, if there is a clear need for it, it can be always allowed +in the future in a backward-compatible manner (to the extent it is possible +in the first place), while the same is not true of disallowing it. +Therefore, it was decided to require the two subkeys to be mutually exclusive. + + +Rename ``paths`` subkey to ``files`` +'''''''''''''''''''''''''''''''''''' + +Initially, it was considered whether to name the ``paths`` subkey of the +``license-files`` table ``files`` instead. However, ``paths`` was ultimately +chosen, as calling the table subkey ``files`` resulted in duplication between +the table name (``license-files``) and the subkey name (``files``), i.e. +``license-files.files = ["LICENSE.txt"]``, made it seem like the preferred/ +default subkey when it was not, and lacked the same parallelism with ``globs`` +in describing the format of the string entry rather than what was being +pointed to. + + +Must be marked dynamic to use defaults +'''''''''''''''''''''''''''''''''''''' + +It may seem outwardly sensible, at least with a particularly restrictive +interpretation of :pep:`621`'s description of the ``dynamic`` list, to +consider requiring the ``license-files`` key to be explicitly marked as +``dynamic`` in order for the default glob patterns to be used, or alternatively +for license files to be matched and included at all. + +However, this is merely declaring a static, strictly-specified default value +for this particular key, required to be used exactly by all conforming tools +(so long as it is not marked ``dynamic``, negating this argument entirely), +and is no less static than any other set of glob patterns the user themself +may specify. Furthermore, the resulting ``License-File`` core metadata values +can still be determined with only a list of files in the source, without +installing or executing any of the code, or even inspecting file contents. + +Moreover, even if this were not so, practicality would trump purity, as this +interpretation would be strictly backwards-incompatible with the existing +format, and be inconsistent with the behavior with the existing tools. +Further, this would create a very serious and likely risk of a large number of +projects unknowingly no longer including legally mandatory license files, +making their distribution technically illegal, and is thus not a sane, +much less sensible default. + +Finally, aside from adding an additional line of default-required boilerplate +to the file, not defining the default as dynamic allows authors to clearly +and unambiguously indicate when their build/packaging tools are going to be +handling the inclusion of license files themselves rather than strictly +conforming to the project source metadata portions of this PEP; +to do otherwise would defeat the primary purpose of the ``dynamic`` list +as a marker and escape hatch. + + +License file paths +------------------ + +Alternatives related to the paths and locations of license files in the source +and built distributions. + + +Flatten license files in subdirectories +''''''''''''''''''''''''''''''''''''''' + +Previous drafts of this PEP were silent on the issue of handling license files +in subdirectories. Currently, the `Wheel `__ and (following its +example) `Setuptools `__ projects flatten all license files +into the ``.dist-info`` directory without preserving the source subdirectory +hierarchy. + +While this is the simplest approach and matches existing ad hoc practice, +this can result in name conflicts and license files clobbering others, +with no obvious defined behavior for how to resolve them, and leaving the +package legally un-distributable without any clear indication to users that +their specified license files have not been included. + +Furthermore, this leads to inconsistent relative file paths for non-root +license files between the source, sdist and wheel, and prevents the paths +given in the "static" ``[project]`` table metadata from being truly static, +as they need to be flattened, and may potentially overwrite one another. +Finally, the source directory structure often implies valuable information +about what the licenses apply to, and where to find them in the source, +which is lost when flattening them and far from trivial to reconstruct. + +To resolve this, the PEP now proposes, as did contributors on both of the +above issues, reproducing the source directory structure of the original +license files inside the ``.dist-info`` directory. This would fully resolve the +concerns above, with the only downside being a more nested ``.dist-info`` +directory. There is still a risk of collision with edge-case custom +filenames (e.g. ``RECORD``, ``METADATA``), but that is also the case +with the previous approach, and in fact with fewer files flattened +into the root, this would actually reduce the risk. Furthermore, +the following proposal rooting the license files under a ``licenses`` +subdirectory eliminates both collisions and the clutter problem entirely. + + +Resolve name conflicts differently +'''''''''''''''''''''''''''''''''' + +Rather than preserving the source directory structure for license files +inside the ``.dist-info`` directory, we could specify some other mechanism +for conflict resolution, such as pre- or appending the parent directory name +to the license filename, traversing up the tree until the name was unique, +to avoid excessively nested directories. + +However, this would not address the path consistency issues, would require +much more discussion, coordination and bikeshedding, and further complicate +the specification and the implementations. Therefore, it was rejected in +favor of the simpler and more obvious solution of just preserving the +source subdirectory layout, as many stakeholders have already advocated for. + + +Dump directly in ``.dist-info`` +''''''''''''''''''''''''''''''' + +Previously, the included license files were stored directly in the top-level +``.dist-info`` directory of built wheels and installed projects. This followed +existing ad hoc practice, ensured most existing wheels currently using this +feature will match new ones, and kept the specification simpler, with the +license files always being stored in the same location relative to the core +metadata regardless of distribution type. + +However, this leads to a more cluttered ``.dist-info`` directory, littered +with arbitrary license files and subdirectories, as opposed to separating +licenses into their own namespace (which per the Zen of Python, :pep:`20`, are +"one honking great idea"). While currently small, there is still a +risk of collision with specific custom license filenames +(e.g. ``RECORD``, ``METADATA``) in the ``.dist-info`` directory, which +would only increase if and when additional files were specified here, and +would require carefully limiting the potential filenames used to avoid +likely conflicts with those of license-related files. Finally, +putting licenses into their own specified subdirectory would allow +humans and tools to quickly, easily and correctly list, copy and manipulate +all of them at once (such as in distro packaging, legal checks, etc) +without having to reference each of their paths from the core metadata. + +Therefore, now is a prudent time to specify an alternate approach. +The simplest and most obvious solution, as suggested by several on the Wheel +and Setuptools implementation issues, is to simply root the license files +relative to a ``licenses`` subdirectory of ``.dist-info``. This is simple +to implement and solves all the problems noted here, without clear significant +drawbacks relative to other more complex options. + +It does make the specification a bit more complex and less elegant, but +implementation should remain equally simple. It does mean that wheels +produced with following this change will have differently-located licenses +than those prior, but as this was already true for those in subdirectories, +and until this PEP there was no way of discovering these files or +accessing them programmatically, this doesn't seem likely to pose +significant problems in practice. Given this will be much harder if not +impossible to change later, once the status quo is standardized, tools are +relying on the current behavior and there is much greater uptake of not +only simply including license files but potentially accessing them as well +using the core metadata, if we're going to change it, now would be the time +(particularly since we're already introducing an edge-case change with how +license files in subdirs are handled, along with other refinements). + +Therefore, the latter has been incorporated into current drafts of this PEP. + + +Add new ``licenses`` category to wheel +'''''''''''''''''''''''''''''''''''''' + +Instead of defining a root license directory (``licenses``) inside +the core metadata directory (``.dist-info``) for wheels, we could instead +define a new category (and, presumably, a corresponding install scheme), +similar to the others currently included under ``.data`` in the wheel archive, +specifically for license files, called (e.g.) ``licenses``. This was mentioned +by the wheel creator, and would allow installing licenses somewhere more +platform-appropriate and flexible than just the ``.dist-info`` directory +in the site path, and potentially be conceptually cleaner than including +them there. + +However, at present, this PEP does not implement this idea, and it is +deferred to a future one. It would add significant complexity and friction +to this PEP, being primarily concerned with standardizing existing practice +and updating the core metadata specification. Furthermore, doing so would +likely require modifying ``sysconfig`` and the install schemes specified +therein, alongside Wheel, Installer and other tools, which would be a +non-trivial undertaking. While potentially slightly more complex for +repackagers (such as those for Linux distributions), the current proposal still +ensures all license files are included, and in a single dedicated directory +(which can easily be copied or relocated downstream), and thus should still +greatly improve the status quo in this regard without the attendant complexity. + +In addition, this approach is not fully backwards compatible (since it +isn't transparent to tools that simply extract the wheel), is a greater +departure from existing practice and would lead to more inconsistent +license install locations from wheels of different versions. Finally, +this would mean licenses would not be installed as proximately to their +associated code, there would be more variability in the license root path +across platforms and between built distributions and installed projects, +accessing installed licenses programmatically would be more difficult, and a +suitable install location and method would need to be created, discussed +and decided that would avoid name clashes. + +Therefore, to keep this PEP in scope, the current approach was retained. + + +Name the subdirectory ``license_files`` +''''''''''''''''''''''''''''''''''''''' + +Both ``licenses`` and ``license_files`` have been suggested as potential +names for the root license directory inside ``.dist-info`` of wheels and +installed projects. An initial draft of the PEP specified the former +due to being slightly clearer and consistent with the +name of the core metadata field (``License-File``) +and the ``[project]`` table key (``license-files``). +However, the current version of the PEP adopts the ``license`` name, +due to a general preference by the community for its shorter length, +greater simplicity and the lack of a separator character (``_``, ``-``, etc.). + + +Other ideas +----------- + +Miscellaneous proposals, possibilities and discussion points that were +ultimately not adopted. + + +Map identifiers to license files +'''''''''''''''''''''''''''''''' + +This would require using a mapping (as two parallel lists would be too prone to +alignment errors), which would add extra complexity to how license +are documented and add an additional nesting level. + +A mapping would be needed, as it cannot be guaranteed that all expressions +(keys) have a single license file associated with them (e.g. +GPL with an exception may be in a single file) and that any expression +does not have more than one. (e.g. an Apache license ``LICENSE`` and +its ``NOTICE`` file, for instance, are two distinct files). +For most common cases, a single license expression and one or more license +files would be perfectly adequate. In the rarer and more complex cases where +there are many licenses involved, authors can still safety use the fields +specified here, just with a slight loss of clarity by not specifying which +text file(s) map to which license identifier (though this should be clear in +practice given each license identifier has corresponding SPDX-registered +full license text), while not forcing the more complex data model +(a mapping) on the large majority of users who do not need or want it. + +We could of course have a data field with multiple possible value types (it's a +string, it's a list, it's a mapping!) but this could be a source of confusion. +This is what has been done, for instance, in npm (historically) and in Rubygems +(still today), and as result tools need to test the type of the metadata field +before using it in code, while users are confused about when to use a list or a +string. Therefore, this approach is rejected. + + +Map identifiers to source files +''''''''''''''''''''''''''''''' + +As discussed previously, file-level notices are out of scope for this PEP, +and the existing ``SPDX-License-Identifier`` `convention `__ can +already be used if this is needed without further specification here. + + +Don't freeze compatibility with a specific SPDX version +''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +This PEP could omit specifying a specific SPDX specification version, +or one for the list of valid license identifiers, which would allow +more flexible updates as the specification evolves without another +PEP or equivalent. + +However, serious concerns were expressed about a future SPDX update breaking +compatibility with existing expressions and identifiers, leaving current +packages with invalid metadata per the definition in this PEP. Requiring +compatibility with a specific version of these specifications here +and a PEP or similar process to update it avoids this contingency, +and follows the practice of other packaging ecosystems. + +Therefore, it was `decided `__ to specify a minimum version +and requires tools to be compatible with it, while still allowing updates +so long as they don't break backward compatibility. This enables +tools to immediate take advantage of improvements and accept new +licenses, but also remain backwards compatible with the version +specified here, balancing flexibility and compatibility. + + +.. _639-rejected-ideas-difference-license-source-binary: + +Different licenses for source and binary distributions +'''''''''''''''''''''''''''''''''''''''''''''''''''''' + +As an additional use case, it was asked whether it was in scope for this +PEP to handle cases where the license expression for a binary distribution +(wheel) is different from that for a source distribution (sdist), such +as in cases of non-pure-Python packages that compile and bundle binaries +under different licenses than the project itself. An example cited was +`PyTorch `__, which contains CUDA from Nvidia, which is freely +distributable but not open source. `NumPy `__ and +`SciPy `__ also had similar issues, as reported by the +original author of this PEP and now resolved for those cases. + +However, given the inherent complexity here and a lack of an obvious +mechanism to do so, the fact that each wheel would need its own license +information, lack of support on PyPI for exposing license info on a +per-distribution archive basis, and the relatively niche use case, it was +determined to be out of scope for this PEP, and left to a future PEP +to resolve if sufficient need and interest exists and an appropriate +mechanism can be found. + + +Open Issues +=========== + +Should the ``License`` field be back-filled, or mutually exclusive? +------------------------------------------------------------------- + +At present, this PEP explicitly allows, but does not formally recommend or +require, build tools to back-fill the ``License`` core metadata field with +the verbatim text from the ``License-Expression`` field. This would +presumably improve backwards compatibility and was suggested +by some on the Discourse thread. On the other hand, allowing it does +increase complexity and is less of a clean, consistent separation, +preventing the ``License`` field from being completely mutually exclusive +with the new ``License-Expression`` field and requiring that their values +match. + +As such, it would be very useful to have a more concrete and specific +rationale and use cases for the back-filled data, and give fuller +consideration to any potential benefits or drawbacks of this approach, +in order to come to a final consensus on this matter that can be appropriately +justified here. + +Therefore, is the status quo expressed here acceptable, allowing tools +leeway to decide this for themselves? Should this PEP formally recommend, +or even require, that tools back-fill this metadata (which would presumably +be reversed once a breaking revision of the metadata spec is issued)? +Or should this not be explicitly allowed, discouraged or even prohibited? + + +Should custom license identifiers be allowed? +--------------------------------------------- + +The current version of this PEP retains the behavior of only specifying +the use of SPDX-defined license identifiers, as well as the explicitly defined +custom identifiers ``LicenseRef-Public-Domain`` and ``LicenseRef-Proprietary`` +to handle the two common cases where projects have a license, but it is not +one that has a recognized SPDX license identifier. + +For maximum flexibility, custom ``LicenseRef-`` license +identifiers could be allowed, which could potentially be useful for niche +cases or corporate environments where ``LicenseRef-Proprietary`` is not +appropriate or insufficiently specific, but relying on mainstream Python +build tooling and the ``License-Expression`` metadata field is still +desirable to use for this purpose. + +This has the downsides, however, of not catching misspellings of the +canonically defined license identifiers and thus producing license metadata +that is not a valid match for what the author intended, as well as users +potentially thinking they have to prepend ``LicenseRef`` in front of valid +license identifiers, as there seems to be some previous confusion about. +Furthermore, this encourages the proliferation of bespoke license identifiers, +which obviates the purpose of enabling clear, unambiguous and well +understood license metadata for which this PEP was created. + +Indeed, for niche cases that need specific, proprietary custom licenses, +they could always simply specify ``LicenseRef-Proprietary``, and then +include the actual license files needed to unambiguously identify the license +regardless (if not using SPDX license identifiers) under the ``License-File`` +fields. Requiring standards-conforming tools to allow custom license +identifiers does not seem very useful, since standard tools will not recognize +bespoke ones or know how to treat them. By contrast, bespoke tools, which +would be required in any case to understand and act on custom identifiers, +are explicitly allowed, with good reason (thus the ``SHOULD`` keyword) +to not require that license identifiers conform to those listed here. +Therefore, this specification still allows such use in private corporate +environments or specific ecosystems, while avoiding the disadvantages of +imposing them on all mainstream packaging tools. + +As an alternative, a literal ``LicenseRef-Custom`` identifier could be +defined, which would more explicitly indicate that the license cannot be +expressed with defined identifiers and the license text should be referenced +for details, without carrying the negative and potentially inappropriate +implications of ``LicenseRef-Proprietary``. This would avoid the main +mentioned downsides (misspellings, confusion, license proliferation) of +the approve approach of allowing an arbitrary ``LicenseRef``, while +addressing several of the potential theoretical scenarios cited for it. + +On the other hand, as SPDX aims to (and generally does) encompass all +FSF-recognized "Free" and OSI-approved "Open Source" licenses, +and those sources are kept closely in sync and are now relatively stable, +anything outside those bounds would generally be covered by +``LicenseRef-Proprietary``, thus making ``LicenseRef-Custom`` less specific +in that regard, and somewhat redundant to it. Furthermore, it may mislead +authors of projects with complex/multiple licenses that they should use it +over specifying a license expression. + +At present, the PEP retains the existing approach over either of these, given +the use cases and benefits were judged to be sufficiently marginal based +on the current understanding of the packaging landscape. For both these +proposals, however, if more concrete use cases emerge, this can certainly +be reconsidered, either for this current PEP or a future one (before or +in tandem with actually removing the legacy unstructured ``License`` +metadata field). Not defining this now enables allowing it later +(or still now, with custom packaging tools), without affecting backward +compatibility, while the same is not so if they are allowed now and later +determined to be unnecessary or too problematic in practice. + + +.. _639-examples: + +Appendix: Examples +================== + +.. _639-example-basic: + +Basic example +------------- + +The Setuptools project itself, as of `version 59.1.1 `__, +does not use the ``License`` field in its own project source metadata. +Further, it no longer explicitly specifies ``license_file``/``license_files`` +as it did previously, since Setuptools relies on its own automatic +inclusion of license-related files matching common patterns, +such as the ``LICENSE`` file it uses. + +It includes the following license-related metadata in its ``setup.cfg``: + +.. code-block:: ini + + [metadata] + classifiers = + License :: OSI Approved :: MIT License + +The simplest migration to this PEP would consist of using this instead: + +.. code-block:: ini + + [metadata] + license_expression = MIT + +Or, in the ``[project]`` table of ``pyproject.toml``: + +.. code-block:: toml + + [project] + license = "MIT" + +The output core metadata for the distribution packages would then be: + +.. code-block:: email + + License-Expression: MIT + License-File: LICENSE + +The ``LICENSE`` file would be stored at ``/setuptools-${VERSION}/LICENSE`` +in the sdist and ``/setuptools-${VERSION}.dist-info/licenses/LICENSE`` +in the wheel, and unpacked from there into the site directory (e.g. +``site-packages``) on installation; ``/`` is the root of the respective archive +and ``${VERSION}`` the version of the Setuptools release in the core metadata. + + +.. _639-example-advanced: + +Advanced example +---------------- + +Suppose Setuptools were to include the licenses of the third-party projects +that are vendored in the ``setuptools/_vendor/`` and ``pkg_resources/_vendor`` +directories; specifically: + +.. code-block:: text + + packaging==21.2 + pyparsing==2.2.1 + ordered-set==3.1.1 + more_itertools==8.8.0 + +The license expressions for these projects are: + +.. code-block:: text + + packaging: Apache-2.0 OR BSD-2-Clause + pyparsing: MIT + ordered-set: MIT + more_itertools: MIT + +A comprehensive license expression covering both Setuptools +proper and its vendored dependencies would contain these metadata, +combining all the license expressions into one. Such an expression might be: + +.. code-block:: text + + MIT AND (Apache-2.0 OR BSD-2-Clause) + +In addition, per the requirements of the licenses, the relevant license files +must be included in the package. Suppose the ``LICENSE`` file contains the text +of the MIT license and the copyrights used by Setuptools, ``pyparsing``, +``more_itertools`` and ``ordered-set``; and the ``LICENSE*`` files in the +``setuptools/_vendor/packaging/`` directory contain the Apache 2.0 and +2-clause BSD license text, and the Packaging copyright statement and +`license choice notice `__. + +Specifically, we assume the license files are located at the following +paths in the project source tree (relative to the project root and +``pyproject.toml``): + +.. code-block:: ini + + LICENSE + setuptools/_vendor/packaging/LICENSE + setuptools/_vendor/packaging/LICENSE.APACHE + setuptools/_vendor/packaging/LICENSE.BSD + +Putting it all together, our ``setup.cfg`` would be: + +.. code-block:: ini + + [metadata] + license_expression = MIT AND (Apache-2.0 OR BSD-2-Clause) + license_files = + LICENSE + setuptools/_vendor/packaging/LICENSE + setuptools/_vendor/packaging/LICENSE.APACHE + setuptools/_vendor/packaging/LICENSE.BSD + +In the ``[project]`` table of ``pyproject.toml``, with license files +specified explicitly via the ``paths`` subkey, this would look like: + +.. code-block:: toml + + [project] + license = "MIT AND (Apache-2.0 OR BSD-2-Clause)" + license-files.paths = [ + "LICENSE", + "setuptools/_vendor/LICENSE", + "setuptools/_vendor/LICENSE.APACHE", + "setuptools/_vendor/LICENSE.BSD", + ] + +Or alternatively, matched via glob patterns, this could be: + +.. code-block:: toml + + [project] + license = "MIT AND (Apache-2.0 OR BSD-2-Clause)" + license-files.globs = [ + "LICENSE*", + "setuptools/_vendor/LICENSE*", + ] + +With either approach, the output core metadata in the distribution +would be: + +.. code-block:: email + + License-Expression: MIT AND (Apache-2.0 OR BSD-2-Clause) + License-File: LICENSE + License-File: setuptools/_vendor/packaging/LICENSE + License-File: setuptools/_vendor/packaging/LICENSE.APACHE + License-File: setuptools/_vendor/packaging/LICENSE.BSD + +In the resulting sdist, with ``/`` as the root of the archive and ``${VERSION}`` +the version of the Setuptools release specified in the core metadata, +the license files would be located at the paths: + +.. code-block:: shell + + /setuptools-${VERSION}/LICENSE + /setuptools-${VERSION}/setuptools/_vendor/packaging/LICENSE + /setuptools-${VERSION}/setuptools/_vendor/packaging/LICENSE.APACHE + /setuptools-${VERSION}/setuptools/_vendor/packaging/LICENSE.BSD + +In the built wheel, with ``/`` being the root of the archive and +``{version}`` as the previous, the license files would be stored at: + +.. code-block:: shell + + /setuptools-${VERSION}.dist-info/licenses/LICENSE + /setuptools-${VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE + /setuptools-${VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.APACHE + /setuptools-${VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.BSD + +Finally, in the installed project, with ``site-packages`` being the site dir +and ``{version}`` as the previous, the license files would be installed to: + +.. code-block:: shell + + site-packages/setuptools-${VERSION}.dist-info/licenses/LICENSE + site-packages/setuptools-${VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE + site-packages/setuptools-${VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.APACHE + site-packages/setuptools-${VERSION}.dist-info/licenses/setuptools/_vendor/packaging/LICENSE.BSD + + +.. _639-example-expression: + +Expression examples +------------------- + +Some additional examples of valid ``License-Expression`` values: + +.. code-block:: email + + License-Expression: MIT + License-Expression: BSD-3-Clause + License-Expression: MIT AND (Apache-2.0 OR BSD-2-clause) + License-Expression: MIT OR GPL-2.0-or-later OR (FSFUL AND BSD-2-Clause) + License-Expression: GPL-3.0-only WITH Classpath-Exception-2.0 OR BSD-3-Clause + License-Expression: LicenseRef-Public-Domain OR CC0-1.0 OR Unlicense + License-Expression: LicenseRef-Proprietary + + +.. _639-user-scenarios: + +Appendix: User Scenarios +======================== + +The following covers the range of common use cases from a user perspective, +providing straightforward guidance for each. Do note that the following +should **not** be considered legal advice, and readers should consult a +licensed legal practitioner in their jurisdiction if they are unsure about +the specifics for their situation. + + +I have a private package that won't be distributed +-------------------------------------------------- + +If your package isn't shared publicly, i.e. outside your company, +organization or household, it *usually* isn't strictly necessary to include +a formal license, so you wouldn't necessarily have to do anything extra here. + +However, it is still a good idea to include ``LicenseRef-Proprietary`` +as a license expression in your package configuration, and/or a +copyright statement and any legal notices in a ``LICENSE.txt`` file +in the root of your project directory, which will be automatically +included by packaging tools. + + +I just want to share my own work without legal restrictions +----------------------------------------------------------- + +While you aren't required to include a license, if you don't, no one has +`any permission to download, use or improve your work `__, +so that's probably the *opposite* of what you actually want. +The `MIT license `__ is a great choice instead, as it's simple, +widely used and allows anyone to do whatever they want with your work +(other than sue you, which you probably also don't want). + +To apply it, just paste `the text `__ into a file named +``LICENSE.txt`` at the root of your repo, and add the year and your name to +the copyright line. Then, just add ``license = "MIT"`` under +``[project]`` in your ``pyproject.toml`` if your packaging tool supports it, +or in its config file/section (e.g. Setuptools ``license_expression = MIT`` +under ``[metadata]`` in ``setup.cfg``). You're done! + + +I want to distribute my project under a specific license +-------------------------------------------------------- + +To use a particular license, simply paste its text into a ``LICENSE.txt`` +file at the root of your repo, if you don't have it in a file starting with +``LICENSE`` or ``COPYING`` already, and add +``license = "LICENSE-ID"`` under ``[project]`` in your +``pyproject.toml`` if your packaging tool supports it, or else in its +config file (e.g. for Setuptools, ``license_expression = LICENSE-ID`` +under ``[metadata]`` in ``setup.cfg``). You can find the ``LICENSE-ID`` +and copyable license text on sites like +`ChooseALicense `__ or `SPDX `__. + +Many popular code hosts, project templates and packaging tools can add the +license file for you, and may support the expression as well in the future. + + +I maintain an existing package that's already licensed +------------------------------------------------------ + +If you already have license files and metadata in your project, you +should only need to make a couple of tweaks to take advantage of the new +functionality. + +In your project config file, enter your license expression under +``license`` (``[project]`` table in ``pyproject.toml``), +``license_expression`` (Setuptools ``setup.cfg`` / ``setup.py``), +or the equivalent for your packaging tool, +and make sure to remove any legacy ``license`` table subkeys or +``License ::`` classifiers. Your existing ``license`` value may already +be valid as one (e.g. ``MIT``, ``Apache-2.0 OR BSD-2-Clause``, etc); +otherwise, check the `SPDX license list `__ for the identifier +that matches the license used in your project. + +If your license files begin with ``LICENSE``, ``COPYING``, ``NOTICE`` or +``AUTHORS``, or you've already configured your packaging tool to add them +(e.g. ``license_files`` in ``setup.cfg``), you should already be good to go. +If not, make sure to list them under ``license-files.paths`` +or ``license-files.globs`` under ``[project]`` in ``pyproject.toml`` +(if your tool supports it), or else in your tool's configuration file +(e.g. ``license_files`` in ``setup.cfg`` for Setuptools). + +See the :ref:`639-example-basic` for a simple but complete real-world demo +of how this works in practice, including some additional technical details. +Packaging tools may support automatically converting legacy licensing +metadata; check your tool's documentation for more information. + + +My package includes other code under different licenses +------------------------------------------------------- + +If your project includes code from others covered by different licenses, +such as vendored dependencies or files copied from other open source +software, you can construct a license expression (or have a tool +help you do so) to describe the licenses involved and the relationship +between them. + +In short, ``License-1 AND License-2`` mean that *both* licenses apply +to your project, or parts of it (for example, you included a file +under another license), and ``License-1 OR License-2`` means that +*either* of the licenses can be used, at the user's option (for example, +you want to allow users a choice of multiple licenses). You can use +parenthesis (``()``) for grouping to form expressions that cover even the most +complex situations. + +In your project config file, enter your license expression under +``license`` (``[project]`` table of ``pyproject.toml``), +``license_expression`` (Setuptools ``setup.cfg`` / ``setup.py``), +or the equivalent for your packaging tool, +and make sure to remove any legacy ``license`` table subkeys +or ``License ::`` classifiers. + +Also, make sure you add the full license text of all the licenses as files +somewhere in your project repository. If all of them are in the root directory +and begin with ``LICENSE``, ``COPYING``, ``NOTICE`` or ``AUTHORS``, +they will be included automatically. Otherwise, you'll need to list the +relative path or glob patterns to each of them under ``license-files.paths`` +or ``license-files.globs`` under ``[project]`` in ``pyproject.toml`` +(if your tool supports it), or else in your tool's configuration file +(e.g. ``license_files`` in ``setup.cfg`` for Setuptools). + +As an example, if your project was licensed MIT but incorporated +a vendored dependency (say, ``packaging``) that was licensed under +either Apache 2.0 or the 2-clause BSD, your license expression would +be ``MIT AND (Apache-2.0 OR BSD-2-Clause)``. You might have a +``LICENSE.txt`` in your repo root, and a ``LICENSE-APACHE.txt`` and +``LICENSE-BSD.txt`` in the ``_vendor`` subdirectory, so to include +all of them, you'd specify ``["LICENSE.txt", "_vendor/packaging/LICENSE*"]`` +as glob patterns, or +``["LICENSE.txt", "_vendor/LICENSE-APACHE.txt", "_vendor/LICENSE-BSD.txt"]`` +as literal file paths. + +See a fully worked out :ref:`639-example-advanced` for a comprehensive end-to-end +application of this to a real-world complex project, with copious technical +details, and consult a `tutorial `__ for more help and examples +using SPDX identifiers and expressions. + + +.. _639-license-doc-python: + +Appendix: License Documentation in Python +========================================= + +There are multiple ways used or recommended to document Python project +licenses today. The most common are listed below. + + +.. _639-license-doc-core-metadata: + +Core metadata +------------- + +There are two overlapping core metadata fields to document a license: the +license ``Classifier`` `strings `__ prefixed with ``License ::`` +and the ``License`` `field `__ as free text. + +The core metadata ``License`` field documentation is currently: + +.. code-block:: rst + + License + ======= + + .. versionadded:: 1.0 + + Text indicating the license covering the distribution where the license + is not a selection from the "License" Trove classifiers. See + :ref:`"Classifier" ` below. + This field may also be used to specify a + particular version of a license which is named via the ``Classifier`` + field, or to indicate a variation or exception to such a license. + + Examples:: + + License: This software may only be obtained by sending the + author a postcard, and then the user promises not + to redistribute it. + + License: GPL version 3, excluding DRM provisions + +Even though there are two fields, it is at times difficult to convey anything +but simpler licensing. For instance, some classifiers lack precision +(GPL without a version) and when multiple license classifiers are +listed, it is not clear if both licenses must apply, or the user may choose +between them. Furthermore, the list of available license classifiers +is rather limited and out-of-date. + + +.. _639-license-doc-setuptools-wheel: + +Setuptools and Wheel +-------------------- + +Beyond a license code or qualifier, license text files are documented and +included in a built package either implicitly or explicitly, +and this is another possible source of confusion: + +- In the `Setuptools `__ and `Wheel `__ projects, + license files are automatically added to the distribution (at their source + location in a source distribution/sdist, and in the ``.dist-info`` + directory of a built wheel) if they match one of a number of common license + file name patterns (``LICEN[CS]E*``, ``COPYING*``, ``NOTICE*`` and + ``AUTHORS*``). Alternatively, a package author can specify a list of license + file paths to include in the built wheel under the ``license_files`` key in + the ``[metadata]`` section of the project's ``setup.cfg``, or as an argument + to the ``setuptools.setup()`` function. At present, following the Wheel + project's lead, Setuptools flattens the collected license files into the + metadata directory, clobbering files with the same name, and dumps license + files directly into the top-level ``.dist-info`` directory, but there is a + `desire to resolve both these issues `__, + contingent on this PEP being accepted. + +- Both tools also support an older, singular ``license_file`` parameter that + allows specifying only one license file to add to the distribution, which + has been deprecated for some time but still sees `some use `__. + +- Following the publication of an earlier draft of this PEP, Setuptools + `added support `__ for ``License-File`` in distribution + metadata as described in this specification. This allows other tools + consuming the resulting metadata to unambiguously locate the license file(s) + for a given package. + + +.. _639-license-doc-pypug: + +PyPA Packaging Guide and Sample Project +--------------------------------------- + +Both the `PyPA beginner packaging tutorial `__ and its more +comprehensive `packaging guide `__ state that it is +important that every package include a license file. They point to the +``LICENSE.txt`` in the official PyPA sample project as an example, which is +`explicitly listed `__ under the ``license_files`` key in +its ``setup.cfg``, following existing practice formally specified by this PEP. + +Both the `beginner packaging tutorial `__ and the +`sample project `__ only use classifiers to declare a +package's license, and do not include or mention the ``License`` field. +The `full packaging guide `__ does mention this field, but +states that authors should use the license classifiers instead, unless the +project uses a non-standard license (which the guide discourages). + + +.. _639-license-doc-source-files: + +Python source code files +------------------------ + +**Note:** Documenting licenses in source code is not in the scope of this PEP. + +Beside using comments and/or ``SPDX-License-Identifier`` conventions, the +license is `sometimes `__ documented in Python code files using +a "dunder" module-level constant, typically named ``__license__``. + +This convention, while perhaps somewhat antiquated, is recognized by the +built-in ``help()`` function and the standard ``pydoc`` module. +The dunder variable will show up in the ``help()`` DATA section for a module. + + +.. _639-license-doc-other-packaging-tools: + +Other Python packaging tools +---------------------------- + +- `Conda package manifests `__ have support for ``license`` and + ``license_file`` fields, and automatically include license files + following similar naming patterns as the Wheel and Setuptools projects. + +- `Flit `__ recommends using classifiers instead of the ``License`` + field (per the current PyPA packaging guide). + +- `PBR `__ uses similar data as Setuptools, but always stored in + ``setup.cfg``. + +- `Poetry `__ specifies the use of the ``license`` key in + ``pyproject.toml`` with SPDX license identifiers. + + +.. _639-license-doc-other-projects: + +Appendix: License Documentation in Other Projects +================================================= + +Here is a survey of how things are done elsewhere. + + +Linux distribution packages +--------------------------- + +**Note:** in most cases, the texts of the most common licenses are included +globally in a shared documentation directory (e.g. ``/usr/share/doc``). + +- Debian documents package licenses with + `machine readable copyright files `__. + It defines its own license expression syntax and list of identifiers for + common licenses, both of which are closely related to those of SPDX. + +- `Fedora packages `__ specify how to include + `License Texts `__ and use a + `License field `__ that must be filled + with appropriate short license identifier(s) from an extensive list + of `"Good Licenses" `__. Fedora also defines its own + license expression syntax, similar to that of SPDX. + +- `OpenSUSE packages `__ use SPDX license expressions with + SPDX license IDs and a + `list of additional license identifiers `__. + +- `Gentoo ebuild `__ uses a ``LICENSE`` variable. + This field is specified in `GLEP-0023 `__ and in the + `Gentoo development manual `__. + Gentoo also defines a list of allowed licenses and a license expression + syntax, which is rather different from SPDX. + +- The `FreeBSD package Makefile `__ provides ``LICENSE`` and + ``LICENSE_FILE`` fields with a list of custom license symbols. For + non-standard licenses, FreeBSD recommends using ``LICENSE=UNKNOWN`` and + adding ``LICENSE_NAME`` and ``LICENSE_TEXT`` fields, as well as sophisticated + ``LICENSE_PERMS`` to qualify the license permissions and ``LICENSE_GROUPS`` + to document a license grouping. The ``LICENSE_COMB`` allows documenting more + than one license and how they apply together, forming a custom license + expression syntax. FreeBSD also recommends the use of + ``SPDX-License-Identifier`` in source code files. + +- `Arch Linux PKGBUILD `__ defines its + `own license identifiers `__. + The value ``'unknown'`` can be used if the license is not defined. + +- `OpenWRT ipk packages `__ use the ``PKG_LICENSE`` and + ``PKG_LICENSE_FILES`` variables and recommend the use of SPDX License + identifiers. + +- `NixOS uses SPDX identifiers `__ and some extra license IDs + in its license field. + +- GNU Guix (based on NixOS) has a single License field, uses its own + `license symbols list `__ and specifies how to use one license or a + `list of them `__. + +- `Alpine Linux packages `__ recommend using SPDX identifiers in the + license field. + + +Language and application packages +--------------------------------- + +- In Java, `Maven POM `__ defines a ``licenses`` XML tag with a list + of licenses, each with a name, URL, comments and "distribution" type. + This is not mandatory, and the content of each field is not specified. + +- The `JavaScript NPM package.json `__ uses a single license field with + a SPDX license expression, or the ``UNLICENSED`` ID if none is specified. + A license file can be referenced as an alternative using + ``SEE LICENSE IN `` in the single ``license`` field. + +- `Rubygems gemspec `__ specifies either a single or list of license + strings. The relationship between multiple licenses in a + list is not specified. They recommend using SPDX license identifiers. + +- `CPAN Perl modules `__ use a single license field, which is either a + single or a list of strings. The relationship between the licenses in + a list is not specified. There is a list of custom license identifiers plus + these generic identifiers: ``open_source``, ``restricted``, ``unrestricted``, + ``unknown``. + +- `Rust Cargo `__ specifies the use of an SPDX license expression + (v2.1) in the ``license`` field. It also supports an alternative expression + syntax using slash-separated SPDX license identifiers, and there is also a + ``license_file`` field. The `crates.io package registry `__ + requires that either ``license`` or ``license_file`` fields are set when + uploading a package. + +- `PHP composer.json `__ uses a ``license`` field with + an SPDX license ID or ``proprietary``. The ``license`` field is either a + single string with resembling the SPDX license expression syntax with + ``and`` and ``or`` keywords; or is a list of strings if there is a + (disjunctive) choice of licenses. + +- `NuGet packages `__ previously used only a simple license URL, but + now specify using a SPDX license expression and/or the path to a license + file within the package. The NuGet.org repository states that they only + accept license expressions that are "approved by the Open Source Initiative + or the Free Software Foundation." + +- Go language modules ``go.mod`` have no provision for any metadata beyond + dependencies. Licensing information is left for code authors and other + community package managers to document. + +- The `Dart/Flutter spec `__ recommends using a single ``LICENSE`` + file that should contain all the license texts, each separated by a line + with 80 hyphens. + +- The `JavaScript Bower `__ ``license`` field is either a single string + or list of strings using either SPDX license identifiers, or a path/URL + to a license file. + +- The `Cocoapods podspec `__ ``license`` field is either a single + string, or a mapping with ``type``, ``file`` and ``text`` keys. + This is mandatory unless there is a ``LICENSE``/``LICENCE`` file provided. + +- `Haskell Cabal `__ accepts an SPDX license expression since + version 2.2. The version of the SPDX license list used is a function of + the Cabal version. The specification also provides a mapping between + legacy (pre-SPDX) and SPDX license Identifiers. Cabal also specifies a + ``license-file(s)`` field that lists license files to be installed with + the package. + +- `Erlang/Elixir mix/hex package `__ specifies a ``licenses`` field as a + required list of license strings, and recommends using SPDX license + identifiers. + +- `D Langanguage dub packages `__ define their own list of license + identifiers and license expression syntax, similar to the SPDX standard. + +- The `R Package DESCRIPTION `__ defines its own sophisticated license + expression syntax and list of licenses identifiers. R has a unique way of + supporting specifiers for license versions (such as ``LGPL (>= 2.0, < 3)``) + in its license expression syntax. + + +Other ecosystems +---------------- + +- The ``SPDX-License-Identifier`` `header `__ is a simple + convention to document the license inside a file. + +- The `Free Software Foundation (FSF) `__ promotes the use of + SPDX license identifiers for clarity in the `GPL `__ and other + versioned free software licenses. + +- The Free Software Foundation Europe (FSFE) `REUSE project `__ + promotes using ``SPDX-License-Identifier``. + +- The `Linux kernel `__ uses ``SPDX-License-Identifier`` + and parts of the FSFE REUSE conventions to document its licenses. + +- `U-Boot `__ spearheaded using ``SPDX-License-Identifier`` in code + and now follows the Linux approach. + +- The Apache Software Foundation projects use `RDF DOAP `__ with + a single license field pointing to SPDX license identifiers. + +- The `Eclipse Foundation `__ promotes using + ``SPDX-license-Identifiers``. + +- The `ClearlyDefined project `__ promotes using SPDX + license identifiers and expressions to improve license clarity. + +- The `Android Open Source Project `__ uses ``MODULE_LICENSE_XXX`` + empty tag files, where ``XXX`` is a license code such as ``BSD``, ``APACHE``, + ``GPL``, etc. It also uses a ``NOTICE`` file that contains license and + notice texts. + + +References +========== + +.. _alpine: https://wiki.alpinelinux.org/wiki/Creating_an_Alpine_package#license +.. _android: https://github.com/aosp-mirror/platform_external_tcpdump/blob/android-platform-12.0.0_r1/MODULE_LICENSE_BSD +.. _apache: https://svn.apache.org/repos/asf/allura/doap_Allura.rdf +.. _archinux: https://wiki.archlinux.org/title/PKGBUILD#license +.. _archlinuxlist: https://archlinux.org/packages/core/any/licenses/files/ +.. _badclassifiers: https://github.com/pypa/trove-classifiers/issues/17#issuecomment-385027197 +.. _bower: https://github.com/bower/spec/blob/b00c4403e22e3f6177c410ed3391b9259687e461/json.md#license +.. _cabal: https://cabal.readthedocs.io/en/3.6/cabal-package.html?highlight=license#pkg-field-license +.. _cargo: https://doc.rust-lang.org/cargo/reference/manifest.html#package-metadata +.. _cc0: https://creativecommons.org/publicdomain/zero/1.0/ +.. _cdstats: https://clearlydefined.io/stats +.. _choosealicense: https://choosealicense.com/ +.. _choosealicenselist: https://choosealicense.com/licenses/ +.. _chooseamitlicense: https://choosealicense.com/licenses/mit/ +.. _classifierissue: https://github.com/pypa/trove-classifiers/issues/17 +.. _classifiers: https://pypi.org/classifiers +.. _classifiersrepo: https://github.com/pypa/trove-classifiers +.. _clearlydefined: https://clearlydefined.io +.. _cocoapod: https://guides.cocoapods.org/syntax/podspec.html#license +.. _composer: https://getcomposer.org/doc/04-schema.md#license +.. _conda: https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#about-section +.. _coremetadataspec: https://packaging.python.org/specifications/core-metadata +.. _coremetadataclassifiers: https://packaging.python.org/en/latest/specifications/core-metadata/#classifier-multiple-use +.. _cran: https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Licensing +.. _cratesio: https://doc.rust-lang.org/cargo/reference/registries.html#publish +.. _dep5: https://dep-team.pages.debian.net/deps/dep5/ +.. _dontchoosealicense: https://choosealicense.com/no-permission/ +.. _dub: https://dub.pm/package-format-json.html#licenses +.. _eclipse: https://www.eclipse.org/legal/epl-2.0/faq.php +.. _fedora: https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/ +.. _fedoralicense: https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_valid_license_short_names +.. _fedoralist: https://fedoraproject.org/wiki/Licensing:Main?rd=Licensing#Good_Licenses +.. _fedoratext: https://docs.fedoraproject.org/en-US/packaging-guidelines/LicensingGuidelines/#_license_text +.. _flit: https://flit.readthedocs.io/en/stable/pyproject_toml.html +.. _flutter: https://flutter.dev/docs/development/packages-and-plugins/developing-packages#adding-licenses-to-the-license-file +.. _freebsd: https://docs.freebsd.org/en/books/porters-handbook/makefiles/#licenses +.. _fsf: https://www.fsf.org/blogs/rms/rms-article-for-claritys-sake-please-dont-say-licensed-under-gnu-gpl-2 +.. _gem: https://guides.rubygems.org/specification-reference/#license= +.. _gentoo: https://devmanual.gentoo.org/ebuild-writing/variables/index.html#license +.. _gentoodev: https://devmanual.gentoo.org/general-concepts/licenses/index.html +.. _glep23: https://www.gentoo.org/glep/glep-0023.html +.. _globmodule: https://docs.python.org/3/library/glob.html +.. _gnu: https://www.gnu.org/licenses/identify-licenses-clearly.html +.. _guix: https://git.savannah.gnu.org/cgit/guix.git/tree/guix/licenses.scm?h=v1.3.0 +.. _guixlicense: https://guix.gnu.org/manual/en/html_node/package-Reference.html#index-license_002c-of-packages +.. _hatch: https://hatch.pypa.io/latest/ +.. _hatchimplementation: https://discuss.python.org/t/12622/22 +.. _installedspec: https://packaging.python.org/specifications/recording-installed-packages/ +.. _interopissue: https://github.com/pypa/interoperability-peps/issues/46 +.. _licenseexplib: https://github.com/nexB/license-expression/ +.. _licensefield: https://packaging.python.org/guides/distributing-packages-using-setuptools/#license +.. _linux: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/process/license-rules.rst +.. _maven: https://maven.apache.org/pom.html#Licenses +.. _mitlicense: https://opensource.org/licenses/MIT +.. _mix: https://hex.pm/docs/publish +.. _nixos: https://github.com/NixOS/nixpkgs/blob/21.05/lib/licenses.nix +.. _npm: https://docs.npmjs.com/cli/v8/configuring-npm/package-json#license +.. _nuget: https://docs.microsoft.com/en-us/nuget/reference/nuspec#licenseurl +.. _numpyissue: https://github.com/numpy/numpy/issues/8689 +.. _opensuse: https://en.opensuse.org/openSUSE:Packaging_guidelines#Licensing +.. _opensuselist: https://docs.google.com/spreadsheets/d/14AdaJ6cmU0kvQ4ulq9pWpjdZL5tkR03exRSYJmPGdfs/pub +.. _openwrt: https://openwrt.org/docs/guide-developer/packages#buildpackage_variables +.. _osi: https://opensource.org +.. _packagingguidetxt: https://packaging.python.org/guides/distributing-packages-using-setuptools/#license-txt +.. _packagingissue: https://github.com/pypa/packaging-problems/issues/41 +.. _packaginglicense: https://github.com/pypa/packaging/blob/21.2/LICENSE +.. _packagingtutkey: https://packaging.python.org/tutorials/packaging-projects/#configuring-metadata +.. _packagingtuttxt: https://packaging.python.org/tutorials/packaging-projects/#creating-a-license +.. _pbr: https://docs.openstack.org/pbr/latest/user/features.html +.. _pep621spec: https://packaging.python.org/specifications/declaring-project-metadata/ +.. _pep621specdynamic: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#dynamic +.. _pepissue: https://github.com/pombredanne/spdx-pypi-pep/issues/1 +.. _perl: https://metacpan.org/pod/CPAN::Meta::Spec#license +.. _pipsetup: https://github.com/pypa/pip/blob/21.3.1/setup.cfg#L114 +.. _poetry: https://python-poetry.org/docs/pyproject/#license +.. _pycode: https://github.com/search?l=Python&q=%22__license__%22&type=Code +.. _pypi: https://pypi.org/ +.. _pypugdistributionpackage: https://packaging.python.org/en/latest/glossary/#term-Distribution-Package +.. _pypugglossary: https://packaging.python.org/glossary/ +.. _pypugproject: https://packaging.python.org/en/latest/glossary/#term-Project +.. _pytorch: https://pypi.org/project/torch/ +.. _reuse: https://reuse.software/ +.. _reusediscussion: https://github.com/pombredanne/spdx-pypi-pep/issues/7 +.. _samplesetupcfg: https://github.com/pypa/sampleproject/blob/3a836905fbd687af334db16b16c37cf51dcbc99c/setup.cfg +.. _samplesetuppy: https://github.com/pypa/sampleproject/blob/3a836905fbd687af334db16b16c37cf51dcbc99c/setup.py#L98 +.. _scancodetk: https://github.com/nexB/scancode-toolkit +.. _scipyissue: https://github.com/scipy/scipy/issues/7093 +.. _sdistspec: https://packaging.python.org/specifications/source-distribution-format/ +.. _setuptools5911: https://github.com/pypa/setuptools/blob/v59.1.1/setup.cfg +.. _setuptoolsfiles: https://github.com/pypa/setuptools/issues/2739 +.. _setuptoolspep639: https://github.com/pypa/setuptools/pull/2645 +.. _setuptoolssdist: https://github.com/pypa/setuptools/pull/1767 +.. _spdx: https://spdx.dev/ +.. _spdxid: https://spdx.dev/ids/ +.. _spdxlist: https://spdx.org/licenses/ +.. _spdxpression: https://spdx.github.io/spdx-spec/v2.2.2/SPDX-license-expressions/ +.. _spdxpy: https://github.com/spdx/tools-python/ +.. _spdxtutorial: https://github.com/david-a-wheeler/spdx-tutorial +.. _spdxversion: https://github.com/pombredanne/spdx-pypi-pep/issues/6 +.. _uboot: https://www.denx.de/wiki/U-Boot/Licensing +.. _unlicense: https://unlicense.org/ +.. _wheelfiles: https://github.com/pypa/wheel/issues/138 +.. _wheelproject: https://wheel.readthedocs.io/en/stable/ +.. _wheels: https://github.com/pypa/wheel/blob/0.37.0/docs/user_guide.rst#including-license-files-in-the-generated-wheel-file +.. _wheelspec: https://packaging.python.org/specifications/binary-distribution-format/ + + +Acknowledgments +=============== + +- Nick Coghlan +- Kevin P. Fleming +- Pradyun Gedam +- Oleg Grenrus +- Dustin Ingram +- Chris Jerdonek +- Cyril Roelandt +- Luis Villa + + +Copyright +========= + +This document is placed in the public domain or under the +`CC0-1.0-Universal license `__, whichever is more permissive. diff --git a/pep-0640.rst b/peps/pep-0640.rst similarity index 97% rename from pep-0640.rst rename to peps/pep-0640.rst index 734d86034..2461480fc 100644 --- a/pep-0640.rst +++ b/peps/pep-0640.rst @@ -43,8 +43,8 @@ potentially conflicts with the use of ``"_"`` in internationalization, where a call like gettext.gettext() is bound to ``"_"`` and used to mark strings for translation. -In the proposal to add Pattern Matching to Python (originally PEP 622, now -split into PEP 634, PEP 635 and PEP 636), ``"_"`` has an *additional* +In the proposal to add Pattern Matching to Python (originally :pep:`622`, now +split into :pep:`634`, :pep:`635` and :pep:`636`), ``"_"`` has an *additional* special meaning. It is a wildcard pattern, used in places where variables could be assigned to, to indicate anything should be matched but not assigned to anything. The choice of ``"_"`` there matches the use of ``"_"`` @@ -157,7 +157,7 @@ places. Alternatively, it could be introduced as part of an explanation on assignment in ``for`` loops, showing an example where the loop variable is unused. -PEP 636 discusses how to teach ``"_"``, and can simply replace ``"_"`` with +:pep:`636` discusses how to teach ``"_"``, and can simply replace ``"_"`` with ``"?"``, perhaps noting that ``"?"`` is similarly usable in other contexts. Reference Implementation diff --git a/pep-0641.rst b/peps/pep-0641.rst similarity index 98% rename from pep-0641.rst rename to peps/pep-0641.rst index f2df690f4..a6b21e556 100644 --- a/pep-0641.rst +++ b/peps/pep-0641.rst @@ -10,7 +10,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 20-Oct-2020 Python-Version: 3.10 -Post-History: 2020-10-21 +Post-History: 21-Oct-2020 Resolution: https://discuss.python.org/t/pep-641-using-an-underscore-in-the-version-portion-of-python-3-10-compatibility-tags/5513/42 Abstract @@ -79,7 +79,7 @@ In non-locale ASCII, ``_`` sorts after any digit, so lexicographic sorting matching a sort by Python version of a wheel file name will be kept. -Since PEP 515 (Python 3.6), underscores in numeric literals are ignored. +Since :pep:`515` (Python 3.6), underscores in numeric literals are ignored. This means that ``int("3_10")`` and ``int("310")`` produce the same result, and ordering based on conversion to an integer will be preserved. **However**, this is still a bad way to sort tags, and the point is raised diff --git a/pep-0642.rst b/peps/pep-0642.rst similarity index 92% rename from pep-0642.rst rename to peps/pep-0642.rst index 47df5281d..6418dfa23 100644 --- a/pep-0642.rst +++ b/peps/pep-0642.rst @@ -4,26 +4,26 @@ Version: $Revision$ Last-Modified: $Date$ Author: Nick Coghlan BDFL-Delegate: -Discussions-To: Python-Dev -Status: Draft +Discussions-To: python-dev@python.org +Status: Rejected Type: Standards Track Content-Type: text/x-rst Requires: 634 Created: 26-Sep-2020 Python-Version: 3.10 -Post-History: 31-Oct-2020, 8-Nov-2020, 3-Jan-2021 -Resolution: +Post-History: 31-Oct-2020, 08-Nov-2020, 03-Jan-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/SQC2FTLFV5A7DV7RCEAR2I2IKJKGK7W3/ Abstract ======== -This PEP covers an alternative syntax proposal for PEP 634's structural pattern +This PEP covers an alternative syntax proposal for :pep:`634`'s structural pattern matching that requires explicit prefixes on all capture patterns and value constraints. It also proposes a new dedicated syntax for instance attribute patterns that aligns more closely with the proposed mapping pattern syntax. While the result is necessarily more verbose than the proposed syntax in -PEP 634, it is still significantly less verbose than the status quo. +:pep:`634`, it is still significantly less verbose than the status quo. As an example, the following match statement would extract "host" and "port" details from a 2 item sequence, a mapping with "host" and "port" keys, any @@ -81,33 +81,33 @@ The intent of this approach is to: adopted in ordinary expressions as a way to more easily retrieve a tuple containing multiple attributes from the same object -Relative to PEP 634, the proposal also deliberately eliminates any syntax that +Relative to :pep:`634`, the proposal also deliberately eliminates any syntax that "binds to the right" without using the ``as`` keyword (using capture patterns -in PEP 634's mapping patterns and class patterns) or binds to both the left and -the right in the same pattern (using PEP 634's capture patterns with AS patterns) +in :pep:`634`'s mapping patterns and class patterns) or binds to both the left and +the right in the same pattern (using :pep:`634`'s capture patterns with AS patterns) Relationship with other PEPs ============================ -This PEP both depends on and competes with PEP 634 - the PEP author agrees that +This PEP both depends on and competes with :pep:`634` - the PEP author agrees that match statements would be a sufficiently valuable addition to the language to be worth the additional complexity that they add to the learning process, but disagrees with the idea that "simple name vs literal or attribute lookup" really offers an adequate syntactic distinction between name binding and value lookup operations in match patterns (at least for Python). -This PEP agrees with the spirit of PEP 640 (that the chosen wildcard pattern to +This PEP agrees with the spirit of :pep:`640` (that the chosen wildcard pattern to skip a name binding should be supported everywhere, not just in match patterns), but is now proposing a different spelling for the wildcard syntax (``__`` rather -than ``?``). As such, it competes with PEP 640 as written, but would complement +than ``?``). As such, it competes with :pep:`640` as written, but would complement a proposal to deprecate the use of ``__`` as an ordinary identifier and instead turn it into a general purpose wildcard marker that always skips making a new local variable binding. While it has not yet been put forward as a PEP, Mark Shannon has a pre-PEP draft -[8_] expressing several concerns about the runtime semantics of the pattern -matching proposal in PEP 634. This PEP is somewhat complementary to that one, as +[8]_ expressing several concerns about the runtime semantics of the pattern +matching proposal in :pep:`634`. This PEP is somewhat complementary to that one, as even though this PEP is mostly about surface syntax changes rather than major semantic changes, it does propose that the Abstract Syntax Tree definition be made more explicit to better separate the details of the surface syntax from the @@ -121,13 +121,13 @@ intentionally be governed by the order of the cases in the match statement. Motivation ========== -The original PEP 622 (which was later split into PEP 634, PEP 635, and PEP 636) +The original :pep:`622` (which was later split into :pep:`634`, :pep:`635`, and :pep:`636`) incorporated an unstated but essential assumption in its syntax design: that neither ordinary expressions *nor* the existing assignment target syntax provide an adequate foundation for the syntax used in match patterns. While the PEP didn't explicitly state this assumption, one of the PEP authors -explained it clearly on python-dev [1_]: +explained it clearly on python-dev [1]_: The actual problem that I see is that we have different cultures/intuitions fundamentally clashing here. In particular, so many programmers welcome @@ -155,16 +155,16 @@ The first iteration of this PEP was then born out of an attempt to show that the second assertion was not accurate, and that match patterns could be treated as a variation on assignment targets without leading to inherent contradictions. (An earlier PR submitted to list this option in the "Rejected Ideas" section -of the original PEP 622 had previously been declined [2_]). +of the original :pep:`622` had previously been declined [2]_). However, the review process for this PEP strongly suggested that not only did the contradictions that Tobias mentioned in his email exist, but they were also -concerning enough to cast doubts on the syntax proposal presented in PEP 634. -Accordingly, this PEP was changed to go even further than PEP 634, and largely +concerning enough to cast doubts on the syntax proposal presented in :pep:`634`. +Accordingly, this PEP was changed to go even further than :pep:`634`, and largely abandon alignment between the sequence matching syntax and the existing iterable unpacking syntax (effectively answering "Not really, as least as far as the exact syntax is concerned" to the first question raised in the DLS'20 paper -[9_]: "Can we extend a feature like iterable unpacking to work for more general +[9]_: "Can we extend a feature like iterable unpacking to work for more general object and data layouts?"). This resulted in a complete reversal of the goals of the PEP: rather than @@ -175,7 +175,7 @@ about the new construct based on experience with existing ones. Finally, before completing the 3rd iteration of the proposal (which dropped inferred patterns entirely), the PEP author spent quite a bit of time reflecting -on the following entries in PEP 20: +on the following entries in :pep:`20`: * Explicit is better than implicit. * Special cases aren't special enough to break the rules. @@ -192,7 +192,7 @@ Specification ============= This PEP retains the overall ``match``/``case`` statement structure and semantics -from PEP 634, but proposes multiple changes that mean that user intent is +from :pep:`634`, but proposes multiple changes that mean that user intent is explicitly specified in the concrete syntax rather than needing to be inferred from the pattern matching context. @@ -242,14 +242,14 @@ Closed patterns are patterns which either consist of a single token (i.e. ``__``), or else have a closing delimiter as a required part of their syntax (e.g. ``[as x, as y]``, ``object{.x as x, .y as y}``). -As in PEP 634, the ``match`` and ``case`` keywords are soft keywords, i.e. they +As in :pep:`634`, the ``match`` and ``case`` keywords are soft keywords, i.e. they are not reserved words in other grammatical contexts (including at the start of a line if there is no colon where expected). This means that they are recognized as keywords when part of a match statement or case block only, and are allowed to be used in all other contexts as variable or argument names. -Unlike PEP 634, patterns are explicitly defined as a new kind of node in the +Unlike :pep:`634`, patterns are explicitly defined as a new kind of node in the abstract syntax tree - even when surface syntax is shared with existing expression nodes, a distinct abstract node is emitted by the parser. @@ -262,24 +262,22 @@ Match Semantics ^^^^^^^^^^^^^^^ This PEP largely retains the overall pattern matching semantics proposed in -PEP 634. +:pep:`634`. The proposed syntax for patterns changes significantly, and is discussed in detail below. There are also some proposed changes to the semantics of class defined -constraints (class patterns in PEP 634) to eliminate the need to special case +constraints (class patterns in :pep:`634`) to eliminate the need to special case any builtin types (instead, the introduction of dedicated syntax for instance attribute constraints allows the behaviour needed by those builtin types to be specified as applying to any type that sets ``__match_args__`` to ``None``) -.. _guards: - Guards ^^^^^^ -This PEP retains the guard clause semantics proposed in PEP 634. +This PEP retains the guard clause semantics proposed in :pep:`634`. However, the syntax is changed slightly to require that when a guard clause is present, the case pattern must be a *closed* pattern. @@ -296,11 +294,11 @@ Irrefutable case blocks ^^^^^^^^^^^^^^^^^^^^^^^ The definition of irrefutable case blocks changes slightly in this PEP relative -to PEP 634, as capture patterns no longer exist as a separate concept from +to :pep:`634`, as capture patterns no longer exist as a separate concept from AS patterns. Aside from that caveat, the handling of irrefutable cases is the same as in -PEP 634: +:pep:`634`: * wildcard patterns are irrefutable * AS patterns whose left-hand side is irrefutable @@ -312,8 +310,6 @@ PEP 634: must be last. -.. _patterns: - Patterns -------- @@ -383,7 +379,7 @@ is not permitted as a capture target (this is what ``!"__"`` expresses). A capture pattern always succeeds. It binds the subject value to the name using the scoping rules for name binding established for named expressions -in PEP 572. (Summary: the name becomes a local +in :pep:`572`. (Summary: the name becomes a local variable in the closest containing function scope unless there's an applicable ``nonlocal`` or ``global`` statement.) @@ -429,8 +425,6 @@ Subpatterns are mostly required to be closed patterns, but the parentheses may be omitted for value constraints. -.. _value_constraints: - Value constraints ^^^^^^^^^^^^^^^^^ @@ -462,7 +456,7 @@ The rule ``primary`` is defined in the standard Python grammar, and only allows expressions that either consist of a single token, or else are required to end with a closing delimiter. -Value constraints replace PEP 634's literal patterns and value patterns. +Value constraints replace :pep:`634`'s literal patterns and value patterns. Equality constraints are written as ``== EXPR``, while identity constraints are written as ``is EXPR``. @@ -481,18 +475,18 @@ parentheses is desirable. When the same constraint expression occurs multiple times in the same match statement, the interpreter may cache the first value calculated and reuse it, -rather than repeat the expression evaluation. (As for PEP 634 value patterns, +rather than repeat the expression evaluation. (As for :pep:`634` value patterns, this cache is strictly tied to a given execution of a given match statement.) -Unlike literal patterns in PEP 634, this PEP requires that complex +Unlike literal patterns in :pep:`634`, this PEP requires that complex literals be parenthesised to be accepted by the parser. See the Deferred Ideas section for discussion on that point. -If this PEP were to be adopted in preference to PEP 634, then all literal and +If this PEP were to be adopted in preference to :pep:`634`, then all literal and value patterns would instead be written more explicitly as value constraints:: # Literal patterns - match number: + match number: case == 0: print("Nothing") case == 1: @@ -505,7 +499,7 @@ value patterns would instead be written more explicitly as value constraints:: print("Good luck with that...") # Additional literal patterns - match value: + match value: case == True: print("True or 1") case == False: @@ -555,8 +549,6 @@ The ``== (1-1j)`` example illustrates the use of parentheses to turn any subexpression into a closed one. -.. _wildcard_pattern: - Wildcard Pattern ^^^^^^^^^^^^^^^^ @@ -568,9 +560,9 @@ Abstract syntax:: MatchAlways -A wildcard pattern always succeeds. As in PEP 634, it binds no name. +A wildcard pattern always succeeds. As in :pep:`634`, it binds no name. -Where PEP 634 chooses the single underscore as its wildcard pattern for +Where :pep:`634` chooses the single underscore as its wildcard pattern for consistency with other languages, this PEP chooses the double underscore as that has a clearer path towards potentially being made consistent across the entire language, whereas that path is blocked for ``"_"`` by i18n related use cases. @@ -601,7 +593,7 @@ abstract syntax tree. It allows users to add parentheses around patterns to emphasize the intended grouping, and to allow nesting of open patterns when the grammar requires a closed pattern. -Unlike PEP 634, there is no potential ambiguity with sequence patterns, as +Unlike :pep:`634`, there is no potential ambiguity with sequence patterns, as this PEP requires that all sequence patterns be written with square brackets. @@ -696,9 +688,9 @@ sequence. Subpatterns are mostly required to be closed patterns, but the parentheses may be omitted for value constraints. Sequence elements may also be captured -unconditionally without parentheses. +unconditionally without parentheses. -Note: where PEP 634 allows all the same syntactic flexibility as iterable +Note: where :pep:`634` allows all the same syntactic flexibility as iterable unpacking in assignment statements, this PEP restricts sequence patterns specifically to the square bracket form. Given that the open and parenthesised forms are far more popular than square brackets for iterable unpacking, this @@ -752,8 +744,8 @@ and a ``ValueError`` is raised. While it would theoretically be possible to checked for duplicated constant keys at compile time, no such check is currently defined or implemented. -(Note: This semantic description is derived from the PEP 634 reference -implementation, which differs from the PEP 634 specification text at time of +(Note: This semantic description is derived from the :pep:`634` reference +implementation, which differs from the :pep:`634` specification text at time of writing. The implementation seems reasonable, so amending the PEP text seems like the best way to resolve the discrepancy) @@ -900,7 +892,7 @@ converted to an instance attributes constraint as follows: Note: the ``__match_args__ is None`` handling in this PEP replaces the special casing of ``bool``, ``bytearray``, ``bytes``, ``dict``, ``float``, -``frozenset``, ``int``, ``list``, ``set``, ``str``, and ``tuple`` in PEP 634. +``frozenset``, ``int``, ``list``, ``set``, ``str``, and ``tuple`` in :pep:`634`. However, the optimised fast path for those types is retained in the implementation. @@ -911,7 +903,7 @@ Design Discussion Requiring explicit qualification of simple names in match patterns ------------------------------------------------------------------ -The first iteration of this PEP accepted the basic premise of PEP 634 that +The first iteration of this PEP accepted the basic premise of :pep:`634` that iterable unpacking syntax would provide a good foundation for defining a new syntax for pattern matching. @@ -919,13 +911,13 @@ During the review process, however, two major and one minor ambiguity problems were highlighted that arise directly from that core assumption: * most problematically, when binding simple names by default is extended to - PEP 634's proposed class pattern syntax, the ``ATTR=TARGET_NAME`` construct + :pep:`634`'s proposed class pattern syntax, the ``ATTR=TARGET_NAME`` construct binds to the right without using the ``as`` keyword, and uses the normal assignment-to-the-left sigil (``=``) to do it! -* when binding simple names by default is extended to PEP 634's proposed mapping +* when binding simple names by default is extended to :pep:`634`'s proposed mapping pattern syntax, the ``KEY: TARGET_NAME`` construct binds to the right without using the ``as`` keyword -* using a PEP 634 capture pattern together with an AS pattern +* using a :pep:`634` capture pattern together with an AS pattern (``TARGET_NAME_1 as TARGET_NAME_2``) gives an odd "binds to both the left and right" behaviour @@ -957,7 +949,7 @@ no longer read poorly: Resisting the temptation to guess --------------------------------- -PEP 635 looks at the way pattern matching is used in other languages, and +:pep:`635` looks at the way pattern matching is used in other languages, and attempts to use that information to make plausible predictions about the way pattern matching will be used in Python: @@ -1001,7 +993,7 @@ AST nodes could be reused. Interaction with caching of attribute lookups in local variables ---------------------------------------------------------------- -One of the major changes between this PEP and PEP 634 is to use ``== EXPR`` +One of the major changes between this PEP and :pep:`634` is to use ``== EXPR`` for equality constraint lookups, rather than only offering ``NAME.ATTR``. The original motivation for this was to avoid the semantic conflict with regular assignment targets, where ``NAME.ATTR`` is already used in assignment statements @@ -1014,7 +1006,7 @@ user's intent, and instead requiring them to state it explicitly in the syntax. However, even within match statements themselves, the ``name.attr`` syntax for value patterns has an undesirable interaction with local variable assignment, where routine refactorings that would be semantically neutral for any other -Python statement introduce a major semantic change when applied to a PEP 634 +Python statement introduce a major semantic change when applied to a :pep:`634` style match statement. Consider the following code:: @@ -1047,7 +1039,7 @@ being functionally equivalent:: case __: ... # Handle the non-matching case -By contrast, when using PEP 634's value and capture pattern syntaxes that omit +By contrast, when using :pep:`634`'s value and capture pattern syntaxes that omit the marker prefix, the following two statements wouldn't be equivalent at all:: # PEP 634's value pattern syntax @@ -1069,14 +1061,14 @@ This PEP ensures the original semantics are retained under this style of simplistic refactoring: use ``== name`` to force interpretation of the result as a value constraint, use ``as name`` for a name binding. -PEP 634's proposal to offer only the shorthand syntax, with no explicitly +:pep:`634`'s proposal to offer only the shorthand syntax, with no explicitly prefixed form, means that the primary answer on offer is "Well, don't do that, then, only compare against attributes in namespaces, don't compare against simple names". -PEP 622's walrus pattern syntax had another odd interaction where it might not +:pep:`622`'s walrus pattern syntax had another odd interaction where it might not bind the same object as the exact same walrus expression in the body of the -case clause, but PEP 634 fixed that discrepancy by replacing walrus patterns +case clause, but :pep:`634` fixed that discrepancy by replacing walrus patterns with AS patterns (where the fact that the value bound to the name on the RHS might not be the same value as returned by the LHS is a standard feature common to all uses of the "as" keyword). @@ -1091,14 +1083,14 @@ next question is to ask exactly what that prefix should be. The initially published version of this PEP proposed using the previously unused ``?`` symbol as the prefix for equality constraints, and ``?is`` as the prefix for identity constraints. When reviewing the PEP, Steven D'Aprano -presented a compelling counterproposal [5_] to use the existing comparison +presented a compelling counterproposal [5]_ to use the existing comparison operators (``==`` and ``is``) instead. There were a few concerns with ``==`` as a prefix that kept it from being chosen as the prefix in the initial iteration of the PEP: * for common use cases, it's even more visually noisy than ``?``, as a lot of - folks with PEP 8 trained aesthetic sensibilities are going to want to put + folks with :pep:`8` trained aesthetic sensibilities are going to want to put a space between it and the following expression, effectively making it a 3 character prefix instead of 1 * when used in a mapping pattern, there needs to be a space between the ``:`` @@ -1107,7 +1099,7 @@ chosen as the prefix in the initial iteration of the PEP: * when used in an OR pattern, there needs to be a space between the ``|`` pattern separator and the ``==`` prefix, or the tokeniser will split them up incorrectly (getting ``|=`` and ``=`` instead of ``|`` and ``==``) -* if used in a PEP 634 style class pattern, there needs to be a space between +* if used in a :pep:`634` style class pattern, there needs to be a space between the ``=`` keyword separator and the ``==`` prefix, or the tokeniser will split them up incorrectly (getting ``==`` and ``=`` instead of ``=`` and ``==``) @@ -1129,7 +1121,7 @@ ambiguity concerns. Instead, the following points apply: reducing the need to combine OR patterns with equality constraints (instead, the values to be checked against would be collected as a set, list, or tuple). -Given that perspective, PEP 635's arguments against using ``?`` as part of the +Given that perspective, :pep:`635`'s arguments against using ``?`` as part of the pattern matching syntax held for this proposal as well, and so the PEP was amended accordingly. @@ -1137,19 +1129,19 @@ amended accordingly. Using ``__`` as the wildcard pattern marker ------------------------------------------- -PEP 635 makes a solid case that introducing ``?`` *solely* as a wildcard pattern +:pep:`635` makes a solid case that introducing ``?`` *solely* as a wildcard pattern marker would be a bad idea. With the syntax for value constraints changed to use existing comparison operations rather than ``?`` and ``?is``, that argument holds for this PEP as well. -However, as noted by Thomas Wouters in [6_], PEP 634's choice of ``_`` remains +However, as noted by Thomas Wouters in [6]_, :pep:`634`'s choice of ``_`` remains problematic as it would likely mean that match patterns would have a *permanent* difference from all other parts of Python - the use of ``_`` in software internationalisation and at the interactive prompt means that there isn't really a plausible path towards using it as a general purpose "skipped binding" marker. ``__`` is an alternative "this value is not needed" marker drawn from a Stack -Overflow answer [7_] (originally posted by the author of this PEP) on the +Overflow answer [7]_ (originally posted by the author of this PEP) on the various meanings of ``_`` in existing Python code. This PEP also proposes adopting an implementation technique that limits @@ -1193,13 +1185,13 @@ be consistent with that existing approach. Representing patterns explicitly in the Abstract Syntax Tree ------------------------------------------------------------ -PEP 634 doesn't explicitly discuss how match statements should be represented +:pep:`634` doesn't explicitly discuss how match statements should be represented in the Abstract Syntax Tree, instead leaving that detail to be defined as part of the implementation. -As a result, while the reference implementation of PEP 634 definitely works (and +As a result, while the reference implementation of :pep:`634` definitely works (and formed the basis of the reference implementation of this PEP), it does contain -a significant design flaw: despite the notes in PEP 635 that patterns should be +a significant design flaw: despite the notes in :pep:`635` that patterns should be considered as distinct from expressions, the reference implementation goes ahead and represents them in the AST as expression nodes. @@ -1225,13 +1217,13 @@ rest of the PEP were to be rejected. Changes to sequence patterns ---------------------------- -This PEP makes one notable change to sequence patterns relative to PEP 634: +This PEP makes one notable change to sequence patterns relative to :pep:`634`: * only the square bracket form of sequence pattern is supported. Neither open (no delimiters) nor tuple style (parentheses as delimiters) sequence patterns are supported. -Relative to PEP 634, sequence patterns are also significantly affected by the +Relative to :pep:`634`, sequence patterns are also significantly affected by the change to require explicit qualification of capture patterns and value constraints, as it means ``case [a, b, c]:`` must instead be written as ``case [as a, as b, as c]:`` and ``case [0, 1]:`` must instead be written as @@ -1245,7 +1237,7 @@ consistency with iterable unpacking. Allowing open and tuple style sequence patterns didn't increase expressivity, only ambiguity of intent (especially relative to group patterns), and encouraged readers down the path of viewing pattern matching syntax as intrinsically linked -to assignment target syntax (which the PEP 634 authors have stated multiple +to assignment target syntax (which the :pep:`634` authors have stated multiple times is not a desirable path to have readers take, and a view the author of this PEP now shares, despite disagreeing with it originally). @@ -1253,7 +1245,7 @@ this PEP now shares, despite disagreeing with it originally). Changes to mapping patterns --------------------------- -This PEP makes two notable changes to mapping patterns relative to PEP 634: +This PEP makes two notable changes to mapping patterns relative to :pep:`634`: * value capturing is written as ``KEY as NAME`` rather than as ``KEY: NAME`` * a wider range of keys are permitted: any "closed expression", rather than @@ -1267,10 +1259,10 @@ The second change is mostly a matter of simplifying the parser and code generator code by reusing the existing expression handling machinery. The restriction to closed expressions is designed to help reduce ambiguity as to where the key expression ends and the match pattern begins. This mostly allows -a superset of what PEP 634 allows, except that complex literals must be written +a superset of what :pep:`634` allows, except that complex literals must be written in parentheses (at least for now). -Adapting PEP 635's mapping pattern examples to the syntax proposed in this PEP:: +Adapting :pep:`635`'s mapping pattern examples to the syntax proposed in this PEP:: match json_pet: case {"type": == "cat", "name" as name, "pattern" as pattern}: @@ -1288,7 +1280,7 @@ Adapting PEP 635's mapping pattern examples to the syntax proposed in this PEP:: for child in children: change_red_to_blue(child) -For reference, the equivalent PEP 634 syntax:: +For reference, the equivalent :pep:`634` syntax:: match json_pet: case {"type": "cat", "name": name, "pattern": pattern}: @@ -1310,7 +1302,7 @@ For reference, the equivalent PEP 634 syntax:: Changes to class patterns ------------------------- -This PEP makes several notable changes to class patterns relative to PEP 634: +This PEP makes several notable changes to class patterns relative to :pep:`634`: * the syntactic alignment with class instantiation is abandoned as being actively misleading and unhelpful. Instead, a new dedicated syntax for @@ -1350,13 +1342,13 @@ matter of considering the leading ``.`` sufficient to render the name usage unambiguous (it's clearly an attribute reference, whereas matching against a variable key in a mapping pattern would be arguably ambiguous) -The final change just supplements a CPython-internal-only check in the PEP 634 +The final change just supplements a CPython-internal-only check in the :pep:`634` reference implementation by making it the default behaviour that classes get if they don't define ``__match_args__`` (the optimised fast path for the builtin -and standard library types named in PEP 634 is retained). +and standard library types named in :pep:`634` is retained). Adapting the class matching example -`linked from PEP 635 `_ +`linked from PEP 635 `_ shows that for purely positional class matching, the main impact comes from the changes to value constraints and name binding, not from the class matching changes:: @@ -1381,7 +1373,7 @@ changes:: case __: raise ValueError(f"Invalid expression value: {repr(expr)}") -For reference, the equivalent PEP 634 syntax:: +For reference, the equivalent :pep:`634` syntax:: match expr: case BinaryOp('+', left, right): @@ -1413,7 +1405,7 @@ checking for named attributes and extracting their values without relying on case object{.host as host}: pass -Compare this to the PEP 634 equivalent, where it really isn't clear which names +Compare this to the :pep:`634` equivalent, where it really isn't clear which names are referring to attributes of the match subject and which names are referring to local variables:: @@ -1470,7 +1462,7 @@ restrictions later can be considered on a case-by-case basis. Accepting complex literals as closed expressions ------------------------------------------------ -PEP 634's reference implementation includes a lot of special casing of binary +:pep:`634`'s reference implementation includes a lot of special casing of binary operations in both the parser and the rest of the compiler in order to accept complex literals without accepting arbitrary binary numeric operations on literal values. @@ -1526,7 +1518,7 @@ Allowing membership checks in match patterns The syntax used for equality and identity constraints would be straightforward to extend to membership checks: ``in container``. -One downside of the proposals in both this PEP and PEP 634 is that checking +One downside of the proposals in both this PEP and :pep:`634` is that checking for multiple values in the same case doesn't look like any existing container membership check in Python:: @@ -1541,7 +1533,7 @@ membership check in Python:: ... Allowing inferred equality constraints under this PEP would only make it look -like the PEP 634 example, it still wouldn't look like the equivalent ``if`` +like the :pep:`634` example, it still wouldn't look like the equivalent ``if`` statement header (``if value in {0, 1, 2, 3}:``). Membership constraints would provide a more explicit, but still concise, way @@ -1583,7 +1575,7 @@ allowing this has been deferred as a topic for possible future consideration. Avoiding special cases in sequence patterns ------------------------------------------- -Sequence patterns in both this PEP and PEP 634 currently special case ``str``, +Sequence patterns in both this PEP and :pep:`634` currently special case ``str``, ``bytes``, and ``bytearray`` as specifically *never* matching a sequence pattern. @@ -1638,14 +1630,14 @@ Restricting permitted expressions in value constraints and mapping pattern keys While it's entirely technically possible to restrict the kinds of expressions permitted in value constraints and mapping pattern keys to just attribute -lookups and constant literals (as PEP 634 does), there isn't any clear runtime +lookups and constant literals (as :pep:`634` does), there isn't any clear runtime value in doing so, so this PEP proposes allowing any kind of primary expression (primary expressions are an existing node type in the grammar that includes things like literals, names, attribute lookups, function calls, container subscripts, parenthesised groups, etc), as well as high precedence unary operations (``+``, ``-``, ``~``) on primary expressions. -While PEP 635 does emphasise several times that literal patterns and value +While :pep:`635` does emphasise several times that literal patterns and value patterns are not full expressions, it doesn't ever articulate a concrete benefit that is obtained from that restriction (just a theoretical appeal to it being useful to separate static checks from dynamic checks, which a code style @@ -1658,7 +1650,7 @@ that just returned their argument) to express the behaviour they wanted before the language definition was finally updated to allow arbitrary expressions and let users make their own decisions about readability. -The situation in PEP 634 that bears a resemblance to the situation with decorator +The situation in :pep:`634` that bears a resemblance to the situation with decorator expressions is that arbitrary expressions are technically supported in value patterns, they just require awkward workarounds where either all the values to match need to be specified in a helper class that is placed before the match @@ -1699,7 +1691,7 @@ pattern caching rule, where the number of times the constraint expression actually gets evaluated will be implementation dependent. Even here, the PEP takes the view of letting users write nonsense if they really want to. -Aside from the recenty updated decorator expressions, another situation where +Aside from the recently updated decorator expressions, another situation where Python's formal syntax offers full freedom of expression that is almost never used in practice is in ``except`` clauses: the exceptions to match against almost always take the form of a simple name, a dotted name, or a tuple of @@ -1722,7 +1714,7 @@ Requiring the use of constraint prefix markers for mapping pattern keys ----------------------------------------------------------------------- The initial (unpublished) draft of this proposal suggested requiring mapping -pattern keys be value constraints, just as PEP 634 requires that they be valid +pattern keys be value constraints, just as :pep:`634` requires that they be valid literal or value patterns:: import constants @@ -1764,8 +1756,8 @@ a human reader as well: ``case {0: == 0}:`` Reference Implementation ======================== -A draft reference implementation for this PEP [3_] has been derived from Brandt -Bucher's reference implementation for PEP 634 [4_]. +A draft reference implementation for this PEP [3]_ has been derived from Brandt +Bucher's reference implementation for :pep:`634` [4]_. Relative to the text of this PEP, the draft reference implementation has not yet complemented the special casing of several builtin and standard library @@ -1774,7 +1766,7 @@ being set to ``None``. Class defined patterns also currently still accept classes that don't define ``__match_args__``. All other modified patterns have been updated to follow this PEP rather than -PEP 634. +:pep:`634`. Unparsing for match patterns has not yet been migrated to the updated v3 AST. @@ -1787,7 +1779,7 @@ are expected. The examples in this PEP have not yet been converted to test cases, so could plausibly contain typos and other errors. -Several of the old PEP 634 tests are still to be converted to new SyntaxError +Several of the old :pep:`634` tests are still to be converted to new SyntaxError tests. The documentation has not yet been updated. @@ -1796,25 +1788,26 @@ The documentation has not yet been updated. Acknowledgments =============== -The PEP 622 and PEP 634/635/636 authors, as the proposal in this PEP is merely +The :pep:`622` and :pep:`634`/:pep:`635`/:pep:`636` authors, as the proposal in +this PEP is merely an attempt to improve the readability of an already well-constructed idea by proposing that starting with a more explicit syntax and potentially introducing syntactic shortcuts for particularly common operations later is a better option than attempting to *only* define the shortcut version. For areas of the specification where the two PEPs are the same (or at least very similar), the text describing the intended behaviour in this PEP is often derived directly -from the PEP 634 text. +from the :pep:`634` text. Steven D'Aprano, who made a compelling case that the key goals of this PEP could be achieved by using existing comparison tokens to tell the ability to override the compiler when our guesses as to "what most users will want most of the time" are inevitably incorrect for at least some users some of the time, and retaining -some of PEP 634's syntactic sugar (with a slightly different semantic definition) -to obtain the same level of brevity as PEP 634 in most situations. (Paul +some of :pep:`634`'s syntactic sugar (with a slightly different semantic definition) +to obtain the same level of brevity as :pep:`634` in most situations. (Paul Sokolosvsky also independently suggested using ``==`` instead of ``?`` as a more easily understood prefix for equality constraints). -Thomas Wouters, whose publication of PEP 640 and public review of the structured +Thomas Wouters, whose publication of :pep:`640` and public review of the structured pattern matching proposals persuaded the author of this PEP to continue advocating for a wildcard pattern syntax that a future PEP could plausibly turn into a hard keyword that always skips binding a reference in any location a @@ -1859,15 +1852,15 @@ References https://gvanrossum.github.io/docs/PyPatternMatching.pdf -.. _Appendix A: +.. _642-appendix-a: Appendix A -- Full Grammar ========================== Here is the full modified grammar for ``match_stmt``, replacing Appendix A -in PEP 634. +in :pep:`634`. -Notation used beyond standard EBNF is as per PEP 534: +Notation used beyond standard EBNF is as per :pep:`534`: - ``'KWD'`` denotes a hard keyword - ``"KWD"`` denotes a soft keyword @@ -1969,7 +1962,7 @@ Notation used beyond standard EBNF is as per PEP 534: | '**' '{' [attrs_constraint_elements] '}' -.. _Appendix B: +.. _642-appendix-b: Appendix B: Summary of Abstract Syntax Tree changes =================================================== @@ -2002,15 +1995,15 @@ The following new nodes are added to the AST by this PEP:: matchop = EqCheck | IdCheck -.. _Appendix C: +.. _642-appendix-c: Appendix C: Summary of changes relative to PEP 634 ================================================== The overall ``match``/``case`` statement syntax and the guard expression syntax -remain the same as they are in PEP 634. +remain the same as they are in :pep:`634`. -Relative to PEP 634 this PEP makes the following key changes: +Relative to :pep:`634` this PEP makes the following key changes: * a new ``pattern`` type is defined in the AST, rather than reusing the ``expr`` type for patterns @@ -2057,7 +2050,7 @@ Relative to PEP 634 this PEP makes the following key changes: the pattern to be matched starts with ``==``, ``is``, or ``as`` * class patterns treat any class that sets ``__match_args__`` to ``None`` as accepting a single positional pattern that is matched against the entire - object (avoiding the special casing required in PEP 634) + object (avoiding the special casing required in :pep:`634`) * class patterns raise ``TypeError`` when used with an object that does not define ``__match_args__`` * dedicated syntax for ducktyping is added, such that ``case cls{...}:`` is @@ -2069,20 +2062,20 @@ Note that postponing literal patterns also makes it possible to postpone the question of whether we need an "INUMBER" token in the tokeniser for imaginary literals. Without it, the parser can't distinguish complex literals from other binary addition and subtraction operations on constants, so proposals like -PEP 634 have to do work in later compilation steps to check for correct usage. +:pep:`634` have to do work in later compilation steps to check for correct usage. -.. _Appendix D: +.. _642-appendix-d: Appendix D: History of changes to this proposal =============================================== -The first published iteration of this proposal mostly followed PEP 634, but +The first published iteration of this proposal mostly followed :pep:`634`, but suggested using ``?EXPR`` for equality constraints and ``?is EXPR`` for -identity constraints rather than PEP 634's value patterns and literal patterns. +identity constraints rather than :pep:`634`'s value patterns and literal patterns. The second published iteration mostly adopted a counter-proposal from Steven -D'Aprano that kept the PEP 634 style inferred constraints in many situations, +D'Aprano that kept the :pep:`634` style inferred constraints in many situations, but also allowed the use of ``== EXPR`` for explicit equality constraints, and ``is EXPR`` for explicit identity constraints. @@ -2103,14 +2096,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0643.rst b/peps/pep-0643.rst similarity index 98% rename from pep-0643.rst rename to peps/pep-0643.rst index 66e52c445..08fc1f675 100644 --- a/pep-0643.rst +++ b/peps/pep-0643.rst @@ -3,14 +3,18 @@ Title: Metadata for Package Source Distributions Author: Paul Moore BDFL-Delegate: Paul Ganssle Discussions-To: https://discuss.python.org/t/pep-643-metadata-for-package-source-distributions/5577 -Status: Accepted +Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 24-Oct-2020 Post-History: 24-Oct-2020, 01-Nov-2020, 02-Nov-2020, 14-Nov-2020 Resolution: https://discuss.python.org/t/pep-643-metadata-for-package-source-distributions/5577/53 +.. canonical-pypa-spec:: :ref:`packaging:core-metadata` + + Abstract ======== diff --git a/pep-0644.rst b/peps/pep-0644.rst similarity index 97% rename from pep-0644.rst rename to peps/pep-0644.rst index 39f990675..4532d0b7b 100644 --- a/pep-0644.rst +++ b/peps/pep-0644.rst @@ -1,7 +1,6 @@ PEP: 644 Title: Require OpenSSL 1.1.1 or newer Author: Christian Heimes -BDFL-Delegate: n/a Discussions-To: https://discuss.python.org/t/pep-644-require-openssl-1-1-or-newer/5584 Status: Final Type: Standards Track @@ -9,8 +8,8 @@ Content-Type: text/x-rst Created: 27-Oct-2020 Python-Version: 3.10 Post-History: 27-Oct-2020, 03-Mar-2021, 17-Mar-2021, 17-Apr-2021 -Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/INLCO2EZVQW7R7J2OL6HWVLVU3TQRAZV/, - https://bugs.python.org/issue43669 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/INLCO2EZVQW7R7J2OL6HWVLVU3TQRAZV/ + Abstract ======== @@ -323,7 +322,7 @@ Debian Buster and Debian Bullseye will be available. Instead Python 3.10 will gain additional documentation and a new ``configure`` option ``--with-openssl-rpath=auto`` to simplify use of custom -OpenSSL builds [11]. +OpenSSL builds [11]_. Backwards Compatibility @@ -372,14 +371,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0645.rst b/peps/pep-0645.rst similarity index 60% rename from pep-0645.rst rename to peps/pep-0645.rst index 9368caaac..922f8e670 100644 --- a/pep-0645.rst +++ b/peps/pep-0645.rst @@ -2,10 +2,11 @@ PEP: 645 Title: Allow writing optional types as ``x?`` Author: Maggie Moss Sponsor: Guido van Rossum -Status: Draft -Type: Process +Status: Withdrawn +Type: Standards Track Content-Type: text/x-rst Created: 25-Aug-2020 +Resolution: https://mail.python.org/archives/list/typing-sig@python.org/message/E75SPV6DDHLEEFSA5MBN5HUOQWDMUQJ2/ Abstract @@ -13,13 +14,28 @@ Abstract This PEP proposes adding a ``?`` operator for types to allow writing ``int?`` in place of ``Optional[int]``. +PEP Withdrawal +============== + +The notation ``T|None`` introduced by :pep:`604` to write ``Optional[T]`` is a +fine alternative to ``T?`` and does not require new syntax. + +Using ``T?`` to mean ``T|None`` is also inconsistent with TypeScript +where it roughly means ``NotRequired[T]``. +Such inconsistency would likely confuse folks coming from TypeScript to Python. + +The above represents the consensus of +`typing-sig `_ +and the sponsor of this PEP. + + Motivation ========== Types have become a valuable and powerful part of the Python language. However, many type annotations are verbose and add considerable friction to using type annotations. By improving the typing syntax, adding types to Python code becomes simpler and improves the development experience for Python users. -In a similar vein, a PEP to introduce short hand syntax for `Union types `_ [1]_ has +In a similar vein, a PEP to introduce short hand syntax for :pep:`Union types <604>` has been approved and implemented. @@ -27,18 +43,18 @@ Rationale ========= Types in Python can be quite verbose, this can be a hindrance when working towards type adoption. Making types more ergonomic, -as was done with the Union type in PEP 604 (e.g., int | str), would reduce the effort needed to add types to new and existing Python code. +as was done with the Union type in :pep:`604` (e.g., int | str), would reduce the effort needed to add types to new and existing Python code. The Optional annotation is used frequently in both partially and fully typed Python code bases. In a small sampling of `5 well-typed open source projects, on average 7% of annotations -`_ [2] included at least one optional type. This indicates +`_ included at least one optional type. This indicates that updating the syntax has the potential to make types more concise, reduce code length and improve readability. -Simplifying the syntax for optionals has been `discussed previously `_ [3] within the typing community. +Simplifying the syntax for optionals has been `discussed previously `_ within the typing community. The consensus during these conversations has been that ``?`` is the preferred operator. There is no native support for unary ``?`` in Python and this will need to be added to the runtime. -Adding the ? sigil to the Python grammar has been proposed previously in `PEP 505 `_ [4], which is currently in a deferred state. -PEP 505 proposes a: +Adding the ? sigil to the Python grammar has been proposed previously in :pep:`505`, which is currently in a deferred state. +:pep:`505` proposes a: - "None coalescing" binary operator ``??`` @@ -47,17 +63,19 @@ PEP 505 proposes a: - "None-aware indexing" operator ``?[]`` ("maybe subscript") -Should PEP 505 be approved in the future, it would not interfere with the typing specific ``?`` proposed in this PEP. As well, +Should :pep:`505` be approved in the future, it would not interfere with the typing specific ``?`` proposed in this PEP. As well, since all uses of the ``?`` would be conceptually related, it would not be confusing in terms of learning Python or a hindrance to quick visual comprehension. The proposed syntax, with the postfix operator, mimics the optional syntax found in other typed languages, like C#, TypeScript and Swift. -The widespread adoption and popularity of these languages means that Python developers are likely already familiar with this syntax.:: +The widespread adoption and popularity of these languages means that Python developers are likely already familiar with this syntax. - // Optional in Swift - var example: String? +.. code:: text - // Optional in C# - string? example; + // Optional in Swift + var example: String? + + // Optional in C# + string? example; Adding this syntax would also follow the often used pattern of using builtin types as annotations. For example, ``list``, ``dict`` and ``None``. This would allow more annotations to be added to Python code without importing from ``typing``. @@ -68,44 +86,44 @@ Specification The new optional syntax should be accepted for function, variable, attribute and parameter annotations. -:: +.. code:: text - # instead of - # def foo(x: Optional[int], y: Optional[str], z: Optional[list[int]): ... - def foo(x: int?, y: str?, x: list[int]?): ... + # instead of + # def foo(x: Optional[int], y: Optional[str], z: Optional[list[int]): ... + def foo(x: int?, y: str?, x: list[int]?): ... - # def bar(x: list[typing.Optional[int]]): ... - def bar(x: list[int?]): ... + # def bar(x: list[typing.Optional[int]]): ... + def bar(x: list[int?]): ... The new optional syntax should be equivalent to the existing typing.Optional syntax -:: +.. code:: text - typing.Optional[int] == int? + typing.Optional[int] == int? The new optional syntax should have the same identity as the existing typing.Optional syntax. -:: +.. code:: text - typing.Optional[int] is int? + typing.Optional[int] is int? It should also be equivalent to a Union with None. -:: +.. code:: text - # old syntax - int? == typing.Union[int, None] + # old syntax + int? == typing.Union[int, None] - # new syntax - int? == int | None + # new syntax + int? == int | None -Since the new Union syntax specified in PEP 604 is supported in ``isinstance`` and ``issubclass``, the new optional syntax should be supported in both ``isinstance`` and ``issubclass``, +Since the new Union syntax specified in :pep:`604` is supported in ``isinstance`` and ``issubclass``, the new optional syntax should be supported in both ``isinstance`` and ``issubclass``, -:: +.. code:: text - isinstance(1, int?) # true - issubclass(Child, Super?) # true + isinstance(1, int?) # true + issubclass(Child, Super?) # true A new dunder method will need to be implemented to allow the ``?`` operator to be overloaded for other functionality. @@ -118,7 +136,7 @@ Backwards Compatibility Reference Implementation ======================== -A reference implementation can be found `here `_ [5]. +A reference implementation can be found `here `_. Rejected Ideas ============== @@ -129,30 +147,7 @@ Discussed alternatives were * A prefix operator (``?int``). -References -========== - -.. [1] PEP 604 - (https://www.python.org/dev/peps/pep-0604/) -.. [2] Use of Optional Annotations in Open Source Python projects - (https://gist.github.com/MaggieMoss/fd8dfe002b2702fae243dbf81a62624e) -.. [3] Github Issue Discussion of Optional syntax - (https://github.com/python/typing/issues/429) -.. [4] PEP 505 - (https://www.python.org/dev/peps/pep-0505/) -.. [5] Reference Implementation - (https://github.com/python/cpython/compare/master...MaggieMoss:new-optional-syntax-postfix) - Copyright ========= This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0646.rst b/peps/pep-0646.rst similarity index 73% rename from pep-0646.rst rename to peps/pep-0646.rst index 7cf08d456..20770d1b6 100644 --- a/pep-0646.rst +++ b/peps/pep-0646.rst @@ -5,17 +5,19 @@ Author: Mark Mendoza , Pradeep Kumar Srinivasan , Vincent Siles Sponsor: Guido van Rossum -Status: Draft +Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 16-Sep-2020 -Python-Version: 3.10 +Python-Version: 3.11 Post-History: 07-Oct-2020, 23-Dec-2020, 29-Dec-2020 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/OR5RKV7GAVSGLVH3JAGQ6OXFAXIP5XDX/ Abstract ======== -PEP 484 introduced ``TypeVar``, enabling creation of generics parameterised +:pep:`484` introduced ``TypeVar``, enabling creation of generics parameterised with a single type. In this PEP, we introduce ``TypeVarTuple``, enabling parameterisation with an *arbitrary* number of types - that is, a *variadic* type variable, enabling *variadic* generics. This enables a wide variety of use cases. @@ -24,6 +26,14 @@ in numerical computing libraries such as NumPy and TensorFlow to be parameterised with the array *shape*, enabling static type checkers to catch shape-related bugs in code that uses these libraries. +Acceptance +========== + +This PEP was accepted for Python 3.11, with the caveat that details around +multiple unpackings in a type expression aren't specified precisely. +This gives individual type checkers some leeway, but can be tightened +in future PEPs. + Motivation ========== @@ -146,18 +156,21 @@ data type.) Specification ============= -In order to support the above use cases, we introduce ``TypeVarTuple``. This serves as a placeholder not for a single type but for an *arbitrary* number of types, and behaving like a number of ``TypeVar`` instances packed in a ``Tuple``. +In order to support the above use cases, we introduce +``TypeVarTuple``. This serves as a placeholder not for a single type +but for a *tuple* of types. In addition, we introduce a new use for the star operator: to 'unpack' -``TypeVarTuple`` instances, in order to access the type variables -contained in the tuple. +``TypeVarTuple`` instances and tuple types such as ``Tuple[int, +str]``. Unpacking a ``TypeVarTuple`` or tuple type is the typing +equivalent of unpacking a variable or a tuple of values. Type Variable Tuples -------------------- -In the same way that a normal type variable is a stand-in for a single type, -a type variable *tuple* is a stand-in for an arbitrary number of types (zero or -more) in a flat ordered list. +In the same way that a normal type variable is a stand-in for a single +type such as ``int``, a type variable *tuple* is a stand-in for a *tuple* type such as +``Tuple[int, str]``. Type variable tuples are created with: @@ -167,6 +180,9 @@ Type variable tuples are created with: Ts = TypeVarTuple('Ts') +Using Type Variable Tuples in Generic Classes +''''''''''''''''''''''''''''''''''''''''''''' + Type variable tuples behave like a number of individual type variables packed in a ``Tuple``. To understand this, consider the following example: @@ -199,6 +215,9 @@ and so on: y: Array[Batch, Height, Width] = Array() z: Array[Time, Batch, Height, Width] = Array() +Using Type Variable Tuples in Functions +''''''''''''''''''''''''''''''''''''''' + Type variable tuples can be used anywhere a normal ``TypeVar`` can. This includes class definitions, as shown above, as well as function signatures and variable annotations: @@ -264,74 +283,6 @@ To keep this PEP minimal, ``TypeVarTuple`` does not yet support specification of We leave the decision of how these arguments should behave to a future PEP, when variadic generics have been tested in the field. As of this PEP, type variable tuples are invariant. -Behaviour when Type Parameters are not Specified -'''''''''''''''''''''''''''''''''''''''''''''''' - -When a generic class parameterised by a type variable tuple is used without -any type parameters, it behaves as if its type parameters are '``Any, ...``' -(an arbitrary number of ``Any``): - -:: - - def takes_any_array(arr: Array): ... - - x: Array[Height, Width] - takes_any_array(x) # Valid - y: Array[Time, Height, Width] - takes_any_array(y) # Also valid - -This enables gradual typing: existing functions accepting, for example, -a plain TensorFlow ``Tensor`` will still be valid even if ``Tensor`` is made -generic and calling code passes a ``Tensor[Height, Width]``. - -This also works in the opposite direction: - -:: - - def takes_specific_array(arr: Array[Height, Width]): ... - - z: Array - takes_specific_array(z) - -This way, even if libraries are updated to use types like ``Array[Height, Width]``, -users of those libraries won't be forced to also apply type annotations to -all of their code; users still have a choice about what parts of their code -to type and which parts to not. - -Type Variable Tuples Must Have Known Length -''''''''''''''''''''''''''''''''''''''''''' - -Type variables tuples may not be bound to a type with unknown length. -That is: - -:: - - def foo(x: Tuple[*Ts]): ... - - x: Tuple[float, ...] - foo(x) # NOT valid; Ts would be bound to ``Tuple[float, ...]`` - -If this is confusing - didn't we say that type variable tuples are a stand-in -for an *arbitrary* number of types? - note the difference between the -length of the type variable tuple *itself*, and the length of the type it is -*bound* to. Type variable tuples themselves can be of arbitrary length - -that is, they can be bound to ``Tuple[int]``, ``Tuple[int, int]``, and -so on - but the types they are bound to must be of known length - -that is, ``Tuple[int, int]``, but not ``Tuple[int, ...]``. - -Note that, as a result of this rule, omitting the type parameter list is the -*only* way of instantiating a generic type with an arbitrary number of -type parameters. (We plan to introduce a more deliberate syntax for this -case in a future PEP.) For example, an unparameterised ``Array`` may -*behave* like ``Array[Any, ...]``, but it cannot be instantiated using -``Array[Any, ...]``, because this would bind its type variable tuple to ``Tuple[Any, ...]``: - -:: - - x: Array # Valid - y: Array[int, ...] # Error - z: Array[Any, ...] # Error - Type Variable Tuple Equality '''''''''''''''''''''''''''' @@ -375,6 +326,11 @@ As of this PEP, only a single type variable tuple may appear in a type parameter class Array(Generic[*Ts1, *Ts2]): ... # Error +The reason is that multiple type variable tuples make it ambiguous +which parameters get bound to which type variable tuple: :: + + x: Array[int, str, bool] # Ts1 = ???, Ts2 = ??? + Type Concatenation ------------------ @@ -397,7 +353,7 @@ prefixed and/or suffixed: b = add_batch_axis(a) # Inferred type is Array[Batch, Height, Width] c = del_batch_axis(b) # Array[Height, Width] d = add_batch_channels(a) # Array[Batch, Height, Width, Channels] - + Normal ``TypeVar`` instances can also be prefixed and/or suffixed: @@ -414,10 +370,106 @@ Normal ``TypeVar`` instances can also be prefixed and/or suffixed: z = prefix_tuple(x=0, y=(True, 'a')) # Inferred type of z is Tuple[int, bool, str] +Unpacking Tuple Types +--------------------- + +We mentioned that a ``TypeVarTuple`` stands for a tuple of types. +Since we can unpack a ``TypeVarTuple``, for consistency, we also +allow unpacking a tuple type. As we shall see, this also enables a +number of interesting features. + + +Unpacking Concrete Tuple Types +'''''''''''''''''''''''''''''' + +Unpacking a concrete tuple type is analogous to unpacking a tuple of +values at runtime. ``Tuple[int, *Tuple[bool, bool], str]`` is +equivalent to ``Tuple[int, bool, bool, str]``. + +Unpacking Unbounded Tuple Types +''''''''''''''''''''''''''''''' + +Unpacking an unbounded tuple preserves the unbounded tuple as it is. +That is, ``*Tuple[int, ...]`` remains ``*Tuple[int, ...]``; there's no +simpler form. This enables us to specify types such as ``Tuple[int, +*Tuple[str, ...], str]`` - a tuple type where the first element is +guaranteed to be of type ``int``, the last element is guaranteed to be +of type ``str``, and the elements in the middle are zero or more +elements of type ``str``. Note that ``Tuple[*Tuple[int, ...]]`` is +equivalent to ``Tuple[int, ...]``. + +Unpacking unbounded tuples is also useful in function signatures where +we don't care about the exact elements and don't want to define an +unnecessary ``TypeVarTuple``: + +:: + + def process_batch_channels( + x: Array[Batch, *Tuple[Any, ...], Channels] + ) -> None: + ... + + + x: Array[Batch, Height, Width, Channels] + process_batch_channels(x) # OK + y: Array[Batch, Channels] + process_batch_channels(y) # OK + z: Array[Batch] + process_batch_channels(z) # Error: Expected Channels. + + +We can also pass a ``*Tuple[int, ...]`` wherever a ``*Ts`` is +expected. This is useful when we have particularly dynamic code and +cannot state the precise number of dimensions or the precise types for +each of the dimensions. In those cases, we can smoothly fall back to +an unbounded tuple: + +:: + + y: Array[*Tuple[Any, ...]] = read_from_file() + + def expect_variadic_array( + x: Array[Batch, *Shape] + ) -> None: ... + + expect_variadic_array(y) # OK + + def expect_precise_array( + x: Array[Batch, Height, Width, Channels] + ) -> None: ... + + expect_precise_array(y) # OK + +``Array[*Tuple[Any, ...]]`` stands for an array with an arbitrary +number of dimensions of type ``Any``. This means that, in the call to +``expect_variadic_array``, ``Batch`` is bound to ``Any`` and ``Shape`` +is bound to ``Tuple[Any, ...]``. In the call to +``expect_precise_array``, the variables ``Batch``, ``Height``, +``Width``, and ``Channels`` are all bound to ``Any``. + +This allows users to handle dynamic code gracefully while still +explicitly marking the code as unsafe (by using ``y: Array[*Tuple[Any, +...]]``). Otherwise, users would face noisy errors from the type +checker every time they tried to use the variable ``y``, which would +hinder them when migrating a legacy code base to use ``TypeVarTuple``. + +Multiple Unpackings in a Tuple: Not Allowed +''''''''''''''''''''''''''''''''''''''''''' + +As with ``TypeVarTuples``, `only one `_ unpacking may appear in a tuple: + + +:: + + x: Tuple[int, *Ts, str, *Ts2] # Error + y: Tuple[int, *Tuple[int, ...], str, *Tuple[str, ...]] # Error + + ``*args`` as a Type Variable Tuple ---------------------------------- -PEP 484 states that when a type annotation is provided for ``*args``, every argument +:pep:`484` states that when a type annotation is provided for ``*args``, every argument must be of the type annotated. That is, if we specify ``*args`` to be type ``int``, then *all* arguments must be of type ``int``. This limits our ability to specify the type signatures of functions that take heterogeneous argument types. @@ -428,13 +480,60 @@ individual arguments become the types in the type variable tuple: :: Ts = TypeVarTuple('Ts') - + def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... args_to_tuple(1, 'a') # Inferred type is Tuple[int, str] -If no arguments are passed, the type variable tuple behaves like an -empty tuple, ``Tuple[()]``. +In the above example, ``Ts`` is bound to ``Tuple[int, str]``. If no +arguments are passed, the type variable tuple behaves like an empty +tuple, ``Tuple[()]``. + +As usual, we can unpack any tuple types. For example, by using a type +variable tuple inside a tuple of other types, we can refer to prefixes +or suffixes of the variadic argument list. For example: + +:: + + # os.execle takes arguments 'path, arg0, arg1, ..., env' + def execle(path: str, *args: *Tuple[*Ts, Env]) -> None: ... + +Note that this is different to + +:: + + def execle(path: str, *args: *Ts, env: Env) -> None: ... + +as this would make ``env`` a keyword-only argument. + +Using an unpacked unbounded tuple is equivalent to the +:pep:`484#arbitrary-argument-lists-and-default-argument-values` +behavior of ``*args: int``, which accepts zero or +more values of type ``int``: + +:: + + def foo(*args: *Tuple[int, ...]) -> None: ... + + # equivalent to: + def foo(*args: int) -> None: ... + +Unpacking tuple types also allows more precise types for heterogeneous +``*args``. The following function expects an ``int`` at the beginning, +zero or more ``str`` values, and a ``str`` at the end: + +:: + + def foo(*args: *Tuple[int, *Tuple[str, ...], str]) -> None: ... + +For completeness, we mention that unpacking a concrete tuple allows us +to specify ``*args`` of a fixed number of heterogeneous types: + +:: + + def foo(*args: *Tuple[int, str]) -> None: ... + + foo(1, "hello") # OK Note that, in keeping with the rule that type variable tuples must always be used unpacked, annotating ``*args`` as being a plain type variable tuple @@ -457,17 +556,6 @@ all arguments must be a ``Tuple`` parameterised with the same types. foo((0,), (1, 2)) # Error foo((0,), ('1',)) # Error -Following `Type Variable Tuples Must Have Known Length`_, note -that the following should *not* type-check as valid (even though it is, of -course, valid at runtime): - -:: - - def foo(*args: *Ts): ... - - def bar(x: Tuple[int, ...]): - foo(*x) # NOT valid - Finally, note that a type variable tuple may *not* be used as the type of ``**kwargs``. (We do not yet know of a use case for this feature, so we prefer to leave the ground fresh for a potential future PEP.) @@ -488,12 +576,12 @@ Type variable tuples can also be used in the arguments section of a class Process: def __init__( self, - target: Callable[[*Ts], Any], - args: Tuple[*Ts] - ): ... + target: Callable[[*Ts], None], + args: Tuple[*Ts], + ) -> None: ... + + def func(arg1: int, arg2: str) -> None: ... - def func(arg1: int, arg2: str): ... - Process(target=func, args=(0, 'foo')) # Valid Process(target=func, args=('foo', 0)) # Error @@ -506,6 +594,63 @@ to the type variable tuple: def foo(f: Callable[[int, *Ts, T], Tuple[T, *Ts]]): ... +The behavior of a Callable containing an unpacked item, whether the +item is a ``TypeVarTuple`` or a tuple type, is to treat the elements +as if they were the type for ``*args``. So, ``Callable[[*Ts], None]`` +is treated as the type of the function: + +:: + + def foo(*args: *Ts) -> None: ... + +``Callable[[int, *Ts, T], Tuple[T, *Ts]]`` is treated as the type of +the function: + +:: + + def foo(*args: *Tuple[int, *Ts, T]) -> Tuple[T, *Ts]: ... + +Behaviour when Type Parameters are not Specified +------------------------------------------------ + +When a generic class parameterised by a type variable tuple is used without +any type parameters, it behaves as if the type variable tuple was +substituted with ``Tuple[Any, ...]``: + +:: + + def takes_any_array(arr: Array): ... + + # equivalent to: + def takes_any_array(arr: Array[*Tuple[Any, ...]]): ... + + x: Array[Height, Width] + takes_any_array(x) # Valid + y: Array[Time, Height, Width] + takes_any_array(y) # Also valid + +This enables gradual typing: existing functions accepting, for example, +a plain TensorFlow ``Tensor`` will still be valid even if ``Tensor`` is made +generic and calling code passes a ``Tensor[Height, Width]``. + +This also works in the opposite direction: + +:: + + def takes_specific_array(arr: Array[Height, Width]): ... + + z: Array + # equivalent to Array[*Tuple[Any, ...]] + + takes_specific_array(z) + +(For details, see the section on `Unpacking Unbounded Tuple Types`_.) + +This way, even if libraries are updated to use types like ``Array[Height, Width]``, +users of those libraries won't be forced to also apply type annotations to +all of their code; users still have a choice about what parts of their code +to type and which parts to not. + Aliases ------- @@ -547,8 +692,9 @@ tuple in the alias is set empty: IntTuple[()] # Equivalent to Tuple[int] NamedArray[()] # Equivalent to Tuple[str, Array[()]] -If the type parameter list is omitted entirely, the alias is -compatible with arbitrary type parameters: +If the type parameter list is omitted entirely, the unspecified type +variable tuples are treated as ``Tuple[Any, ...]`` (similar to +`Behaviour when Type Parameters are not Specified`_): :: @@ -573,9 +719,190 @@ Normal ``TypeVar`` instances can also be used in such aliases: Foo[str, int] # T bound to float, Ts to Tuple[()] Foo[float] - # T bound to Any, Ts to an arbitrary number of Any + # T bound to Any, Ts to an Tuple[Any, ...] Foo - + + +Substitution in Aliases +----------------------- + +In the previous section, we only discussed simple usage of generic aliases +in which the type arguments were just simple types. However, a number of +more exotic constructions are also possible. + + +Type Arguments can be Variadic +'''''''''''''''''''''''''''''' + +First, type arguments to generic aliases can be variadic. For example, a +``TypeVarTuple`` can be used as a type argument: + +:: + + Ts1 = TypeVar('Ts1') + Ts2 = TypeVar('Ts2') + + IntTuple = Tuple[int, *Ts1] + IntFloatTuple = IntTuple[float, *Ts2] # Valid + +Here, ``*Ts1`` in the ``IntTuple`` alias is bound to ``Tuple[float, *Ts2]``, +resulting in an alias ``IntFloatTuple`` equivalent to +``Tuple[int, float, *Ts2]``. + +Unpacked arbitrary-length tuples can also be used as type arguments, with +similar effects: + +:: + + IntFloatsTuple = IntTuple[*Tuple[float, ...]] # Valid + +Here, ``*Ts1`` is bound to ``*Tuple[float, ...]``, resulting in +``IntFloatsTuple`` being equivalent to ``Tuple[int, *Tuple[float, ...]]``: a tuple +consisting of an ``int`` then zero or more ``float``\s. + + +Variadic Arguments Require Variadic Aliases +''''''''''''''''''''''''''''''''''''''''''' + +Variadic type arguments can only be used with generic aliases that are +themselves variadic. For example: + +:: + + T = TypeVar('T') + + IntTuple = Tuple[int, T] + + IntTuple[str] # Valid + IntTuple[*Ts] # NOT valid + IntTuple[*Tuple[float, ...]] # NOT valid + +Here, ``IntTuple`` is a *non*-variadic generic alias that takes exactly one +type argument. Hence, it cannot accept ``*Ts`` or ``*Tuple[float, ...]`` as type +arguments, because they represent an arbitrary number of types. + + +Aliases with Both TypeVars and TypeVarTuples +'''''''''''''''''''''''''''''''''''''''''''' + +In `Aliases`_, we briefly mentioned that aliases can be generic in both +``TypeVar``\s and ``TypeVarTuple``\s: + +:: + + T = TypeVar('T') + Foo = Tuple[T, *Ts] + + Foo[str, int] # T bound to str, Ts to Tuple[int] + Foo[str, int, float] # T bound to str, Ts to Tuple[int, float] + +In accordance with `Multiple Type Variable Tuples: Not Allowed`_, at most one +``TypeVarTuple`` may appear in the type parameters to an alias. However, a +``TypeVarTuple`` can be combined with an arbitrary number of ``TypeVar``\s, +both before and after: + +:: + + T1 = TypeVar('T1') + T2 = TypeVar('T2') + T3 = TypeVar('T3') + + Tuple[*Ts, T1, T2] # Valid + Tuple[T1, T2, *Ts] # Valid + Tuple[T1, *Ts, T2, T3] # Valid + +In order to substitute these type variables with supplied type arguments, +any type variables at the beginning or end of the type parameter list first +consume type arguments, and then any remaining type arguments are bound +to the ``TypeVarTuple``: + +:: + + Shrubbery = Tuple[*Ts, T1, T2] + + Shrubbery[str, bool] # T2=bool, T1=str, Ts=Tuple[()] + Shrubbery[str, bool, float] # T2=float, T1=bool, Ts=Tuple[str] + Shrubbery[str, bool, float, int] # T2=int, T1=float, Ts=Tuple[str, bool] + + Ptang = Tuple[T1, *Ts, T2, T3] + + Ptang[str, bool, float] # T1=str, T3=float, T2=bool, Ts=Tuple[()] + Ptang[str, bool, float, int] # T1=str, T3=int, T2=float, Ts=Tuple[bool] + +Note that the minimum number of type arguments in such cases is set by +the number of ``TypeVar``\s: + +:: + + Shrubbery[int] # Not valid; Shrubbery needs at least two type arguments + + +Splitting Arbitrary-Length Tuples +''''''''''''''''''''''''''''''''' + +A final complication occurs when an unpacked arbitrary-length tuple is used +as a type argument to an alias consisting of both ``TypeVar``\s and a +``TypeVarTuple``: + +:: + + Elderberries = Tuple[*Ts, T1] + Hamster = Elderberries[*Tuple[int, ...]] # valid + +In such cases, the arbitrary-length tuple is split between the ``TypeVar``\s +and the ``TypeVarTuple``. We assume the arbitrary-length tuple contains +at least as many items as there are ``TypeVar``\s, such that individual +instances of the inner type - here ``int`` - are bound to any ``TypeVar``\s +present. The 'rest' of the arbitrary-length tuple - here ``*Tuple[int, ...]``, +since a tuple of arbitrary length minus two items is still arbitrary-length - +is bound to the ``TypeVarTuple``. + +Here, therefore, ``Hamster`` is equivalent to ``Tuple[*Tuple[int, ...], int]``: +a tuple consisting of zero or more ``int``\s, then a final ``int``. + +Of course, such splitting only occurs if necessary. For example, if we instead +did: + +:: + + Elderberries[*Tuple[int, ...], str] + +Then splitting would not occur; ``T1`` would be bound to ``str``, and +``Ts`` to ``*Tuple[int, ...]``. + +In particularly awkward cases, a ``TypeVarTuple`` may consume both a type +*and* a part of an arbitrary-length tuple type: + +:: + + Elderberries[str, *Tuple[int, ...]] + +Here, ``T1`` is bound to ``int``, and ``Ts`` is bound to +``Tuple[str, *Tuple[int, ...]]``. This expression is therefore equivalent to +``Tuple[str, *Tuple[int, ...], int]``: a tuple consisting of a ``str``, then +zero or more ``int``\s, ending with an ``int``. + + +TypeVarTuples Cannot be Split +''''''''''''''''''''''''''''' + +Finally, although any arbitrary-length tuples in the type argument list can be +split between the type variables and the type variable tuple, the same is not +true of ``TypeVarTuple``\s in the argument list: + +:: + + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + Camelot = Tuple[T, *Ts1] + Camelot[*Ts2] # NOT valid + +This is not possible because, unlike in the case of an unpacked arbitrary-length +tuple, there is no way to 'peer inside' the ``TypeVarTuple`` to see what its +individual types are. + + Overloads for Accessing Individual Types ---------------------------------------- @@ -654,8 +981,8 @@ otherwise imply. Also, we may later wish to support arguments that should not be We therefore settled on ``TypeVarTuple``. -Behaviour when Type Parameters are not Specified ------------------------------------------------- +Unspecified Type Parameters: Tuple vs TypeVarTuple +-------------------------------------------------- In order to support gradual typing, this PEP states that *both* of the following examples should type-check correctly: @@ -736,9 +1063,7 @@ shape properties of numerical computing programs. Grammar Changes =============== -This PEP requires two grammar changes. Full diffs of ``python.gram`` -and simple tests to confirm correct behaviour are available at -https://github.com/mrahtz/cpython/commits/pep646-grammar. +This PEP requires two grammar changes. Change 1: Star Expressions in Indexes ------------------------------------- @@ -756,7 +1081,7 @@ within square brackets), necessary to support star-unpacking of TypeVarTuples: Before: :: - + slices: | slice !',' | ','.slice+ [','] @@ -764,7 +1089,7 @@ Before: After: :: - + slices: | slice !',' | ','.(slice | starred_expression)+ [','] @@ -776,7 +1101,7 @@ passed to ``__getitem__``. For example, if we do ``foo[a, *b, c]``, and ``foo.__getitem__`` would receive ``(a, d, e, c)``. To put it another way, note that ``x[..., *a, ...]`` produces the same result -as ``x[(..., a*, ...)]``` (with any slices ``i:j`` in ``...`` replaced with +as ``x[(..., *a, ...)]`` (with any slices ``i:j`` in ``...`` replaced with ``slice(i, j)``, with the one edge case that ``x[*a]`` becomes ``x[(*a,)]``). TypeVarTuple Implementation @@ -792,7 +1117,7 @@ implementation. class TypeVarTuple: def __init__(self, name): self._name = name - self._unpacked = UnpackedTypeVarTuple(name) + self._unpacked = UnpackedTypeVarTuple(name) def __iter__(self): yield self._unpacked def __repr__(self): @@ -817,8 +1142,9 @@ as lists, within indexing operations: :: - idxs_to_select = (1, 2) - array[0, *idxs_to_select, -1] # Equivalent to [0, 1, 2, -1] + idxs = (1, 2) + array_slice = array[0, *idxs, -1] # Equivalent to [0, 1, 2, -1] + array[0, *idxs, -1] = array_slice # Also allowed Second, more than one instance of a star-unpack can occur within an index: @@ -880,21 +1206,58 @@ Where: star_annotation: ':' star_expression -This accomplishes the desired outcome (making ``*args: *Ts`` not be a syntax -error) while matching the behaviour of star-unpacking in other contexts: -at runtime, ``__iter__`` is called on the starred object, and a tuple -containing the items of the resulting iterator is set as the type annotion -for ``args``. In other words, at runtime ``*args: *foo`` is equivalent to -``*args: tuple(foo)``. +We also need to deal with the ``star_expression`` that results from this +construction. Normally, a ``star_expression`` occurs within the context +of e.g. a list, so a ``star_expression`` is handled by essentially +calling ``iter()`` on the starred object, and inserting the results +of the resulting iterator into the list at the appropriate place. For +``*args: *Ts``, however, we must process the ``star_expression`` in a +different way. + +We do this by instead making a special case for the ``star_expression`` +resulting from ``*args: *Ts``, emitting code equivalent to +``[annotation_value] = [*Ts]``. That is, we create an iterator from +``Ts`` by calling ``Ts.__iter__``, fetch a single value from the iterator, +verify that the iterator is exhausted, and set that value as the annotation +value. This results in the unpacked ``TypeVarTuple`` being set directly +as the runtime annotation for ``*args``: :: >>> Ts = TypeVarTuple('Ts') - >>> def foo(*args: *Ts): pass # Equivalent to `*args: tuple(Ts)` + >>> def foo(*args: *Ts): pass >>> foo.__annotations__ - {'args': (*Ts,)} + {'args': *Ts} # *Ts is the repr() of Ts._unpacked, an instance of UnpackedTypeVarTuple +This allows the runtime annotation to be consistent with an AST representation +that uses a ``Starred`` node for the annotations of ``args`` - in turn important +for tools that rely on the AST such as mypy to correctly recognise the construction: + +:: + + >>> print(ast.dump(ast.parse('def foo(*args: *Ts): pass'), indent=2)) + Module( + body=[ + FunctionDef( + name='foo', + args=arguments( + posonlyargs=[], + args=[], + vararg=arg( + arg='args', + annotation=Starred( + value=Name(id='Ts', ctx=Load()), + ctx=Load())), + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Pass()], + decorator_list=[])], + type_ignores=[]) + + Note that the only scenario in which this grammar change allows ``*Ts`` to be used as a direct annotation (rather than being wrapped in e.g. ``Tuple[*Ts]``) is ``*args``. Other uses are still invalid: @@ -910,14 +1273,18 @@ Implications As with the first grammar change, this change also has a number of side effects. In particular, the annotation of ``*args`` could be set to a starred object other than a ``TypeVarTuple`` - for example, the following nonsensical -annotation is possible: +annotations are possible: :: - >>> foo = [1, 2, 3] - >>> def bar(*args: *foo): pass # Equivalent to `*args: tuple(foo)` + >>> foo = [1] + >>> def bar(*args: *foo): pass >>> bar.__annotations__ - {'args': (1, 2, 3)} + {'args': 1} + + >>> foo = [1, 2] + >>> def bar(*args: *foo): pass + ValueError: too many values to unpack (expected 1) Again, prevention of such annotations will need to be done by, say, static checkers, rather than at the level of syntax. @@ -945,7 +1312,7 @@ reasons: the user is familiar with star-unpacking in other contexts; if the user is reading or writing code that uses variadic generics, this seems reasonable.) - + If even change 1 is thought too significant a change, therefore, it might be better for us to reconsider our options before going ahead with this second alternative. @@ -971,7 +1338,7 @@ one in Pyre, as of v0.9.0, and one in Pyright, as of v1.1.108. A preliminary implementation of the ``Unpack`` version of the PEP in CPython is available in `cpython/23527`_. A preliminary version of the version -using the star operator, based on an early implementation of PEP 637, +using the star operator, based on an early implementation of :pep:`637`, is also available at `mrahtz/cpython/pep637+646`_. Appendix A: Shape Typing Use Cases @@ -994,13 +1361,13 @@ We can attach names to each parameter using normal type variables: K = TypeVar('K') N = TypeVar('N') - def matrix_vector_multiply(x: Array[K, N], Array[N]) -> Array[K]: ... + def matrix_vector_multiply(x: Array[K, N], y: Array[N]) -> Array[K]: ... a: Array[Literal[64], Literal[32]] b: Array[Literal[32]] matrix_vector_multiply(a, b) # Result is Array[Literal[64]] - + Note that such names have a purely local scope. That is, the name ``K`` is bound to ``Literal[64]`` only within ``matrix_vector_multiply``. To put it another way, there's no relationship between the value of ``K`` in different @@ -1022,7 +1389,7 @@ type operators that enable arithmetic on array shapes - for example: :: def repeat_each_element(x: Array[N]) -> Array[Mul[2, N]]: ... - + Such arithmetic type operators would only make sense if names such as ``N`` refer to axis size. Use Case 2: Specifying Shape Semantics @@ -1037,16 +1404,16 @@ For example: :: # lib.py - + class Batch: pass class Time: pass - + def make_array() -> Array[Batch, Time]: ... - + # user.py - + from lib import Batch, Time - + # `Batch` and `Time` have the same identity as in `lib`, # so must take array as produced by `lib.make_array` def use_array(x: Array[Batch, Time]): ... @@ -1069,18 +1436,20 @@ without knowing the type ahead of time. For example, we can still write: K = TypeVar('K') N = TypeVar('N') - def matrix_vector_multiply(x: Array[K, N], Array[N]) -> Array[K]: ... - + def matrix_vector_multiply(x: Array[K, N], y: Array[N]) -> Array[K]: ... + We can then use this with: - + +:: + class Batch: pass class Values: pass - + batch_of_values: Array[Batch, Values] value_weights: Array[Values] matrix_vector_multiply(batch_of_values, value_weights) # Result is Array[Batch] - + The disadvantages are the inverse of the advantages from use case 1. In particular, this approach does not lend itself well to arithmetic on axis types: ``Mul[2, Batch]`` would be as meaningless as ``2 * int``. @@ -1103,7 +1472,7 @@ Consider the following 'normal' code: :: def f(x: int): ... - + Note that we have symbols for both the value of the thing (``x``) and the type of the thing (``int``). Why can't we do the same with axes? For example, with an imaginary syntax, we could write: @@ -1111,7 +1480,7 @@ syntax, we could write: :: def f(array: Array[TimeValue: TimeType]): ... - + This would allow us to access the axis size (say, 32) through the symbol ``TimeValue`` *and* the type through the symbol ``TypeType``. @@ -1120,7 +1489,7 @@ This might even be possible using existing syntax, through a second level of par :: def f(array: array[TimeValue[TimeType]]): .. - + However, we leave exploration of this approach to the future. Appendix B: Shaped Types vs Named Axes @@ -1166,7 +1535,7 @@ this PEP? We're not sure. One area of overlap is that in some contexts, we could mean(im, axis=Image.axes.index(Channels) Ideally, we might write something like ``im: Array[Height=64, Width=64, Channels=3]`` - -but this won't be possible in the short term, due to the rejection of PEP 637. +but this won't be possible in the short term, due to the rejection of :pep:`637`. In any case, our attitude towards this is mostly "Wait and see what happens before taking any further steps". @@ -1198,7 +1567,7 @@ From **Stephan Hoyer**, member of the NumPy Steering Council: [#stephan-endorsement]_ I just wanted to thank Matthew & Pradeep for writing this PEP and for - clarifications to the broader context of PEP 646 for array typing in + clarifications to the broader context of :pep:`646` for array typing in https://github.com/python/peps/pull/1904. As someone who is heavily involved in the Python numerical computing @@ -1220,14 +1589,14 @@ From **Bas van Beek**, who has worked on preliminary support for shape-generics in NumPy: I very much share Stephan's opinion here and look forward to integrating the - new PEP 646 variadics into numpy. + new :pep:`646` variadics into numpy. In the context of numpy (and tensor typing general): the typing of array shapes is a fairly complicated subject and the introduction of variadics will likely play in big role in laying its foundation, as it allows for the expression of both dimensioability as well as basic shape manipulation. - All in all, I'm very interested in where both PEP 646 and future PEPs will + All in all, I'm very interested in where both :pep:`646` and future PEPs will take us and look forward to further developments. From **Dan Moldovan**, a Senior Software Engineer on the TensorFlow Dev Team @@ -1271,6 +1640,9 @@ Expanding on these ideas, **Mark Mendoza** and **Vincent Siles** gave a presenta 'Variadic Type Variables for Decorators and Tensors' [#variadic-type-variables]_ at the 2019 Python Typing Summit. +Discussion over how type substitution in generic aliases should behave +took place in `cpython#91162`_. + References ========== @@ -1314,6 +1686,8 @@ References .. [#dan-endorsement] https://mail.python.org/archives/list/python-dev@python.org/message/HTCARTYYCHETAMHB6OVRNR5EW5T2CP4J/ +.. _cpython#91162: https://github.com/python/cpython/issues/91162 + Copyright ========= diff --git a/pep-0647.rst b/peps/pep-0647.rst similarity index 99% rename from pep-0647.rst rename to peps/pep-0647.rst index b25fd3c63..593372ab3 100644 --- a/pep-0647.rst +++ b/peps/pep-0647.rst @@ -4,13 +4,14 @@ Version: $Revision$ Last-Modified: $Date$ Author: Eric Traut Sponsor: Guido van Rossum -Discussions-To: Typing-Sig +Discussions-To: typing-sig@python.org Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 07-Oct-2020 Python-Version: 3.10 -Post-History: 28-Dec-2020, 9-Apr-2021 +Post-History: 28-Dec-2020, 09-Apr-2021 Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/2ME6F6YUVKHOQYKSHTVQQU5WD4CVAZU4/ @@ -336,7 +337,7 @@ decided unnecessary to burden the Python implementation of user-defined type guards with additional complexity to support a contrived use case. If such use cases are identified in the future, there are ways the TypeGuard mechanism could be extended. This could involve the use of keyword indexing, as proposed -in PEP 637. +in :pep:`637`. Narrowing of Implicit "self" and "cls" Parameters diff --git a/pep-0648.rst b/peps/pep-0648.rst similarity index 98% rename from pep-0648.rst rename to peps/pep-0648.rst index a355d99ec..05b045aba 100644 --- a/pep-0648.rst +++ b/peps/pep-0648.rst @@ -2,14 +2,13 @@ PEP: 648 Title: Extensible customizations of the interpreter at startup Author: Mario Corchero Sponsor: Pablo Galindo -BDFL-Delegate: XXXX Discussions-To: https://discuss.python.org/t/pep-648-extensible-customizations-of-the-interpreter-at-startup/6403 -Status: Draft +Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 30-Dec-2020 Python-Version: 3.11 -Post-History: python-ideas: 16th Dec. python-dev: 18th Dec. +Post-History: 16-Dec-2020, 18-Dec-2020 Abstract ======== @@ -17,6 +16,13 @@ Abstract This PEP proposes supporting extensible customization of the interpreter by allowing users to install files that will be executed at startup. +PEP Rejection +============= + +PEP 648 was rejected `by the steering council +`__ +as it has a limited number of use cases and further complicates the startup sequence. + Motivation ========== diff --git a/peps/pep-0649.rst b/peps/pep-0649.rst new file mode 100644 index 000000000..e91eb99ff --- /dev/null +++ b/peps/pep-0649.rst @@ -0,0 +1,1480 @@ +PEP: 649 +Title: Deferred Evaluation Of Annotations Using Descriptors +Author: Larry Hastings +Discussions-To: https://discuss.python.org/t/pep-649-deferred-evaluation-of-annotations-tentatively-accepted/21331/ +Status: Accepted +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 11-Jan-2021 +Python-Version: 3.13 +Post-History: `11-Jan-2021 `__, + `12-Apr-2021 `__, + `18-Apr-2021 `__, + `09-Aug-2021 `__, + `20-Oct-2021 `__, + `20-Oct-2021 `__, + `17-Nov-2021 `__, + `15-Mar-2022 `__, + `23-Nov-2022 `__, + `07-Feb-2023 `__, + `11-Apr-2023 `__, +Resolution: https://discuss.python.org/t/pep-649-deferred-evaluation-of-annotations-tentatively-accepted/21331/43 + +******** +Abstract +******** + +Annotations are a Python technology that allows expressing +type information and other metadata about Python functions, +classes, and modules. But Python's original semantics +for annotations required them to be eagerly evaluated, +at the time the annotated object was bound. This caused +chronic problems for static type analysis users using +"type hints", due to forward-reference and circular-reference +problems. + +Python solved this by accepting :pep:`563`, incorporating +a new approach called "stringized annotations" in which +annotations were automatically converted into strings by +Python. This solved the forward-reference and circular-reference +problems, and also fostered intriguing new uses for annotation +metadata. But stringized annotations in turn caused chronic +problems for runtime users of annotations. + +This PEP proposes a new and comprehensive third approach +for representing and computing annotations. It adds a new +internal mechanism for lazily computing annotations on demand, +via a new object method called ``__annotate__``. +This approach, when combined with a novel technique for +coercing annotation values into alternative formats, solves +all the above problems, supports all existing use cases, +and should foster future innovations in annotations. + + +******** +Overview +******** + +This PEP adds a new dunder attribute to the objects that +support annotations--functions, classes, and modules. +The new attribute is called ``__annotate__``, and is +a reference to a function which computes and returns +that object's annotations dict. + +At compile time, if the definition of an object includes +annotations, the Python compiler will write the expressions +computing the annotations into its own function. When run, +the function will return the annotations dict. The Python +compiler then stores a reference to this function in +``__annotate__`` on the object. + +Furthermore, ``__annotations__`` is redefined to be a +"data descriptor" which calls this annotation function once +and caches the result. + +This mechanism delays the evaluation of annotations expressions +until the annotations are examined, which solves many circular +reference problems. + +This PEP also defines new functionality for two functions +in the Python standard library: +``inspect.get_annotations`` and ``typing.get_type_hints``. +The functionality is accessed via a new keyword-only parameter, +``format``. ``format`` allows the user to request +the annotations from these functions +in a specific format. +Format identifiers are always predefined integer values. +The formats defined by this PEP are: + + +* ``inspect.VALUE = 1`` + + The default value. + The function will return the conventional Python + values for the annotations. This format is identical + to the return value for these functions under Python 3.11. + +* ``inspect.FORWARDREF = 2`` + + The function will attempt to return the conventional + Python values for the annotations. However, if it + encounters an undefined name, or a free variable that + has not yet been associated with a value, it dynamically + creates a proxy object (a ``ForwardRef``) that substitutes + for that value in the expression, then continues evaluation. + The resulting dict may contain a mixture of proxies and + real values. If all real values are defined at the time + the function is called, ``inspect.FORWARDREF`` and + ``inspect.VALUE`` produce identical results. + +* ``inspect.SOURCE = 3`` + + The function will produce an annotation dictionary + where the values have been replaced by strings containing + the original source code for the annotation expressions. + These strings may only be approximate, as they may be + reverse-engineered from another format, rather than + preserving the original source code, but the differences + will be minor. + +If accepted, this PEP would *supersede* :pep:`563`, +and :pep:`563`'s behavior would be deprecated and +eventually removed. + + +Comparison Of Annotation Semantics +================================== + +.. note:: The code presented in this section is simplified + for clarity, and is intentionally inaccurate in some + critical aspects. This example is intended merely to + communicate the high-level concepts involved without + getting lost in the details. But readers should note + that the actual implementation is quite different in + several important ways. See the Implementation_ + section later in this PEP for a far more accurate + description of what this PEP proposes from a technical + level. + +Consider this example code: + +.. code-block:: + + def foo(x: int = 3, y: MyType = None) -> float: + ... + class MyType: + ... + foo_y_annotation = foo.__annotations__['y'] + +As we see here, annotations are available at runtime through an +``__annotations__`` attribute on functions, classes, and modules. +When annotations are specified on one of these objects, +``__annotations__`` is a dictionary mapping the names of the +fields to the value specified as that field's annotation. + +The default behavior in Python is to evaluate the expressions +for the annotations, and build the annotations dict, at the time +the function, class, or module is bound. At runtime the above +code actually works something like this: + +.. code-block:: + + annotations = {'x': int, 'y': MyType, 'return': float} + def foo(x = 3, y = "abc"): + ... + foo.__annotations__ = annotations + class MyType: + ... + foo_y_annotation = foo.__annotations__['y'] + +The crucial detail here is that the values ``int``, ``MyType``, +and ``float`` are looked up at the time the function object is +bound, and these values are stored in the annotations dict. +But this code doesn't run—it throws a ``NameError`` on the first +line, because ``MyType`` hasn't been defined yet. + +:pep:`563`'s solution is to decompile the expressions back +into strings during compilation and store those strings as the +values in the annotations dict. The equivalent runtime code +would look something like this: + +.. code-block:: + + annotations = {'x': 'int', 'y': 'MyType', 'return': 'float'} + def foo(x = 3, y = "abc"): + ... + foo.__annotations__ = annotations + class MyType: + ... + foo_y_annotation = foo.__annotations__['y'] + +This code now runs successfully. However, ``foo_y_annotation`` +is no longer a reference to ``MyType``, it is the *string* +``'MyType'``. To turn the string into the real value ``MyType``, +the user would need to evaluate the string using ``eval``, +``inspect.get_annotations``, or ``typing.get_type_hints``. + +This PEP proposes a third approach, delaying the evaluation of +the annotations by computing them in their own function. If +this PEP was active, the generated code would work something +like this: + +.. code-block:: + + class function: + # __annotations__ on a function object is already a + # "data descriptor" in Python, we're just changing + # what it does + @property + def __annotations__(self): + return self.__annotate__() + + # ... + + def annotate_foo(): + return {'x': int, 'y': MyType, 'return': float} + def foo(x = 3, y = "abc"): + ... + foo.__annotate__ = annotate_foo + class MyType: + ... + foo_y_annotation = foo.__annotations__['y'] + +The important change is that the code constructing the +annotations dict now lives in a function—here, called +``annotate_foo()``. But this function isn't called +until we ask for the value of ``foo.__annotations__``, +and we don't do that until *after* the definition of ``MyType``. +So this code also runs successfully, and ``foo_y_annotation`` now +has the correct value--the class ``MyType``--even though +``MyType`` wasn't defined until *after* the annotation was +defined. + + +Mistaken Rejection Of This Approach In November 2017 +==================================================== + +During the early days of discussion around :pep:`563`, +in a November 2017 thread in ``comp.lang.python-dev``, +the idea of using code to delay the evaluation of +annotations was briefly discussed. At the time the +technique was termed an "implicit lambda expression". + +Guido van Rossum—Python's BDFL at the time—replied, +asserting that these "implicit lambda expression" wouldn't +work, because they'd only be able to resolve symbols at +module-level scope: + + IMO the inability of referencing class-level definitions + from annotations on methods pretty much kills this idea. + +https://mail.python.org/pipermail/python-dev/2017-November/150109.html + +This led to a short discussion about extending lambda-ized +annotations for methods to be able to refer to class-level +definitions, by maintaining a reference to the class-level +scope. This idea, too, was quickly rejected. + +:pep:`PEP 563 summarizes the above discussion +<563#keeping-the-ability-to-use-function-local-state-when-defining-annotations>` + +The approach taken by this PEP doesn't suffer from these +restrictions. Annotations can access module-level definitions, +class-level definitions, and even local and free variables. + + +********** +Motivation +********** + +A History Of Annotations +======================== + +Python 3.0 shipped with a new syntax feature, "annotations", +defined in :pep:`3107`. +This allowed specifying a Python value that would be +associated with a parameter of a Python function, or +with the value that function returns. +Said another way, annotations gave Python users an interface +to provide rich metadata about a function parameter or return +value, for example type information. +All the annotations for a function were stored together in +a new attribute ``__annotations__``, in an "annotation dict" +that mapped parameter names (or, in the case of the return +annotation, using the name ``'return'``) to their Python value. + +In an effort to foster experimentation, Python +intentionally didn't define what form this metadata should take, +or what values should be used. User code began experimenting with +this new facility almost immediately. But popular libraries that +make use of this functionality were slow to emerge. + +After years of little progress, the BDFL chose a particular +approach for expressing static type information, called +*type hints,* as defined in :pep:`484`. Python 3.5 shipped +with a new :mod:`typing` module which quickly became very popular. + +Python 3.6 added syntax to annotate local variables, +class attributes, and module attributes, using the approach +proposed in :pep:`526`. Static type analysis continued to +grow in popularity. + +However, static type analysis users were increasingly frustrated +by an inconvenient problem: forward references. In classic +Python, if a class C depends on a later-defined class D, +it's normally not a problem, because user code will usually +wait until both are defined before trying to use either. +But annotations added a new complication, because they were +computed at the time the annotated object (function, class, +or module) was bound. If methods on class C are annotated with +type D, and these annotation expressions are computed at the +time that the method is bound, D may not be defined yet. +And if methods in D are also annotated with type C, you now +have an unresolvable circular reference problem. + +Initially, static type users worked around this problem +by defining their problematic annotations as strings. +This worked because a string containing the type hint was +just as usable for the static type analysis tool. +And users of static type analysis tools rarely examine the +annotations at runtime, so this representation wasn't +itself an inconvenience. But manually stringizing type +hints was clumsy and error-prone. Also, code bases were +adding more and more annotations, which consumed more and +more CPU time to create and bind. + +To solve these problems, the BDFL accepted :pep:`563`, which +added a new feature to Python 3.7: "stringized annotations". +It was activated with a future import:: + + from __future__ import annotations + +Normally, annotation expressions were evaluated at the time +the object was bound, with their values being stored in the +annotations dict. When stringized annotations were active, +these semantics changed: instead, at compile time, the compiler +converted all annotations in that module into string +representations of their source code--thus, *automatically* +turning the users's annotations into strings, obviating the +need to *manually* stringize them as before. :pep:`563` +suggested users could evaluate this string with ``eval`` +if the actual value was needed at runtime. + +(From here on out, this PEP will refer to the classic +semantics of :pep:`3107` and :pep:`526`, where the +values of annotation expressions are computed at the time +the object is bound, as *"stock" semantics,* to differentiate +them from the new :pep:`563` "stringized" annotation semantics.) + +The Current State Of Annotation Use Cases +========================================= + +Although there are many specific use cases for annotations, +annotation users in the discussion around this PEP tended +to fall into one of these four categories. + + +Static typing users +------------------- + +Static typing users use annotations to add type information +to their code. But they largely don't examine the annotations +at runtime. Instead, they use static type analysis tools +(mypy, pytype) to examine their source tree and determine +whether or not their code is using types consistently. +This is almost certainly the most popular use case for +annotations today. + +Many of the annotations use *type hints,* a la :pep:`484` +(and many subsequent PEPs). Type hints are passive objects, +mere representation of type information; they don't do any actual work. +Type hints are often parameterized with other types or other type hints. +Since they're agnostic about what these actual values are, type hints +work fine with ``ForwardRef`` proxy objects. +Users of static type hints discovered that extensive type hinting under +stock semantics often created large-scale circular reference and circular +import problems that could be difficult to solve. :pep:`563` was designed +specifically to solve this problem, and the solution worked great for +these users. The difficulty of rendering stringized annotations into +real values largely didn't inconvenience these users because of how +infrequently they examine annotations at runtime. + +Static typing users often combine :pep:`563` with the +``if typing.TYPE_CHECKING`` idiom to prevent their type hints from being +loaded at runtime. This means they often aren't able to evaluate their +stringized annotations and produce real values at runtime. On the rare +occasion that they do examine annotations at runtime, they often forgo +``eval``, instead using lexical analysis directly on the stringized +annotations. + +Under this PEP, static typing users will probably prefer ``FORWARDREF`` +or ``SOURCE`` format. + + +Runtime annotation users +------------------------ + +Runtime annotation users use annotations as a means of expressing rich +metadata about their functions and classes, which they use as input to +runtime behavior. Specific use cases include runtime type verification +(Pydantic) and glue logic to expose Python APIs in another domain +(FastAPI, Typer). The annotations may or may not be type hints. + +As runtime annotation users examine annotations at runtime, they were +traditionally better served with stock semantics. This use case is +largely incompatible with :pep:`563`, particularly with the +``if typing.TYPE_CHECKING`` idiom. + +Under this PEP, runtime annotation users will most likely prefer ``VALUE`` +format, though some (e.g. if they evaluate annotations eagerly in a decorator +and want to support forward references) may also use ``FORWARDREF`` format. + + +Wrappers +-------- + +Wrappers are functions or classes that wrap user functions or +classes and add functionality. Examples of this would be +:func:`~dataclasses.dataclass`, :func:`functools.partial`, +``attrs``, and ``wrapt``. + +Wrappers are a distinct subcategory of runtime annotation users. +Although they do use annotations at runtime, they may or may not +actually examine the annotations of the objects they wrap--it depends +on the functionality the wrapper provides. As a rule they should +propagate the annotations of the wrapped object to the wrapper +they create, although it's possible they may modify those annotations. + +Wrappers were generally designed to work well under stock semantics. +Whether or not they work well under :pep:`563` semantics depends on the +degree to which they examine the wrapped object's annotations. +Often wrappers don't care about the value per se, only needing +specific information about the annotations. Even so, :pep:`563` +and the ``if typing.TYPE_CHECKING`` idiom can make it difficult +for wrappers to reliably determine the information they need at +runtime. This is an ongoing, chronic problem. +Under this PEP, wrappers will probably prefer ``FORWARDREF`` format +for their internal logic. But the wrapped objects need to support +all formats for their users. + + +Documentation +------------- + +:pep:`563` stringized annotations were a boon for tools that +mechanically construct documentation. + +Stringized type hints make for excellent documentation; type hints +as expressed in source code are often succinct and readable. However, +at runtime these same type hints can produce value at runtime whose repr +is a sprawling, nested, unreadable mess. Thus documentation users were +well-served by :pep:`563` but poorly served with stock semantics. + +Under this PEP, documentation users are expected to use ``SOURCE`` format. + + +Motivation For This PEP +======================= + +Python's original semantics for annotations made its use for +static type analysis painful due to forward reference problems. +:pep:`563` solved the forward reference problem, and many +static type analysis users became happy early adopters of it. +But its unconventional solution created new problems for two +of the above cited use cases: runtime annotation users, +and wrappers. + +First, stringized annotations didn't permit referencing local or +free variables, which meant many useful, reasonable approaches +to creating annotations were no longer viable. This was +particularly inconvenient for decorators that wrap existing +functions and classes, as these decorators often use closures. + +Second, in order for ``eval`` to correctly look up globals in a +stringized annotation, you must first obtain a reference +to the correct module. +But class objects don't retain a reference to their globals. +:pep:`563` suggests looking up a class's module by name in +``sys.modules``—a surprising requirement for a language-level +feature. + +Additionally, complex but legitimate constructions can make it +difficult to determine the correct globals and locals dicts to +give to ``eval`` to properly evaluate a stringized annotation. +Even worse, in some situations it may simply be infeasible. + +For example, some libraries (e.g. ``typing.TypedDict``, :mod:`dataclasses`) +wrap a user class, then merge all the annotations from all that +class's base classes together into one cumulative annotations dict. +If those annotations were stringized, calling ``eval`` on them later +may not work properly, because the globals dictionary used for the +``eval`` will be the module where the *user class* was defined, +which may not be the same module where the *annotation* was +defined. However, if the annotations were stringized because +of forward-reference problems, calling ``eval`` on them early +may not work either, due to the forward reference not being +resolvable yet. This has proved to be difficult to reconcile; +of the three bug reports linked to below, only one has been +marked as fixed. + +* https://github.com/python/cpython/issues/89687 +* https://github.com/python/cpython/issues/85421 +* https://github.com/python/cpython/issues/90531 + +Even with proper globals *and* locals, ``eval`` can be unreliable +on stringized annotations. +``eval`` can only succeed if all the symbols referenced in +an annotations are defined. If a stringized annotation refers +to a mixture of defined and undefined symbols, a simple ``eval`` +of that string will fail. This is a problem for libraries with +that need to examine the annotation, because they can't reliably +convert these stringized annotations into real values. + +* Some libraries (e.g. :mod:`dataclasses`) solved this by foregoing real + values and performing lexical analysis of the stringized annotation, + which requires a lot of work to get right. + +* Other libraries still suffer with this problem, + which can produce surprising runtime behavior. + https://github.com/python/cpython/issues/97727 + +Also, ``eval()`` is slow, and it isn't always available; it's +sometimes removed for space reasons on certain platforms. +``eval()`` on MicroPython doesn't support the ``locals`` +argument, which makes converting stringized annotations +into real values at runtime even harder. + +Finally, :pep:`563` requires Python implementations to +stringize their annotations. This is surprising behavior—unprecedented +for a language-level feature, with a complicated implementation, +that must be updated whenever a new operator is added to the +language. + +These problems motivated the research into finding a new +approach to solve the problems facing annotations users, +resulting in this PEP. + + +************** +Implementation +************** + +Observed semantics for annotations expressions +============================================== + +For any object ``o`` that supports annotations, +provided that all names evaluated in the annotations expressions +are bound before ``o`` is defined and never subsequently rebound, +``o.__annotations__`` will produce an identical annotations dict both +when "stock" semantics are active and when this PEP is active. +In particular, name resolution will be performed identically in +both scenarios. + +When this PEP is active, the value of ``o.__annotations__`` +won't be calculated until the first time ``o.__annotations__`` +itself is evaluated. All evaluation of the annotation expressions +is delayed until this moment, which also means that + +* names referenced in the annotations expressions will use their + *current* value at this moment, and +* if evaluating the annotations expressions raises an exception, + that exception will be raised at this moment. + +Once ``o.__annotations__`` is successfully calculated for the +first time, this value is cached and will be returned by future +requests for ``o.__annotations__``. + +__annotate__ and __annotations__ +================================ + +Python supports annotations on three different types: +functions, classes, and modules. This PEP modifies +the semantics on all three of these types in a similar +way. + +First, this PEP adds a new "dunder" attribute, ``__annotate__``. +``__annotate__`` must be a "data descriptor", +implementing all three actions: get, set, and delete. +The ``__annotate__`` attribute is always defined, +and may only be set to either ``None`` or to a callable. +(``__annotate__`` cannot be deleted.) If an object +has no annotations, ``__annotate__`` should be +initialized to ``None``, rather than to a function +that returns an empty dict. + +The ``__annotate__`` data descriptor must have dedicated +storage inside the object to store the reference to its value. +The location of this storage at runtime is an implementation +detail. Even if it's visible to Python code, it should still +be considered an internal implementation detail, and Python +code should prefer to interact with it only via the +``__annotate__`` attribute. + +The callable stored in ``__annotate__`` must accept a +single required positional argument called ``format``, +which will always be an ``int`` (or a subclass of ``int``). +It must either return a dict (or subclass of dict) or +raise ``NotImplementedError()``. + +Here's a formal definition of ``__annotate__``, as it will +appear in the "Magic methods" section of the Python +Language Reference: + + ``__annotate__(format: int) -> dict`` + + Returns a new dictionary object mapping attribute/parameter + names to their annotation values. + + Takes a ``format`` parameter specifying the format in which + annotations values should be provided. Must be one of the + following: + + ``inspect.VALUE`` (equivalent to the ``int`` constant ``1``) + + Values are the result of evaluating the annotation expressions. + + ``inspect.FORWARDREF`` (equivalent to the ``int`` constant ``2``) + + Values are real annotation values (as per ``inspect.VALUE`` format) + for defined values, and ``ForwardRef`` proxies for undefined values. + Real objects may be exposed to, or contain references to, + ``ForwardRef`` proxy objects. + + ``inspect.SOURCE`` (equivalent to the ``int`` constant ``3``) + + Values are the text string of the annotation as it + appears in the source code. May only be approximate; + whitespace may be normalized, and constant values may + be optimized. It's possible the exact values of these + strings could change in future version of Python. + + If an ``__annotate__`` function doesn't support the requested + format, it must raise ``NotImplementedError()``. + ``__annotate__`` functions must always support ``1`` (``inspect.VALUE``) + format; they must not raise ``NotImplementedError()`` when called with + ``format=1``. + + When called with ``format=1``, an ``__annotate__`` function + may raise ``NameError``; it must not raise ``NameError`` when called + requesting any other format. + + If an object doesn't have any annotations, ``__annotate__`` should + preferably be set to ``None`` (it can't be deleted), rather than set to a + function that returns an empty dict. + +When the Python compiler compiles an object with +annotations, it simultaneously compiles the appropriate +annotate function. This function, called with +the single positional argument ``inspect.VALUE``, +computes and returns the annotations dict as defined +on that object. The Python compiler and runtime work +in concert to ensure that the function is bound to +the appropriate namespaces: + +* For functions and classes, the globals dictionary will + be the module where the object was defined. If the object + is itself a module, its globals dictionary will be its + own dict. +* For methods on classes, and for classes, the locals dictionary + will be the class dictionary. +* If the annotations refer to free variables, the closure will + be the appropriate closure tuple containing cells for free variables. + +Second, this PEP requires that the existing +``__annotations__`` must be a "data descriptor", +implementing all three actions: get, set, and delete. +``__annotations__`` must also have its own internal +storage it uses to cache a reference to the annotations dict: + +* Class and module objects must + cache the annotations dict in their ``__dict__``, using the key + ``__annotations__``. This is required for backwards + compatibility reasons. +* For function objects, storage for the annotations dict + cache is an implementation detail. It's preferably internal + to the function object and not visible in Python. + +This PEP defines semantics on how ``__annotations__`` and +``__annotate__`` interact, for all three types that implement them. +In the following examples, ``fn`` represents a function, ``cls`` +represents a class, ``mod`` represents a module, and ``o`` represents +an object of any of these three types: + +* When ``o.__annotations__`` is evaluated, and the internal storage + for ``o.__annotations__`` is unset, and ``o.__annotate__`` is set + to a callable, the getter for ``o.__annotations__`` calls + ``o.__annotate__(1)``, then caches the result in its internal + storage and returns the result. + + - To explicitly clarify one question that has come up multiple times: + this ``o.__annotations__`` cache is the *only* caching mechanism + defined in this PEP. There are *no other* caching mechanisms defined + in this PEP. The ``__annotate__`` functions generated by the Python + compiler explicitly don't cache any of the values they compute. + +* Setting ``o.__annotate__`` to a callable invalidates the + cached annotations dict. + +* Setting ``o.__annotate__`` to ``None`` has no effect on + the cached annotations dict. + +* Deleting ``o.__annotate__`` raises ``TypeError``. + ``__annotate__`` must always be set; this prevents unannotated + subclasses from inheriting the ``__annotate__`` method of one + of their base classes. + +* Setting ``o.__annotations__`` to a legal value + automatically sets ``o.__annotate__`` to ``None``. + + * Setting ``cls.__annotations__`` or ``mod.__annotations__`` + to ``None`` otherwise works like any other attribute; the + attribute is set to ``None``. + + * Setting ``fn.__annotations__`` to ``None`` invalidates + the cached annotations dict. If ``fn.__annotations__`` + doesn't have a cached annotations value, and ``fn.__annotate__`` + is ``None``, the ``fn.__annotations__`` data descriptor + creates, caches, and returns a new empty dict. (This is for + backwards compatibility with :pep:`3107` semantics.) + +Changes to allowable annotations syntax +======================================= + +``__annotate__`` now delays the evaluation of annotations until +``__annotations__`` is referenced in the future. It also means +annotations are evaluated in a new function, rather than in the +original context where the object they were defined on was bound. +There are four operators with significant runtime side-effects +that were permitted in stock semantics, but are disallowed when +``from __future__ import annotations`` is active, and will have +to be disallowed when this PEP is active: + +* ``:=`` +* ``yield`` +* ``yield from`` +* ``await`` + +Changes to ``inspect.get_annotations`` and ``typing.get_type_hints`` +==================================================================== + +(This PEP makes frequent reference to these two functions. In the future +it will refer to them collectively as "the helper functions", as they help +user code work with annotations.) + +These two functions extract and return the annotations from an object. +``inspect.get_annotations`` returns the annotations unchanged; +for the convenience of static typing users, ``typing.get_type_hints`` +makes some modifications to the annotations before it returns them. + +This PEP adds a new keyword-only parameter to these two functions, +``format``. ``format`` specifies what format the values in the +annotations dict should be returned in. +The ``format`` parameter on these two functions accepts the same values +as the ``format`` parameter on the ``__annotate__`` magic method +defined above; however, these ``format`` parameters also have a default +value of ``inspect.VALUE``. + +When either ``__annotations__`` or ``__annotate__`` is updated on an +object, the other of those two attributes is now out-of-date and should also +either be updated or deleted (set to ``None``, in the case of ``__annotate__`` +which cannot be deleted). In general, the semantics established in the previous +section ensure that this happens automatically. However, there's one case which +for all practical purposes can't be handled automatically: when the dict cached +by ``o.__annotations__`` is itself modified, or when mutable values inside that +dict are modified. + +Since this can't be handled in code, it must be handled in +documentation. This PEP proposes amending the documentation +for ``inspect.get_annotations`` (and similarly for +``typing.get_type_hints``) as follows: + + If you directly modify the ``__annotations__`` dict on an object, + by default these changes may not be reflected in the dictionary + returned by ``inspect.get_annotations`` when requesting either + ``SOURCE`` or ``FORWARDREF`` format on that object. Rather than + modifying the ``__annotations__`` dict directly, consider replacing + that object's ``__annotate__`` method with a function computing + the annotations dict with your desired values. Failing that, it's + best to overwrite the object's ``__annotate__`` method with ``None`` + to prevent ``inspect.get_annotations`` from generating stale results + for ``SOURCE`` and ``FORWARDREF`` formats. + + + +The ``stringizer`` and the ``fake globals`` environment +======================================================= + +As originally proposed, this PEP supported many runtime +annotation user use cases, and many static type user use cases. +But this was insufficient--this PEP could not be accepted +until it satisfied *all* extant use cases. This became +a longtime blocker of this PEP until Carl Meyer proposed +the "stringizer" and the "fake globals" environment as +described below. These techniques allow this PEP to support +both the ``FORWARDREF`` and ``SOURCE`` formats, ably +satisfying all remaining uses cases. + +In a nutshell, this technique involves running a +Python-compiler-generated ``__annotate__`` function in +an exotic runtime environment. Its normal ``globals`` +dict is replaced with what's called a "fake globals" dict. +A "fake globals" dict is a dict with one important difference: +every time you "get" a key from it that isn't mapped, +it creates, caches, and returns a new value for that key +(as per the ``__missing__`` callback for a dictionary). +That value is a an instance of a novel type referred to +as a "stringizer". + +A "stringizer" is a Python class with highly unusual behavior. +Every stringizer is initialized with its "value", initially +the name of the missing key in the "fake globals" dict. The +stringizer then implements every Python "dunder" method used to +implement operators, and the value returned by that method +is a new stringizer whose value is a text representation +of that operation. + +When these stringizers are used in expressions, the result +of the expression is a new stringizer whose name textually +represents that expression. For example, let's say +you have a variable ``f``, which is a reference to a +stringizer initialized with the value ``'f'``. Here are +some examples of operations you could perform on ``f`` and +the values they would return:: + + >>> f + Stringizer('f') + >>> f + 3 + Stringizer('f + 3') + >> f["key"] + Stringizer('f["key"]') + +Bringing it all together: if we run a Python-generated +``__annotate__`` function, but we replace its globals +with a "fake globals" dict, all undefined symbols it +references will be replaced with stringizer proxy objects +representing those symbols, and any operations performed +on those proxies will in turn result in proxies +representing that expression. This allows ``__annotate__`` +to complete, and to return an annotations dict, with +stringizer instances standing in for names and entire +expressions that could not have otherwise been evaluated. + +In practice, the "stringizer" functionality will be implemented +in the ``ForwardRef`` object currently defined in the +``typing`` module. ``ForwardRef`` will be extended to +implement all stringizer functionality; it will also be +extended to support evaluating the string it contains, +to produce the real value (assuming all symbols referenced +are defined). This means the ``ForwardRef`` object +will retain references to the appropriate "globals", +"locals", and even "closure" information needed to +evaluate the expression. + +This technique is the core of how ``inspect.get_annotations`` +supports ``FORWARDREF`` and ``SOURCE`` formats. Initially, +``inspect.get_annotations`` will call the object's +``__annotate__`` method requesting the desired format. +If that raises ``NotImplementedError``, ``inspect.get_annotations`` +will construct a "fake globals" environment, then call +the object's ``__annotate__`` method. + +* ``inspect.get_annotations`` produces ``SOURCE`` format + by creating a new empty "fake globals" dict, binding it + to the object's ``__annotate__`` method, calling that + requesting ``VALUE`` format, and then extracting the string + "value" from each ``ForwardRef`` object + in the resulting dict. + +* ``inspect.get_annotations`` produces ``FORWARDREF`` format + by creating a new empty "fake globals" dict, pre-populating + it with the current contents of the ``__annotate__`` method's + globals dict, binding the "fake globals" dict to the object's + ``__annotate__`` method, calling that requesting ``VALUE`` + format, and returning the result. + +This entire technique works because the ``__annotate__`` functions +generated by the compiler are controlled by Python itself, and +are simple and predictable. They're +effectively a single ``return`` statement, computing and +returning the annotations dict. Since most operations needed +to compute an annotation are implemented in Python using dunder +methods, and the stringizer supports all the relevant dunder +methods, this approach is a reliable, practical solution. + +However, it's not reasonable to attempt this technique with +just any ``__annotate__`` method. This PEP assumes that +third-party libraries may implement their own ``__annotate__`` +methods, and those functions would almost certainly work +incorrectly when run in this "fake globals" environment. +For that reason, this PEP allocates a flag on code objects, +one of the unused bits in ``co_flags``, to mean "This code +object can be run in a 'fake globals' environment." This +makes the "fake globals" environment strictly opt-in, and +it's expected that only ``__annotate__`` methods generated +by the Python compiler will set it. + +The weakness in this technique is in handling operators which +don't directly map to dunder methods on an object. These are +all operators that implement some manner of flow control, +either branching or iteration: + +* Short-circuiting ``or`` +* Short-circuiting ``and`` +* Ternary operator (the ``if`` / ``then`` operator) +* Generator expressions +* List / dict / set comprehensions +* Iterable unpacking + +As a rule these techniques aren't used in annotations, +so it doesn't pose a problem in practice. However, the +recent addition of ``TypeVarTuple`` to Python does use +iterable unpacking. The dunder methods +involved (``__iter__`` and ``__next__``) don't permit +distinguishing between iteration use cases; in order to +correctly detect which use case was involved, mere +"fake globals" and a "stringizer" wouldn't be sufficient; +this would require a custom bytecode interpreter designed +specifically around producing ``SOURCE`` and ``FORWARDREF`` +formats. + +Thankfully there's a shortcut that will work fine: +the stringizer will simply assume that when its +iteration dunder methods are called, it's in service +of iterator unpacking being performed by ``TypeVarTuple``. +It will hard-code this behavior. This means no other +technique using iteration will work, but in practice +this won't inconvenience real-world use cases. + + +Finally, note that the "fake globals" environment +will also require constructing a matching "fake locals" +dictionary, which for ``FORWARDREF`` format will be +pre-populated with the relevant locals dict. The +"fake globals" environment will also have to create +a fake "closure", a tuple of ``ForwardRef`` objects +pre-created with the names of the free variables +referenced by the ``__annotate__`` method. + +``ForwardRef`` proxies created from ``__annotate__`` +methods that reference free variables will map the +names and closure values of those free variables into +the locals dictionary, to ensure that ``eval`` uses +the correct values for those names. + + +Compiler-generated ``__annotate__`` functions +============================================== + +As mentioned in the previous section, the ``__annotate__`` +functions generated by the compiler are simple. They're +mainly a single ``return`` statement, computing and +returning the annotations dict. + +However, the protocol for ``inspect.get_annotations`` +to request either ``FORWARDREF`` or ``SOURCE`` format +requires first asking the ``__annotate__`` method to +produce it. ``__annotate__`` methods generated by +the Python compiler won't support either of these +formats and will raise ``NotImplementedError()``. + + +Third-party ``__annotate__`` functions +====================================== + +Third-party classes and functions will likely need +to implement their own ``__annotate__`` methods, +so that downstream users of +those objects can take full advantage of annotations. +In particular, wrappers will likely need to transform +the annotation dicts produced by the wrapped object: adding, +removing, or modifying the dictionary in some way. + +Most of the time, third-party code will implement +their ``__annotate__`` methods by calling +``inspect.get_annotations`` on some existing upstream +object. For example, wrappers will likely request the +annotations dict for their wrapped object, +in the format that was requested from them, then +modify the returned annotations dict as appropriate +and return that. This allows third-party code to +leverage the "fake globals" technique without +having to understand or participate in it. + +Third-party libraries that support both pre- and +post-PEP-649 versions of Python will have to innovate +their own best practices on how to support both. +One sensible approach would be for their wrapper to +always support ``__annotate__``, then call it requesting +``VALUE`` format and store the result as the +``__annotations__`` on their wrapper object. +This would support pre-649 Python semantics, and be +forward-compatible with post-649 semantics. + + + +Pseudocode +========== + +Here's high-level pseudocode for ``inspect.get_annotations``:: + + def get_annotations(o, format): + if format == VALUE: + return dict(o.__annotations__) + + if format == FORWARDREF: + try: + return dict(o.__annotations__) + except NameError: + pass + + if not hasattr(o.__annotate__): + return {} + + c_a = o.__annotate__ + try: + return c_a(format) + except NotImplementedError: + if not can_be_called_with_fake_globals(c_a): + return {} + c_a_with_fake_globals = make_fake_globals_version(c_a, format) + return c_a_with_fake_globals(VALUE) + +Here's what a Python compiler-generated ``__annotate__`` method +might look like if it was written in Python:: + + def __annotate__(self, format): + if format != 1: + raise NotImplementedError() + return { ... } + +Here's how a third-party wrapper class might implement +``__annotate__``. In this example, the wrapper works +like ``functools.partial``, pre-binding one parameter of +the wrapped callable, which for simplicity must be named +``arg``:: + + def __annotate__(self, format): + ann = inspect.get_annotations(self.wrapped_fn, format) + if 'arg' in ann: + del ann['arg'] + return ann + + +Other modifications to the Python runtime +========================================= + +This PEP does not dictate exactly how it should be +implemented; that is left up to the language implementation +maintainers. However, the best implementation of this +PEP may require adding additional information to existing +Python objects, which is implicitly condoned by the acceptance +of this PEP. + +For example, it may be necessary to add a +``__globals__`` attribute to class objects, so that the +``__annotate__`` function for that class can be lazily +bound, only on demand. Also, ``__annotate__`` functions +defined on methods defined in a class may need to retain +a reference to the class's ``__dict__``, in order to +correctly evaluate names bound in that class. It's expected +that the CPython implementation of this PEP will include +both those new attributes. + +All such new information added to existing Python objects +should be done with "dunder" attributes, as they will of +course be implementation details. + + +Interactive REPL Shell +====================== + +The semantics established in this PEP also hold true when executing +code in Python's interactive REPL shell, except for module annotations +in the interactive module (``__main__``) itself. Since that module is +never "finished", there's no specific point where we can compile the +``__annotate__`` function. + +For the sake of simplicity, in this case we forego delayed evaluation. +Module-level annotations in the REPL shell will continue to work +exactly as they do with "stock semantics", evaluating immediately and +setting the result directly inside the ``__annotations__`` dict. + + +Annotations On Local Variables Inside Functions +=============================================== + +Python supports syntax for local variable annotations inside +functions. However, these annotations have no runtime +effect--they're discarded at compile-time. Therefore, this +PEP doesn't need to do anything to support them, the same +as stock semantics and :pep:`563`. + + + +Prototype +========= + +The original prototype implementation of this PEP can be found here: + +https://github.com/larryhastings/co_annotations/ + +As of this writing, the implementation is severely out of date; +it's based on Python 3.10 and implements the semantics of the +first draft of this PEP, from early 2021. It will be updated +shortly. + + + +Performance Comparison +====================== + +Performance with this PEP is generally favorable. There are four +scenarios to consider: + +* the runtime cost when annotations aren't defined, +* the runtime cost when annotations are defined but *not* referenced, and +* the runtime cost when annotations are defined and referenced as objects. +* the runtime cost when annotations are defined and referenced as strings. + +We'll examine each of these scenarios in the context of all three +semantics for annotations: stock, :pep:`563`, and this PEP. + +When there are no annotations, all three semantics have the same +runtime cost: zero. No annotations dict is created and no code is +generated for it. This requires no runtime processor time and +consumes no memory. + +When annotations are defined but not referenced, the runtime cost +of Python with this PEP is roughly the same as :pep:`563`, and +improved over stock. The specifics depend on the object +being annotated: + +* With stock semantics, the annotations dict is always built, and + set as an attribute of the object being annotated. +* In :pep:`563` semantics, for function objects, a precompiled + constant (a specially constructed tuple) is set as an attribute + of the function. For class and module objects, the annotations + dict is always built and set as an attribute of the class or module. +* With this PEP, a single object is set as an attribute of the + object being annotated. Most of the time, this object is + a constant (a code object), but when the annotations require a + class namespace or closure, this object will be a tuple constructed + at binding time. + +When annotations are both defined and referenced as objects, code using +this PEP should be much faster than :pep:`563`, and be as fast +or faster than stock. :pep:`563` semantics requires invoking +``eval()`` for every value inside an annotations dict which is +enormously slow. And the implementation of this PEP generates measurably +more efficient bytecode for class and module annotations than stock +semantics; for function annotations, this PEP and stock semantics +should be about the same speed. + +The one case where this PEP will be noticeably slower than :pep:`563` is when +annotations are requested as strings; it's hard to beat "they are already +strings." But stringified annotations are intended for online documentation use +cases, where performance is less likely to be a key factor. + +Memory use should also be comparable in all three scenarios across +all three semantic contexts. In the first and third scenarios, +memory usage should be roughly equivalent in all cases. +In the second scenario, when annotations are defined but not +referenced, using this PEP's semantics will mean the +function/class/module will store one unused code object (possibly +bound to an unused function object); with the other two semantics, +they'll store one unused dictionary or constant tuple. + + +*********************** +Backwards Compatibility +*********************** + +Backwards Compatibility With Stock Semantics +============================================ + +This PEP preserves nearly all existing behavior of +annotations from stock semantics: + +* The format of the annotations dict stored in + the ``__annotations__`` attribute is unchanged. + Annotations dicts contain real values, not strings + as per :pep:`563`. +* Annotations dicts are mutable, and any changes to them are + preserved. +* The ``__annotations__`` attribute can be explicitly set, + and any legal value set this way will be preserved. +* The ``__annotations__`` attribute can be deleted using + the ``del`` statement. + +Most code that works with stock semantics should +continue to work when this PEP is active without any +modification necessary. But there are exceptions, +as follows. + +First, there's a well-known idiom for accessing class +annotations which may not work correctly when this +PEP is active. The original implementation of class +annotations had what can only be called a bug: if a class +didn't define any annotations of its own, but one +of its base classes did define annotations, the class +would "inherit" those annotations. This behavior +was never desirable, so user code found a workaround: +instead of accessing the annotations on the class +directly via ``cls.__annotations__``, code would +access the class's annotations via its dict as in +``cls.__dict__.get("__annotations__", {})``. This +idiom worked because classes stored their annotations +in their ``__dict__``, and accessing them this way +avoided the lookups in the base classes. The technique +relied on implementation details of CPython, so it +was never supported behavior--though it was necessary. +However, when this PEP is active, a class may have +annotations defined but hasn't yet called ``__annotate__`` +and cached the result, in which case this approach +would lead to mistakenly assuming the class didn't have +annotations. +In any case, the bug was fixed as of Python 3.10, and the +idiom should no longer be used. Also as of Python 3.10, +there's an +`Annotations HOWTO `_ +that defines best practices +for working with annotations; code that follows these +guidelines will work correctly even when this PEP is +active, because it suggests using different approaches +to get annotations from class objects based on the +Python version the code runs under. + + +Since delaying the evaluation of annotations until they are +introspected changes the semantics of the language, it's observable +from within the language. Therefore it's *possible* to write code +that behaves differently based on whether annotations are +evaluated at binding time or at access time, e.g. + +.. code-block:: + + mytype = str + def foo(a:mytype): pass + mytype = int + print(foo.__annotations__['a']) + +This will print ```` with stock semantics +and ```` when this PEP is active. This is +therefore a backwards-incompatible change. However, this +example is poor programming style, so this change seems +acceptable. + + +There are two uncommon interactions possible with class +and module annotations that work with stock semantics +that would no longer work when this PEP was active. +These two interactions would have to be prohibited. The +good news is, neither is common, and neither is considered +good practice. In fact, they're rarely seen outside of +Python's own regression test suite. They are: + +* *Code that sets annotations on module or class attributes + from inside any kind of flow control statement.* It's + currently possible to set module and class attributes with + annotations inside an ``if`` or ``try`` statement, and it works + as one would expect. It's untenable to support this behavior + when this PEP is active. +* *Code in module or class scope that references or modifies the + local* ``__annotations__`` *dict directly.* Currently, when + setting annotations on module or class attributes, the generated + code simply creates a local ``__annotations__`` dict, then adds + mappings to it as needed. It's possible for user code + to directly modify this dict, though this doesn't seem to be + an intentional feature. Although it would be possible to support + this after a fashion once this PEP was active, the semantics + would likely be surprising and wouldn't make anyone happy. + +Note that these are both also pain points for static type checkers, +and are unsupported by those tools. It seems reasonable to +declare that both are at the very least unsupported, and their +use results in undefined behavior. It might be worth making a +small effort to explicitly prohibit them with compile-time checks. + +Finally, if this PEP is active, annotation values shouldn't use +the ``if / else`` ternary operator. Although this will work +correctly when accessing ``o.__annotations__`` or requesting +``inspect.VALUE`` from a helper function, the boolean expression +may not compute correctly with ``inspect.FORWARDREF`` when +some names are defined, and would be far less correct with +``inspect.SOURCE``. + + +Backwards Compatibility With PEP 563 Semantics +============================================== + +:pep:`563` changed the semantics of annotations. When its semantics +are active, annotations must assume they will be evaluated in +*module-level* or *class-level* scope. They may no longer refer directly +to local variables in the current function or an enclosing function. +This PEP removes that restriction, and annotations may refer any +local variable. + +:pep:`563` requires using ``eval`` (or a helper function like +``typing.get_type_hints`` or ``inspect.get_annotations`` that +uses ``eval`` for you) to convert stringized annotations into +their "real" values. Existing code that activates stringized +annotations, and calls ``eval()`` directly to convert the strings +back into real values, can simply remove the ``eval()`` call. +Existing code using a helper function would continue to work +unchanged, though use of those functions may become optional. + +Static typing users often have modules that only contain +inert type hint definitions--but no live code. These modules +are only needed when running static type checking; they aren't +used at runtime. But under stock semantics, these modules +have to be imported in order for the runtime to evaluate and +compute the annotations. Meanwhile, these modules often +caused circular import problems that could be difficult or +even impossible to solve. :pep:`563` allowed users to solve +these circular import problems by doing two things. First, +they activated :pep:`563` in their modules, which meant annotations +were constant strings, and didn't require the real symbols to +be defined in order for the annotations to be computable. +Second, this permitted users to only import the problematic +modules in an ``if typing.TYPE_CHECKING`` block. This allowed +the static type checkers to import the modules and the type +definitions inside, but they wouldn't be imported at runtime. +So far, this approach will work unchanged when this PEP is +active; ``if typing.TYPE_CHECKING`` is supported behavior. + +However, some codebases actually *did* examine their +annotations at runtime, even when using the ``if typing.TYPE_CHECKING`` +technique and not importing definitions used in their annotations. +These codebases examined the annotation strings *without +evaluating them,* instead relying on identity checks or +simple lexical analysis on the strings. + +This PEP supports these techniques too. But users will need +to port their code to it. First, user code will need to use +``inspect.get_annotations`` or ``typing.get_type_hints`` to +access the annotations; they won't be able to simply get the +``__annotations__`` attribute from their object. Second, +they will need to specify either ``inspect.FORWARDREF`` +or ``inspect.SOURCE`` for the ``format`` when calling that +function. This means the helper function can succeed in +producing the annotations dict, even when not all the symbols +are defined. Code expecting stringized annotations should +work unmodified with ``inspect.SOURCE`` formatted annotations +dicts; however, users should consider switching to +``inspect.FORWARDREF``, as it may make their analysis easier. + +Similarly, :pep:`563` permitted use of class decorators on +annotated classes in a way that hadn't previously been possible. +Some class decorators (e.g. :mod:`dataclasses`) examine the annotations +on the class. Because class decorators using the ``@`` decorator +syntax are run before the class name is bound, they can cause +unsolvable circular-definition problems. If you annotate attributes +of a class with references to the class itself, or annotate attributes +in multiple classes with circular references to each other, you +can't decorate those classes with the ``@`` decorator syntax +using decorators that examine the annotations. :pep:`563` allowed +this to work, as long as the decorators examined the strings lexically +and didn't use ``eval`` to evaluate them (or handled the ``NameError`` +with further workarounds). When this PEP is active, decorators will +be able to compute the annotations dict in ``inspect.SOURCE`` or +``inspect.FORWARDREF`` format using the helper functions. This +will permit them to analyze annotations containing undefined +symbols, in the format they prefer. + +Early adopters of :pep:`563` discovered that "stringized" +annotations were useful for automatically-generated documentation. +Users experimented with this use case, and Python's ``pydoc`` +has expressed some interest in this technique. This PEP supports +this use case; the code generating the documentation will have to be +updated to use a helper function to access the annotations in +``inspect.SOURCE`` format. + +Finally, the warnings about using the ``if / else`` ternary +operator in annotations apply equally to users of :pep:`563`. +It currently works for them, but could produce incorrect +results when requesting some formats from the helper functions. + +If this PEP is accepted, :pep:`563` will be deprecated and +eventually removed. To facilitate this transition for early +adopters of :pep:`563`, who now depend on its semantics, +``inspect.get_annotations`` and ``typing.get_type_hints`` will +implement a special affordance. + +The Python compiler won't generate annotation code objects +for objects defined in a module where :pep:`563` semantics are +active, even if this PEP is accepted. So, under normal +circumstances, requesting ``inspect.SOURCE`` format from a +helper function would return an empty dict. As an affordance, +to facilitate the transition, if the helper functions detect +that an object was defined in a module with :pep:`563` active, +and the user requests ``inspect.SOURCE`` format, they'll return +the current value of the ``__annotations__`` dict, which in +this case will be the stringized annotations. This will allow +:pep:`563` users who lexically analyze stringized annotations +to immediately change over to requesting ``inspect.SOURCE`` format +from the helper functions, which will hopefully smooth their +transition away from :pep:`563`. + + +************** +Rejected Ideas +************** + +"Just store the strings" +======================== + +One proposed idea for supporting ``SOURCE`` format was for +the Python compiler to emit the actual source code for the +annotation values somewhere, and to furnish that when +the user requested ``SOURCE`` format. + +This idea wasn't rejected so much as categorized as +"not yet". We already know we need to support ``FORWARDREF`` +format, and that technique can be adapted to support +``SOURCE`` format in just a few lines. There are many +unanswered questions about this approach: + +* Where would we store the strings? Would they always + be loaded when the annotated object was created, or + would they be lazy-loaded on demand? If so, how + would the lazy-loading work? +* Would the "source code" include the newlines and + comments of the original? Would it preserve all + whitespace, including indents and extra spaces used + purely for formatting? + +It's possible we'll revisit this topic in the future, +if improving the fidelity of ``SOURCE`` values to the +original source code is judged sufficiently important. + + +**************** +Acknowledgements +**************** + +Thanks to Carl Meyer, Barry Warsaw, Eric V. Smith, +Mark Shannon, Jelle Ziljstra, and Guido van Rossum for ongoing +feedback and encouragement. + +Particular thanks to several individuals who contributed key ideas +that became some of the best aspects of this proposal: + +* Carl Meyer suggested the "stringizer" technique that made + ``FORWARDREF`` and ``SOURCE`` formats possible, which + allowed making forward progress on this PEP possible after + a year of languishing due to seemingly-unfixable problems. + He also suggested the affordance for :pep:`563` users where + ``inspect.SOURCE`` will return the stringized annotations, + and many more suggestions besides. Carl was also the primary + correspondent in private email threads discussing this PEP, + and was a tireless resource and voice of sanity. This PEP + would almost certainly not have been accepted it were it not + for Carl's contributions. +* Mark Shannon suggested building the entire annotations dict + inside a single code object, and only binding it to a function + on demand. +* Guido van Rossum suggested that ``__annotate__`` + functions should duplicate the name visibility rules of + annotations under "stock" semantics. +* Jelle Zijlstra contributed not only feedback--but code! + + +********** +References +********** + +* https://github.com/larryhastings/co_annotations/issues + +* https://discuss.python.org/t/two-polls-on-how-to-revise-pep-649/23628 + +* https://discuss.python.org/t/a-massive-pep-649-update-with-some-major-course-corrections/25672 + + + +********* +Copyright +********* + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0650.rst b/peps/pep-0650.rst similarity index 99% rename from pep-0650.rst rename to peps/pep-0650.rst index dec2ee717..b1cbac068 100644 --- a/pep-0650.rst +++ b/peps/pep-0650.rst @@ -4,11 +4,12 @@ Author: Vikram Jayanthi , Dustin Ingram , Brett Cannon Discussions-To: https://discuss.python.org/t/pep-650-specifying-installer-requirements-for-python-projects/6657 -Status: Draft -Type: Process +Status: Withdrawn +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 16-Jul-2020 -Post-History: 2021-01-14 +Post-History: 14-Jan-2021 Abstract diff --git a/pep-0651.rst b/peps/pep-0651.rst similarity index 100% rename from pep-0651.rst rename to peps/pep-0651.rst diff --git a/pep-0652.rst b/peps/pep-0652.rst similarity index 97% rename from pep-0652.rst rename to peps/pep-0652.rst index ede2de38e..19bdff979 100644 --- a/pep-0652.rst +++ b/peps/pep-0652.rst @@ -10,6 +10,9 @@ Python-Version: 3.10 Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/IN4XMFLQJ6D6V67EXU27GV3QWSEHHNNH/ +.. canonical-doc:: :ref:`python:stable` (user docs) and + :ref:`devguide:c-api` (development docs) + Abstract ======== @@ -37,9 +40,9 @@ In theory, this brings several advantages: details, this API is becoming a viable target for alternate Python implementations that would be incompatible with the full C API. -However, in hindsight, PEP 384 and its implementation has several issues: +However, in hindsight, :pep:`384` and its implementation has several issues: -* It is ill-defined. According to PEP 384, functions are *opt-out*: +* It is ill-defined. According to :pep:`384`, functions are *opt-out*: all functions not specially marked are part of the Stable ABI. In practice, for Windows there's a list that's *opt-in*. For users there is a ``#define`` that should make only the Stable ABI @@ -207,7 +210,7 @@ The following will be generated from the ABI manifest: * Source for the Windows shared library, ``PC/python3dll.c``. * Input for documentation (see below). -* Test case that checks the runtime availablility of symbols (see below). +* Test case that checks the runtime availability of symbols (see below). The following will be checked against the Stable ABI manifest as part of continuous integration: @@ -242,7 +245,7 @@ The initial Stable ABI manifest will include: Items that are no longer in CPython when this PEP is accepted will be removed from the list. -Additional items may be aded to the initial manifest according to +Additional items may be added to the initial manifest according to the checklist below. diff --git a/pep-0653.rst b/peps/pep-0653.rst similarity index 93% rename from pep-0653.rst rename to peps/pep-0653.rst index 7c97e5910..bfb251122 100644 --- a/pep-0653.rst +++ b/peps/pep-0653.rst @@ -11,11 +11,11 @@ Post-History: 18-Feb-2021 Abstract ======== -This PEP proposes a semantics for pattern matching that respects the general concept of PEP 634, +This PEP proposes a semantics for pattern matching that respects the general concept of :pep:`634`, but is more precise, easier to reason about, and should be faster. The object model will be extended with two special (dunder) attributes, ``__match_container__`` and -``__match_class__``, in addition to the ``__match_args__`` attribute from PEP 634, to support pattern matching. +``__match_class__``, in addition to the ``__match_args__`` attribute from :pep:`634`, to support pattern matching. Both of these new attributes must be integers and ``__match_args__`` is required to be a tuple of unique strings. With this PEP: @@ -27,14 +27,14 @@ With this PEP: Motivation ========== -Pattern matching in Python, as described in PEP 634, is to be added to Python 3.10. -Unfortunately, PEP 634 is not as precise about the semantics as it could be, +Pattern matching in Python, as described in :pep:`634`, is to be added to Python 3.10. +Unfortunately, :pep:`634` is not as precise about the semantics as it could be, nor does it allow classes sufficient control over how they match patterns. Precise semantics ----------------- -PEP 634 explicitly includes a section on undefined behavior. +:pep:`634` explicitly includes a section on undefined behavior. Large amounts of undefined behavior may be acceptable in a language like C, but in Python it should be kept to a minimum. Pattern matching in Python can be defined more precisely without losing expressiveness or performance. @@ -42,11 +42,11 @@ Pattern matching in Python can be defined more precisely without losing expressi Improved control over class matching ------------------------------------ -PEP 634 delegates the decision over whether a class is a sequence or mapping to ``collections.abc``. +:pep:`634` delegates the decision over whether a class is a sequence or mapping to ``collections.abc``. Not all classes that could be considered sequences are registered as subclasses of ``collections.abc.Sequence``. This PEP allows them to match sequence patterns, without the full ``collections.abc.Sequence`` machinery. -PEP 634 privileges some builtin classes with a special form of matching, the "self" match. +:pep:`634` privileges some builtin classes with a special form of matching, the "self" match. For example the pattern ``list(x)`` matches a list and assigns the list to ``x``. By allowing classes to choose which kinds of pattern they match, other classes can use this form as well. @@ -57,7 +57,7 @@ For example, using ``sympy``, we might want to write:: return Pow(a, 2) Which requires the sympy class ``Symbol`` to "self" match. -For ``sympy`` to support this pattern with PEP 634 is possible, but a bit tricky. +For ``sympy`` to support this pattern with :pep:`634` is possible, but a bit tricky. With this PEP it can be implemented very easily [1]_. Robustness @@ -67,7 +67,7 @@ With this PEP, access to attributes during pattern matching becomes well defined This makes pattern matching less error prone when matching objects with hidden side effects, such as object-relational mappers. Objects will have more control over their own deconstruction, which can help prevent unintended consequences should attribute access have side-effects. -PEP 634 relies on the ``collections.abc`` module when determining which patterns a value can match, implicitly importing it if necessary. +:pep:`634` relies on the ``collections.abc`` module when determining which patterns a value can match, implicitly importing it if necessary. This PEP will eliminate surprising import errors and misleading audit events from those imports. @@ -107,7 +107,7 @@ Specification Additions to the object model ----------------------------- -The ``__match_container__ ``and ``__match_class__`` attributes will be added to ``object``. +The ``__match_container__`` and ``__match_class__`` attributes will be added to ``object``. ``__match_container__`` should be overridden by classes that want to match mapping or sequence patterns. ``__match_class__`` should be overridden by classes that want to change the default behavior when matching class patterns. @@ -144,7 +144,7 @@ These special attributes will be inherited as normal. If ``__match_args__`` is overridden, then it is required to hold a tuple of unique strings. It may be empty. .. note:: - ``__match_args__`` will be automatically generated for dataclasses and named tuples, as specified in PEP 634. + ``__match_args__`` will be automatically generated for dataclasses and named tuples, as specified in :pep:`634`. The pattern matching implementation is *not* required to check that any of these attributes behave as specified. If the value of ``__match_container__``, ``__match_class__`` or ``__match_args__`` is not as specified, then @@ -156,7 +156,7 @@ Semantics of the matching process In the following, all variables of the form ``$var`` are temporary variables and are not visible to the Python program. They may be visible via introspection, but that is an implementation detail and should not be relied on. -The psuedo-statement ``FAIL`` is used to signify that matching failed for this pattern and that matching should move to the next pattern. +The pseudo-statement ``FAIL`` is used to signify that matching failed for this pattern and that matching should move to the next pattern. If control reaches the end of the translation without reaching a ``FAIL``, then it has matched, and following patterns are ignored. Variables of the form ``$ALL_CAPS`` are meta-variables holding a syntactic element, they are not normal variables. @@ -165,10 +165,10 @@ but an unpacking of ``$items`` into the variables that ``$VARS`` holds. For example, with the abstract syntax ``case [$VARS]:``, and the concrete syntax ``case[a, b]:`` then ``$VARS`` would hold the variables ``(a, b)``, not the values of those variables. -The psuedo-function ``QUOTE`` takes a variable and returns the name of that variable. +The pseudo-function ``QUOTE`` takes a variable and returns the name of that variable. For example, if the meta-variable ``$VAR`` held the variable ``foo`` then ``QUOTE($VAR) == "foo"``. -All additional code listed below that is not present in the original source will not trigger line events, conforming to PEP 626. +All additional code listed below that is not present in the original source will not trigger line events, conforming to :pep:`626`. Preamble @@ -521,7 +521,7 @@ Implementation The naive implementation that follows from the specification will not be very efficient. Fortunately, there are some reasonably straightforward transformations that can be used to improve performance. -Performance should be comparable to the implementation of PEP 634 (at time of writing) by the release of 3.10. +Performance should be comparable to the implementation of :pep:`634` (at time of writing) by the release of 3.10. Further performance improvements may have to wait for the 3.11 release. Possible optimizations @@ -593,7 +593,7 @@ Can be compiled roughly as: Mapping patterns '''''''''''''''' -The best stategy here is probably to form a decision tree based on the size of the mapping and which keys are present. +The best strategy here is probably to form a decision tree based on the size of the mapping and which keys are present. There is no point repeatedly testing for the presence of a key. For example:: @@ -645,7 +645,7 @@ The changes to the semantics can be summarized as: * Allows classes to opt out of deconstruction altogether, if necessary, but setting ``__match_class__ = 0``. * The behavior when matching patterns is more precisely defined, but is otherwise unchanged. -There are no changes to syntax. All examples given in the PEP 636 tutorial should continue to work as they do now. +There are no changes to syntax. All examples given in the :pep:`636` tutorial should continue to work as they do now. Rejected Ideas ============== @@ -668,7 +668,7 @@ For the class:: ... Ideally we would match the attributes "a" and "p", but not "m". -However, there is no general way to do that, so this PEP now follows the semantics of PEP 634. +However, there is no general way to do that, so this PEP now follows the semantics of :pep:`634`. Lookup of ``__match_args__`` on the subject not the pattern ----------------------------------------------------------- @@ -715,11 +715,6 @@ In an earlier version of this PEP, there was a distinct value for ``__match_clas pattern that would have required deconstruction. However, this would become redundant once ``MATCH_POSITIONAL`` is introduced, and complicates the specification for an extremely rare case. -References -========== - -PEP 634 -https://www.python.org/dev/peps/pep-0634 Code examples ============= @@ -876,14 +871,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0654.rst b/peps/pep-0654.rst similarity index 95% rename from pep-0654.rst rename to peps/pep-0654.rst index a43eb754e..113f657fc 100644 --- a/pep-0654.rst +++ b/peps/pep-0654.rst @@ -7,6 +7,7 @@ Status: Accepted Type: Standards Track Content-Type: text/x-rst Created: 22-Feb-2021 +Python-Version: 3.11 Post-History: 22-Feb-2021, 20-Mar-2021 @@ -45,7 +46,7 @@ together as the stack unwinds. Several real world use cases are listed below. are proposing (see also the `Programming Without 'except \*'`_ section.) Implementing a better task spawning API in asyncio, inspired by Trio - nurseries, was the main motivation for this PEP. That work is currently + nurseries [13]_, was the main motivation for this PEP. That work is currently blocked on Python not having native language level support for exception groups. @@ -150,12 +151,14 @@ fields ``message`` and ``exceptions``. For example: ``ExceptionGroup('issues', [ValueError('bad value'), TypeError('bad type')])``. The difference between them is that ``ExceptionGroup`` can only wrap ``Exception`` subclasses while ``BaseExceptionGroup`` can wrap any -``BaseException`` subclass. A factory method that inspects the nested -exceptions and selects between ``ExceptionGroup`` and ``BaseExceptionGroup`` -makes the choice automatic. In the rest of the document, when we refer to -an exception group, we mean either an ``ExceptionGroup`` or a -``BaseExceptionGroup``. When it is necessary to make the distinction, we -use the class name. For brevity, we will use ``ExceptionGroup`` in code +``BaseException`` subclass. The ``BaseExceptionGroup`` constructor +inspects the nested exceptions and if they are all ``Exception`` subclasses, +it returns an ``ExceptionGroup`` rather than a ``BaseExceptionGroup``. The +``ExceptionGroup`` constructor raises a ``TypeError`` if any of the nested +exceptions is not an ``Exception`` instance. In the rest of the document, +when we refer to an exception group, we mean either an ``ExceptionGroup`` +or a ``BaseExceptionGroup``. When it is necessary to make the distinction, +we use the class name. For brevity, we will use ``ExceptionGroup`` in code examples that are relevant to both. Since an exception group can be nested, it represents a tree of exceptions, @@ -184,30 +187,31 @@ contains only those exceptions for which the condition is true: ... ) ... ] ... ) + >>> import traceback >>> traceback.print_exception(eg) - | ExceptionGroup: one + | ExceptionGroup: one (3 sub-exceptions) +-+---------------- 1 ---------------- | TypeError: 1 +---------------- 2 ---------------- - | ExceptionGroup: two + | ExceptionGroup: two (2 sub-exceptions) +-+---------------- 1 ---------------- | TypeError: 2 +---------------- 2 ---------------- | ValueError: 3 +------------------------------------ +---------------- 3 ---------------- - | ExceptionGroup: three + | ExceptionGroup: three (1 sub-exception) +-+---------------- 1 ---------------- | OSError: 4 +------------------------------------ >>> type_errors = eg.subgroup(lambda e: isinstance(e, TypeError)) >>> traceback.print_exception(type_errors) - | ExceptionGroup: one + | ExceptionGroup: one (2 sub-exceptions) +-+---------------- 1 ---------------- | TypeError: 1 +---------------- 2 ---------------- - | ExceptionGroup: two + | ExceptionGroup: two (1 sub-exception) +-+---------------- 1 ---------------- | TypeError: 2 +------------------------------------ @@ -238,23 +242,23 @@ If both the subgroup and its complement are needed, the >>> type_errors, other_errors = eg.split(lambda e: isinstance(e, TypeError)) >>> traceback.print_exception(type_errors) - | ExceptionGroup: one + | ExceptionGroup: one (2 sub-exceptions) +-+---------------- 1 ---------------- | TypeError: 1 +---------------- 2 ---------------- - | ExceptionGroup: two + | ExceptionGroup: two (1 sub-exception) +-+---------------- 1 ---------------- | TypeError: 2 +------------------------------------ >>> traceback.print_exception(other_errors) - | ExceptionGroup: one + | ExceptionGroup: one (2 sub-exceptions) +-+---------------- 1 ---------------- - | ExceptionGroup: two + | ExceptionGroup: two (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 3 +------------------------------------ +---------------- 2 ---------------- - | ExceptionGroup: three + | ExceptionGroup: three (1 sub-exception) +-+---------------- 1 ---------------- | OSError: 4 +------------------------------------ @@ -385,7 +389,7 @@ in the following example: >>> raise ExceptionGroup("two", [f(2), eg]) + Exception Group Traceback (most recent call last): | File "", line 1, in - | ExceptionGroup: two + | ExceptionGroup: two (2 sub-exceptions) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "", line 3, in f @@ -393,7 +397,7 @@ in the following example: +---------------- 2 ---------------- | Exception Group Traceback (most recent call last): | File "", line 2, in - | ExceptionGroup: one + | ExceptionGroup: one (1 sub-exception) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "", line 3, in f @@ -800,29 +804,29 @@ merged with the unhandled ``TypeErrors``. ... *ValueError: ExceptionGroup('eg', [ValueError(1), ExceptionGroup('nested', [ValueError(6)])]) *OSError: ExceptionGroup('eg', [OSError(3), ExceptionGroup('nested', [OSError(4)])]) - | ExceptionGroup + | ExceptionGroup: (2 sub-exceptions) +-+---------------- 1 ---------------- | Exception Group Traceback (most recent call last): | File "", line 15, in | File "", line 2, in - | ExceptionGroup: eg + | ExceptionGroup: eg (2 sub-exceptions) +-+---------------- 1 ---------------- | ValueError: 1 +---------------- 2 ---------------- - | ExceptionGroup: nested + | ExceptionGroup: nested (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 6 +------------------------------------ +---------------- 2 ---------------- | Exception Group Traceback (most recent call last): | File "", line 2, in - | ExceptionGroup: eg + | ExceptionGroup: eg (3 sub-exceptions) +-+---------------- 1 ---------------- | TypeError: 2 +---------------- 2 ---------------- | OSError: 3 +---------------- 3 ---------------- - | ExceptionGroup: nested + | ExceptionGroup: nested (2 sub-exceptions) +-+---------------- 1 ---------------- | OSError: 4 +---------------- 2 ---------------- @@ -846,11 +850,11 @@ it into the new ``ExceptionGroup``. ... except* ValueError: ... raise ExceptionGroup("two", [KeyError('x'), KeyError('y')]) ... - | ExceptionGroup + | ExceptionGroup: (2 sub-exceptions) +-+---------------- 1 ---------------- | Exception Group Traceback (most recent call last): | File "", line 2, in - | ExceptionGroup: one + | ExceptionGroup: one (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: a +------------------------------------ @@ -859,7 +863,7 @@ it into the new ``ExceptionGroup``. | | Exception Group Traceback (most recent call last): | File "", line 4, in - | ExceptionGroup: two + | ExceptionGroup: two (2 sub-exceptions) +-+---------------- 1 ---------------- | KeyError: 'x' +---------------- 2 ---------------- @@ -868,7 +872,7 @@ it into the new ``ExceptionGroup``. +---------------- 2 ---------------- | Exception Group Traceback (most recent call last): | File "", line 2, in - | ExceptionGroup: one + | ExceptionGroup: one (1 sub-exception) +-+---------------- 1 ---------------- | TypeError: b +------------------------------------ @@ -889,21 +893,18 @@ chaining: ... except* TypeError as e: ... raise ValueError('bad value') from e ... - | ExceptionGroup + | ExceptionGroup: (1 sub-exception) +-+---------------- 1 ---------------- - | ExceptionGroup - +-+---------------- 1 ---------------- - | Traceback (most recent call last): - | File "", line 2, in - | TypeError: bad type - +------------------------------------ - | - | The above exception was the direct cause of the following exception: - | | Traceback (most recent call last): - | File "", line 4, in - | ValueError: bad value + | File "", line 2, in + | TypeError: bad type +------------------------------------ + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 4, in + ValueError: bad value >>> @@ -919,12 +920,9 @@ other clauses from the same ``try`` statement: ... except* ValueError: ... print('never') ... - | ExceptionGroup - +-+---------------- 1 ---------------- - | Traceback (most recent call last): - | File "", line 4, in - | ValueError: 2 - +---------------------------------------- + Traceback (most recent call last): + File "", line 4, in + ValueError: 2 >>> @@ -941,11 +939,11 @@ direct child of the new exception group created for that: ... except* ValueError: ... raise KeyError('x') ... - | ExceptionGroup + | ExceptionGroup: (1 sub-exception) +-+---------------- 1 ---------------- | Exception Group Traceback (most recent call last): | File "", line 2, in - | ExceptionGroup: eg + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: a +------------------------------------ @@ -962,11 +960,11 @@ direct child of the new exception group created for that: ... except* ValueError: ... raise KeyError('x') ... - | ExceptionGroup + | ExceptionGroup: (2 sub-exceptions) +-+---------------- 1 ---------------- | Exception Group Traceback (most recent call last): | File "", line 2, in - | ExceptionGroup: eg + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: a +------------------------------------ @@ -979,7 +977,7 @@ direct child of the new exception group created for that: +---------------- 2 ---------------- | Exception Group Traceback (most recent call last): | File "", line 2, in - | ExceptionGroup: eg + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | TypeError: b +------------------------------------ @@ -1463,7 +1461,7 @@ Steve Stagg, Victor Stinner, Marco Sulla, Petr Viktorin and Barry Warsaw. Acceptance ========== -PEP 654 was `accepted by Thomas Wouters on Sep 24, 2021 +:pep:`654` was `accepted by Thomas Wouters on Sep 24, 2021 `_. @@ -1476,7 +1474,7 @@ References .. [3] https://github.com/python-trio/trio/issues/611 -.. [4] https://bugs.python.org/issue29980 +.. [4] https://github.com/python/cpython/issues/74166 .. [5] https://docs.python.org/3/library/atexit.html#atexit.register @@ -1484,7 +1482,7 @@ References .. [7] https://hypothesis.readthedocs.io/en/latest/settings.html#hypothesis.settings.report_multiple_bugs -.. [8] https://bugs.python.org/issue40857 +.. [8] https://github.com/python/cpython/issues/85034 .. [9] https://trio.readthedocs.io/en/stable/reference-core.html#trio.MultiError @@ -1501,13 +1499,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0655.rst b/peps/pep-0655.rst similarity index 54% rename from pep-0655.rst rename to peps/pep-0655.rst index 012f88817..f2dfa1059 100644 --- a/pep-0655.rst +++ b/peps/pep-0655.rst @@ -2,30 +2,35 @@ PEP: 655 Title: Marking individual TypedDict items as required or potentially-missing Author: David Foster Sponsor: Guido van Rossum -Discussions-To: typing-sig at python.org -Status: Draft +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/53XVOD5ZUKJ263MWA6AUPEA6J7LBBLNV/ +Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst -Requires: 604 Created: 30-Jan-2021 Python-Version: 3.11 -Post-History: 31-Jan-2021, 11-Feb-2021, 20-Feb-2021, 26-Feb-2021 +Post-History: 31-Jan-2021, 11-Feb-2021, 20-Feb-2021, 26-Feb-2021, 17-Jan-2022, 28-Jan-2022 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/AJEDNVC3FXM5QXNNW5CR4UCT4KI5XVUE/ Abstract ======== -`PEP 589 `__ defines syntax -for declaring a TypedDict with all required keys and syntax for defining -a TypedDict with `all potentially-missing -keys `__ however it -does not provide any syntax to declare some keys as required and others -as potentially-missing. This PEP introduces two new syntaxes: -``Required[...]`` which can be used on individual items of a +:pep:`589` defines notation +for declaring a TypedDict with all required keys and notation for defining +a TypedDict with :pep:`all potentially-missing keys <589#totality>`, however it +does not provide a mechanism to declare some keys as required and others +as potentially-missing. This PEP introduces two new notations: +``Required[]``, which can be used on individual items of a TypedDict to mark them as required, and -``NotRequired[...]`` which can be used on individual items +``NotRequired[]``, which can be used on individual items to mark them as potentially-missing. +This PEP makes no Python grammar changes. Correct usage +of required and potentially-missing keys of TypedDicts is intended to be +enforced only by static type checkers and need not be enforced by +Python itself at runtime. + Motivation ========== @@ -47,15 +52,39 @@ different value for ``total``: Having to declare two different TypedDict types for this purpose is cumbersome. +This PEP introduces two new type qualifiers, ``typing.Required`` and +``typing.NotRequired``, which allow defining a *single* TypedDict with +a mix of both required and potentially-missing keys: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + +This PEP also makes it possible to define TypedDicts in the +:pep:`alternative functional syntax <589#alternative-syntax>` +with a mix of required and potentially-missing keys, +which is not currently possible at all because the alternative syntax does +not support inheritance: + +:: + + Actor = TypedDict('Actor', { + 'name': str, + # "in" is a keyword, so the functional syntax is necessary + 'in': NotRequired[List[str]], + }) + Rationale ========= -One might think it unusual to propose syntax that prioritizes marking -*required* keys rather than syntax for *potentially-missing* keys, as is +One might think it unusual to propose notation that prioritizes marking +*required* keys rather than *potentially-missing* keys, as is customary in other languages like TypeScript: -:: +.. code-block:: typescript interface Movie { title: string; @@ -63,7 +92,7 @@ customary in other languages like TypeScript: } The difficulty is that the best word for marking a potentially-missing -key, ``Optional[...]``, is already used in Python for a completely +key, ``Optional[]``, is already used in Python for a completely different purpose: marking values that could be either of a particular type or ``None``. In particular the following does not work: @@ -74,17 +103,17 @@ type or ``None``. In particular the following does not work: year: Optional[int] # means int|None, not potentially-missing! Attempting to use any synonym of “optional” to mark potentially-missing -keys (like ``Missing[...]``) would be too similar to ``Optional[...]`` +keys (like ``Missing[]``) would be too similar to ``Optional[]`` and be easy to confuse with it. Thus it was decided to focus on positive-form phrasing for required keys -instead, which is straightforward to spell as ``Required[...]``. +instead, which is straightforward to spell as ``Required[]``. Nevertheless it is common for folks wanting to extend a regular (``total=True``) TypedDict to only want to add a small number of potentially-missing keys, which necessitates a way to mark keys that are *not* required and potentially-missing, and so we also allow the -``NotRequired[...]`` form for that case. +``NotRequired[]`` form for that case. Specification @@ -109,10 +138,11 @@ potentially-missing key: title: str year: NotRequired[int] -It is an error to use ``Required[...]`` or ``NotRequired[...]`` in any +It is an error to use ``Required[]`` or ``NotRequired[]`` in any location that is not an item of a TypedDict. +Type checkers must enforce this restriction. -It is valid to use ``Required[...]`` and ``NotRequired[...]`` even for +It is valid to use ``Required[]`` and ``NotRequired[]`` even for items where it is redundant, to enable additional explicitness if desired: :: @@ -121,6 +151,137 @@ items where it is redundant, to enable additional explicitness if desired: title: Required[str] # redundant year: NotRequired[int] +It is an error to use both ``Required[]`` and ``NotRequired[]`` at the +same time: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[Required[int]] # ERROR + +Type checkers must enforce this restriction. +The runtime implementations of ``Required[]`` and ``NotRequired[]`` +may also enforce this restriction. + +The :pep:`alternative functional syntax <589#alternative-syntax>` +for TypedDict also supports +``Required[]`` and ``NotRequired[]``: + +:: + + Movie = TypedDict('Movie', {'name': str, 'year': NotRequired[int]}) + + +Interaction with ``total=False`` +-------------------------------- + +Any :pep:`589`-style TypedDict declared with ``total=False`` is equivalent +to a TypedDict with an implicit ``total=True`` definition with all of its +keys marked as ``NotRequired[]``. + +Therefore: + +:: + + class _MovieBase(TypedDict): # implicitly total=True + title: str + + class Movie(_MovieBase, total=False): + year: int + + +is equivalent to: + +:: + + class _MovieBase(TypedDict): + title: str + + class Movie(_MovieBase): + year: NotRequired[int] + + +Interaction with ``Annotated[]`` +----------------------------------- + +``Required[]`` and ``NotRequired[]`` can be used with ``Annotated[]``, +in any nesting order: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[Annotated[int, ValueRange(-9999, 9999)]] # ok + +:: + + class Movie(TypedDict): + title: str + year: Annotated[NotRequired[int], ValueRange(-9999, 9999)] # ok + +In particular allowing ``Annotated[]`` to be the outermost annotation +for an item allows better interoperability with non-typing uses of +annotations, which may always want ``Annotated[]`` as the outermost annotation. +[3]_ + + +Runtime behavior +---------------- + + +Interaction with ``get_type_hints()`` +''''''''''''''''''''''''''''''''''''' + +``typing.get_type_hints(...)`` applied to a TypedDict will by default +strip out any ``Required[]`` or ``NotRequired[]`` type qualifiers, +since these qualifiers are expected to be inconvenient for code +casually introspecting type annotations. + +``typing.get_type_hints(..., include_extras=True)`` however +*will* retain ``Required[]`` and ``NotRequired[]`` type qualifiers, +for advanced code introspecting type annotations that +wishes to preserve *all* annotations in the original source: + +:: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + assert get_type_hints(Movie) == \ + {'title': str, 'year': int} + assert get_type_hints(Movie, include_extras=True) == \ + {'title': str, 'year': NotRequired[int]} + + +Interaction with ``get_origin()`` and ``get_args()`` +'''''''''''''''''''''''''''''''''''''''''''''''''''' + +``typing.get_origin()`` and ``typing.get_args()`` will be updated to +recognize ``Required[]`` and ``NotRequired[]``: + +:: + + assert get_origin(Required[int]) is Required + assert get_args(Required[int]) == (int,) + + assert get_origin(NotRequired[int]) is NotRequired + assert get_args(NotRequired[int]) == (int,) + + +Interaction with ``__required_keys__`` and ``__optional_keys__`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +An item marked with ``Required[]`` will always appear +in the ``__required_keys__`` for its enclosing TypedDict. Similarly an item +marked with ``NotRequired[]`` will always appear in ``__optional_keys__``. + +:: + + assert Movie.__required_keys__ == frozenset({'title'}) + assert Movie.__optional_keys__ == frozenset({'year'}) + Backwards Compatibility ======================= @@ -133,16 +294,17 @@ How to Teach This To define a TypedDict where most keys are required and some are potentially-missing, define a single TypedDict as normal -and mark those few keys that are potentially-missing with ``NotRequired[...]``. +(without the ``total`` keyword) +and mark those few keys that are potentially-missing with ``NotRequired[]``. To define a TypedDict where most keys are potentially-missing and a few are required, define a ``total=False`` TypedDict -and mark those few keys that are required with ``Required[...]``. +and mark those few keys that are required with ``Required[]``. If some items accept ``None`` in addition to a regular value, it is -recommended that the ``TYPE|None`` syntax be preferred over +recommended that the ``TYPE|None`` notation be preferred over ``Optional[TYPE]`` for marking such item values, to avoid using -``Required[...]`` or ``NotRequired[...]`` alongside ``Optional[...]`` +``Required[]`` or ``NotRequired[]`` alongside ``Optional[]`` within the same TypedDict definition: Yes: @@ -155,7 +317,15 @@ Yes: name: str owner: NotRequired[str|None] -Avoid (unless Python 3.5-3.6): +Okay (required for Python 3.5.3-3.6): + +:: + + class Dog(TypedDict): + name: str + owner: 'NotRequired[str|None]' + +No: :: @@ -164,19 +334,66 @@ Avoid (unless Python 3.5-3.6): # ick; avoid using both Optional and NotRequired owner: NotRequired[Optional[str]] +Usage in Python <3.11 +--------------------- + +If your code supports Python <3.11 and wishes to use ``Required[]`` or +``NotRequired[]`` then it should use ``typing_extensions.TypedDict`` rather +than ``typing.TypedDict`` because the latter will not understand +``(Not)Required[]``. In particular ``__required_keys__`` and +``__optional_keys__`` on the resulting TypedDict type will not be correct: + +Yes (Python 3.11+ only): + +:: + + from typing import NotRequired, TypedDict + + class Dog(TypedDict): + name: str + owner: NotRequired[str|None] + +Yes (Python <3.11 and 3.11+): + +:: + + from __future__ import annotations # for Python 3.7-3.9 + + from typing_extensions import NotRequired, TypedDict # for Python <3.11 with (Not)Required + + class Dog(TypedDict): + name: str + owner: NotRequired[str|None] + +No (Python <3.11 and 3.11+): + +:: + + from typing import TypedDict # oops: should import from typing_extensions instead + from typing_extensions import NotRequired + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + assert Movie.__required_keys__ == frozenset({'title', 'year'}) # yikes + assert Movie.__optional_keys__ == frozenset() # yikes + Reference Implementation ======================== -The goal is to be able to make the following statement: +The `mypy `__ +`0.930 `__, +`pyright `__ +`1.1.117 `__, +and `pyanalyze `__ +`0.4.0 `__ +type checkers support ``Required`` and ``NotRequired``. - The `mypy `__ type checker supports - ``Required`` and ``NotRequired``. A reference implementation of the - runtime component is provided in the - `typing_extensions `__ - module. - -The mypy implementation is currently still being worked on. +A reference implementation of the runtime component is provided in the +`typing_extensions `__ +module. Rejected Ideas @@ -189,19 +406,20 @@ Special syntax around the *key* of a TypedDict item class MyThing(TypedDict): opt1?: str # may not exist, but if exists, value is string - opt2: Optional[str] # always exists, but may have null value + opt2: Optional[str] # always exists, but may have None value -or: +This notation would require Python grammar changes and it is not +believed that marking TypedDict items as required or potentially-missing +would meet the high bar required to make such grammar changes. :: class MyThing(TypedDict): Optional[opt1]: str # may not exist, but if exists, value is string - opt2: Optional[str] # always exists, but may have null value + opt2: Optional[str] # always exists, but may have None value -These syntaxes would require Python grammar changes and it is not -believed that marking TypedDict items as required or potentially-missing -would meet the high bar required to make such grammar changes. +This notation causes ``Optional[]`` to take on different meanings depending +on where it is positioned, which is inconsistent and confusing. Also, “let’s just not put funny syntax before the colon.” [1]_ @@ -216,13 +434,13 @@ with opposite-of-normal totality: :: class MyThing(TypedDict, total=False): - req1: +int # + means a required key, or Required[...] + req1: +int # + means a required key, or Required[] opt1: str req2: +float class MyThing(TypedDict): req1: int - opt1: -str # - means a potentially-missing key, or NotRequired[...] + opt1: -str # - means a potentially-missing key, or NotRequired[] req2: float class MyThing(TypedDict): @@ -234,10 +452,20 @@ Such operators could be implemented on ``type`` via the ``__pos__``, ``__neg__`` and ``__invert__`` special methods without modifying the grammar. -It was decided that it would be prudent to introduce longform syntax -(i.e. ``Required[...]`` and ``NotRequired[...]``) before introducing -any shortform syntax. Future PEPs may reconsider introducing this -or other shortform syntax options. +It was decided that it would be prudent to introduce long-form notation +(i.e. ``Required[]`` and ``NotRequired[]``) before introducing +any short-form notation. Future PEPs may reconsider introducing this +or other short-form notation options. + +Note when reconsidering introducing this short-form notation that +``+``, ``-``, and ``~`` already have existing meanings in the Python +typing world: covariant, contravariant, and invariant: + +:: + + >>> from typing import TypeVar + >>> (TypeVar('T', covariant=True), TypeVar('U', contravariant=True), TypeVar('V')) + (+T, -U, ~V) Marking absence of a value with a special constant @@ -261,7 +489,7 @@ as the type of a variable which is only conditionally defined: class MyClass: attr: int|Missing - + def __init__(self, set_attr: bool) -> None: if set_attr: self.attr = 10 @@ -330,7 +558,7 @@ or a check against ``locals()`` for local variables: packet_bytes: Union[str, Missing] if packet_data is not None: packet_bytes = packet.data.encode('utf-8') - + if 'packet_bytes' in locals(): reveal_type(packet_bytes) # bytes socket.send(packet_bytes) @@ -361,7 +589,7 @@ Difficult to implement '''''''''''''''''''''' Eric Traut from the Pyright type checker team has stated that -implementing a ``Union[..., Missing]``-style syntax would be +implementing a ``Union[..., Missing]``-style notation would be difficult. [2]_ Introduces a second null-like value into Python @@ -379,9 +607,8 @@ distinguishing between its analogous constants ``null`` and Replace Optional with Nullable. Repurpose Optional to mean “optional item”. --------------------------------------------------------------------------- -``Optional[...]`` is too ubiquitous to deprecate. Although use of it -*may* fade over time in favor of the ``T|None`` syntax specified by `PEP -604 `__. +``Optional[]`` is too ubiquitous to deprecate, although use of it +*may* fade over time in favor of the ``T|None`` notation specified by :pep:`604`. Change Optional to mean “optional item” in certain contexts instead of “nullable” @@ -406,7 +633,7 @@ or: opt1: Optional[str] This would add more confusion for users because it would mean that in -*some* contexts the meaning of ``Optional[...]`` is different than in +*some* contexts the meaning of ``Optional[]`` is different than in other contexts, and it would be easy to overlook the flag. @@ -433,6 +660,7 @@ References .. [2] https://mail.python.org/archives/list/typing-sig@python.org/message/S2VJSVG6WCIWPBZ54BOJPG56KXVSLZK6/ +.. [3] https://bugs.python.org/issue46491 Copyright ========= diff --git a/pep-0656.rst b/peps/pep-0656.rst similarity index 98% rename from pep-0656.rst rename to peps/pep-0656.rst index 06d10c4f5..554589af5 100644 --- a/pep-0656.rst +++ b/peps/pep-0656.rst @@ -4,11 +4,12 @@ Author: Tzu-ping Chung Sponsor: Brett Cannon PEP-Delegate: Paul Moore Discussions-To: https://discuss.python.org/t/7165 -Status: Accepted -Type: Informational +Status: Final +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 17-Mar-2021 -Post-History: 17-Mar-2021 18-Apr-2021 +Post-History: 17-Mar-2021, 18-Apr-2021 Resolution: https://discuss.python.org/t/7165/32 diff --git a/pep-0657.rst b/peps/pep-0657.rst similarity index 100% rename from pep-0657.rst rename to peps/pep-0657.rst diff --git a/pep-0658.rst b/peps/pep-0658.rst similarity index 99% rename from pep-0658.rst rename to peps/pep-0658.rst index 6adcbcf8b..84b9834a5 100644 --- a/pep-0658.rst +++ b/peps/pep-0658.rst @@ -6,6 +6,7 @@ PEP-Delegate: Donald Stufft Discussions-To: https://discuss.python.org/t/8651 Status: Accepted Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 10-May-2021 Post-History: 10-May-2021 diff --git a/peps/pep-0659.rst b/peps/pep-0659.rst new file mode 100644 index 000000000..19b06a52a --- /dev/null +++ b/peps/pep-0659.rst @@ -0,0 +1,389 @@ +PEP: 659 +Title: Specializing Adaptive Interpreter +Author: Mark Shannon +Status: Draft +Type: Informational +Content-Type: text/x-rst +Created: 13-Apr-2021 +Post-History: 11-May-2021 + + +Abstract +======== + +In order to perform well, virtual machines for dynamic languages must +specialize the code that they execute to the types and values in the +program being run. This specialization is often associated with "JIT" +compilers, but is beneficial even without machine code generation. + +A specializing, adaptive interpreter is one that speculatively specializes +on the types or values it is currently operating on, and adapts to changes +in those types and values. + +Specialization gives us improved performance, and adaptation allows the +interpreter to rapidly change when the pattern of usage in a program alters, +limiting the amount of additional work caused by mis-specialization. + +This PEP proposes using a specializing, adaptive interpreter that specializes +code aggressively, but over a very small region, and is able to adjust to +mis-specialization rapidly and at low cost. + +Adding a specializing, adaptive interpreter to CPython will bring significant +performance improvements. It is hard to come up with meaningful numbers, +as it depends very much on the benchmarks and on work that has not yet happened. +Extensive experimentation suggests speedups of up to 50%. +Even if the speedup were only 25%, this would still be a worthwhile enhancement. + +Motivation +========== + +Python is widely acknowledged as slow. +Whilst Python will never attain the performance of low-level languages like C, +Fortran, or even Java, we would like it to be competitive with fast +implementations of scripting languages, like V8 for Javascript or luajit for +lua. +Specifically, we want to achieve these performance goals with CPython to +benefit all users of Python including those unable to use PyPy or +other alternative virtual machines. + +Achieving these performance goals is a long way off, and will require a lot of +engineering effort, but we can make a significant step towards those goals by +speeding up the interpreter. +Both academic research and practical implementations have shown that a fast +interpreter is a key part of a fast virtual machine. + +Typical optimizations for virtual machines are expensive, so a long "warm up" +time is required to gain confidence that the cost of optimization is justified. +In order to get speed-ups rapidly, without noticeable warmup times, +the VM should speculate that specialization is justified even after a few +executions of a function. To do that effectively, the interpreter must be able +to optimize and de-optimize continually and very cheaply. + +By using adaptive and speculative specialization at the granularity of +individual virtual machine instructions, +we get a faster interpreter that also generates profiling information +for more sophisticated optimizations in the future. + +Rationale +========= + +There are many practical ways to speed-up a virtual machine for a dynamic +language. +However, specialization is the most important, both in itself and as an +enabler of other optimizations. +Therefore it makes sense to focus our efforts on specialization first, +if we want to improve the performance of CPython. + +Specialization is typically done in the context of a JIT compiler, +but research shows specialization in an interpreter can boost performance +significantly, even outperforming a naive compiler [1]_. + +There have been several ways of doing this proposed in the academic +literature, but most attempt to optimize regions larger than a +single bytecode [1]_ [2]_. +Using larger regions than a single instruction requires code to handle +de-optimization in the middle of a region. +Specialization at the level of individual bytecodes makes de-optimization +trivial, as it cannot occur in the middle of a region. + +By speculatively specializing individual bytecodes, we can gain significant +performance improvements without anything but the most local, +and trivial to implement, de-optimizations. + +The closest approach to this PEP in the literature is +"Inline Caching meets Quickening" [3]_. +This PEP has the advantages of inline caching, +but adds the ability to quickly de-optimize making the performance +more robust in cases where specialization fails or is not stable. + +Performance +----------- + +The speedup from specialization is hard to determine, as many specializations +depend on other optimizations. Speedups seem to be in the range 10% - 60%. + +* Most of the speedup comes directly from specialization. The largest + contributors are speedups to attribute lookup, global variables, and calls. +* A small, but useful, fraction is from improved dispatch such as + super-instructions and other optimizations enabled by quickening. + +Implementation +============== + +Overview +-------- + +Any instruction that would benefit from specialization will be replaced by an +"adaptive" form of that instruction. When executed, the adaptive instructions +will specialize themselves in response to the types and values that they see. +This process is known as "quickening". + +Once an instruction in a code object has executed enough times, +that instruction will be "specialized" by replacing it with a new instruction +that is expected to execute faster for that operation. + +Quickening +---------- + +Quickening is the process of replacing slow instructions with faster variants. + +Quickened code has a number of advantages over immutable bytecode: + +* It can be changed at runtime. +* It can use super-instructions that span lines and take multiple operands. +* It does not need to handle tracing as it can fallback to the original + bytecode for that. + +In order that tracing can be supported, the quickened instruction format +should match the immutable, user visible, bytecode format: +16-bit instructions of 8-bit opcode followed by 8-bit operand. + +Adaptive instructions +--------------------- + +Each instruction that would benefit from specialization is replaced by an +adaptive version during quickening. For example, +the ``LOAD_ATTR`` instruction would be replaced with ``LOAD_ATTR_ADAPTIVE``. + +Each adaptive instruction periodically attempts to specialize itself. + +Specialization +-------------- + +CPython bytecode contains many instructions that represent high-level +operations, and would benefit from specialization. Examples include ``CALL``, +``LOAD_ATTR``, ``LOAD_GLOBAL`` and ``BINARY_ADD``. + +By introducing a "family" of specialized instructions for each of these +instructions allows effective specialization, +since each new instruction is specialized to a single task. +Each family will include an "adaptive" instruction, that maintains a counter +and attempts to specialize itself when that counter reaches zero. + +Each family will also include one or more specialized instructions that +perform the equivalent of the generic operation much faster provided their +inputs are as expected. +Each specialized instruction will maintain a saturating counter which will +be incremented whenever the inputs are as expected. Should the inputs not +be as expected, the counter will be decremented and the generic operation +will be performed. +If the counter reaches the minimum value, the instruction is de-optimized by +simply replacing its opcode with the adaptive version. + +Ancillary data +-------------- + +Most families of specialized instructions will require more information than +can fit in an 8-bit operand. To do this, a number of 16 bit entries immediately +following the instruction are used to store this data. This is a form of inline +cache, an "inline data cache". Unspecialized, or adaptive, instructions will +use the first entry of this cache as a counter, and simply skip over the others. + +Example families of instructions +-------------------------------- + +LOAD_ATTR +''''''''' + +The ``LOAD_ATTR`` instruction loads the named attribute of the object on top of the stack, +then replaces the object on top of the stack with the attribute. + +This is an obvious candidate for specialization. Attributes might belong to +a normal instance, a class, a module, or one of many other special cases. + +``LOAD_ATTR`` would initially be quickened to ``LOAD_ATTR_ADAPTIVE`` which +would track how often it is executed, and call the ``_Py_Specialize_LoadAttr`` +internal function when executed enough times, or jump to the original +``LOAD_ATTR`` instruction to perform the load. When optimizing, the kind +of the attribute would be examined, and if a suitable specialized instruction +was found, it would replace ``LOAD_ATTR_ADAPTIVE`` in place. + +Specialization for ``LOAD_ATTR`` might include: + +* ``LOAD_ATTR_INSTANCE_VALUE`` A common case where the attribute is stored in + the object's value array, and not shadowed by an overriding descriptor. +* ``LOAD_ATTR_MODULE`` Load an attribute from a module. +* ``LOAD_ATTR_SLOT`` Load an attribute from an object whose + class defines ``__slots__``. + +Note how this allows optimizations that complement other optimizations. +The ``LOAD_ATTR_INSTANCE_VALUE`` works well with the "lazy dictionary" used for +many objects. + +LOAD_GLOBAL +''''''''''' + +The ``LOAD_GLOBAL`` instruction looks up a name in the global namespace +and then, if not present in the global namespace, +looks it up in the builtins namespace. +In 3.9 the C code for the ``LOAD_GLOBAL`` includes code to check to see +whether the whole code object should be modified to add a cache, +whether either the global or builtins namespace, +code to lookup the value in a cache, and fallback code. +This makes it complicated and bulky. +It also performs many redundant operations even when supposedly optimized. + +Using a family of instructions makes the code more maintainable and faster, +as each instruction only needs to handle one concern. + +Specializations would include: + +* ``LOAD_GLOBAL_ADAPTIVE`` would operate like ``LOAD_ATTR_ADAPTIVE`` above. +* ``LOAD_GLOBAL_MODULE`` can be specialized for the case where the value is in + the globals namespace. After checking that the keys of the namespace have + not changed, it can load the value from the stored index. +* ``LOAD_GLOBAL_BUILTIN`` can be specialized for the case where the value is + in the builtins namespace. It needs to check that the keys of the global + namespace have not been added to, and that the builtins namespace has not + changed. Note that we don't care if the values of the global namespace + have changed, just the keys. + +See [4]_ for a full implementation. + +.. note:: + + This PEP outlines the mechanisms for managing specialization, and does not + specify the particular optimizations to be applied. + It is likely that details, or even the entire implementation, may change + as the code is further developed. + +Compatibility +============= + +There will be no change to the language, library or API. + +The only way that users will be able to detect the presence of the new +interpreter is through timing execution, the use of debugging tools, +or measuring memory use. + +Costs +===== + +Memory use +---------- + +An obvious concern with any scheme that performs any sort of caching is +"how much more memory does it use?". +The short answer is "not that much". + +Comparing memory use to 3.10 +'''''''''''''''''''''''''''' + +CPython 3.10 used 2 bytes per instruction, until the execution count +reached ~2000 when it allocates another byte per instruction and +32 bytes per instruction with a cache (``LOAD_GLOBAL`` and ``LOAD_ATTR``). + +The following table shows the additional bytes per instruction to support the +3.10 opcache or the proposed adaptive interpreter, on a 64 bit machine. + +================ ========== ========== ====== + Version 3.10 cold 3.10 hot 3.11 + Specialised 0% ~15% ~25% +---------------- ---------- ---------- ------ + code 2 2 2 + opcache_map 0 1 0 + opcache/data 0 4.8 4 +---------------- ---------- ---------- ------ + Total 2 7.8 6 +================ ========== ========== ====== + +``3.10 cold`` is before the code has reached the ~2000 limit. +``3.10 hot`` shows the cache use once the threshold is reached. + +The relative memory use depends on how much code is "hot" enough to trigger +creation of the cache in 3.10. The break even point, where the memory used +by 3.10 is the same as for 3.11 is ~70%. + +It is also worth noting that the actual bytecode is only part of a code +object. Code objects also include names, constants and quite a lot of +debugging information. + +In summary, for most applications where many of the functions are relatively +unused, 3.11 will consume more memory than 3.10, but not by much. + + +Security Implications +===================== + +None + + +Rejected Ideas +============== + +By implementing a specializing adaptive interpreter with inline data caches, +we are implicitly rejecting many alternative ways to optimize CPython. +However, it is worth emphasizing that some ideas, such as just-in-time +compilation, have not been rejected, merely deferred. + +Storing data caches before the bytecode. +---------------------------------------- + +An earlier implementation of this PEP for 3.11 alpha used a different caching +scheme as described below: + + + Quickened instructions will be stored in an array (it is neither necessary not + desirable to store them in a Python object) with the same format as the + original bytecode. Ancillary data will be stored in a separate array. + + Each instruction will use 0 or more data entries. + Each instruction within a family must have the same amount of data allocated, + although some instructions may not use all of it. + Instructions that cannot be specialized, e.g. ``POP_TOP``, + do not need any entries. + Experiments show that 25% to 30% of instructions can be usefully specialized. + Different families will need different amounts of data, + but most need 2 entries (16 bytes on a 64 bit machine). + + In order to support larger functions than 256 instructions, + we compute the offset of the first data entry for instructions + as ``(instruction offset)//2 + (quickened operand)``. + + Compared to the opcache in Python 3.10, this design: + + * is faster; it requires no memory reads to compute the offset. + 3.10 requires two reads, which are dependent. + * uses much less memory, as the data can be different sizes for different + instruction families, and doesn't need an additional array of offsets. + can support much larger functions, up to about 5000 instructions + per function. 3.10 can support about 1000. + +We rejected this scheme as the inline cache approach is both faster +and simpler. + +References +========== + +.. [1] The construction of high-performance virtual machines for + dynamic languages, Mark Shannon 2011. + https://theses.gla.ac.uk/2975/1/2011shannonphd.pdf + +.. [2] Dynamic Interpretation for Dynamic Scripting Languages + https://www.scss.tcd.ie/publications/tech-reports/reports.09/TCD-CS-2009-37.pdf + +.. [3] Inline Caching meets Quickening + https://www.unibw.de/ucsrl/pubs/ecoop10.pdf/view + +.. [4] The adaptive and specialized instructions are implemented in + https://github.com/python/cpython/blob/main/Python/ceval.c + + The optimizations are implemented in: + https://github.com/python/cpython/blob/main/Python/specialize.c + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-0660.rst b/peps/pep-0660.rst similarity index 93% rename from pep-0660.rst rename to peps/pep-0660.rst index e18b86631..7547eadeb 100644 --- a/pep-0660.rst +++ b/peps/pep-0660.rst @@ -3,17 +3,19 @@ Title: Editable installs for pyproject.toml based builds (wheel based) Author: Daniel Holth , StĂ©phane Bidoul Sponsor: Paul Moore Discussions-To: https://discuss.python.org/t/draft-pep-editable-installs-for-pep-517-style-build-backends/8510 -Status: Accepted +Status: Final Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 30-Mar-2021 Post-History: -Resolution: https://discuss.python.org/t/pronouncement-on-peps-660-and-662-editable-installs +Resolution: https://discuss.python.org/t/pronouncement-on-peps-660-and-662-editable-installs/9450 + Abstract ======== -This document describes a PEP 517 style method for the installation of packages +This document describes a :pep:`517` style method for the installation of packages in editable mode. Motivation @@ -33,14 +35,14 @@ The installation of projects in such a way that the python code being imported remains in the source directory is known as the *editable* installation mode. -Now that PEP 517 provides a mechanism to create alternatives to setuptools, and +Now that :pep:`517` provides a mechanism to create alternatives to setuptools, and decouple installation front ends from build backends, we need a new mechanism to install packages in editable mode. Rationale ========= -PEP 517 deferred "Editable installs", meaning non-``setup.py`` +:pep:`517` deferred "Editable installs", meaning non-``setup.py`` distributions lacked that feature. The only way to retain ``editable`` installs for these distributions was to provide a compatible ``setup.py develop`` implementation. By defining an editable hook other build frontends gain @@ -79,7 +81,7 @@ encouraged to document such potential differences. The Mechanism ============= -This PEP adds three optional hooks to the PEP 517 backend interface. These hooks +This PEP adds three optional hooks to the :pep:`517` backend interface. These hooks are used to build a wheel that, when installed, allows that distribution to be imported from its source folder. @@ -109,7 +111,7 @@ Build-backends must produce wheels that have the same dependencies with the exception that they can add dependencies necessary for their editable mechanism to function at runtime (such as `editables`_). -The filename for the "editable" wheel needs to be PEP 427 compliant too. It +The filename for the "editable" wheel needs to be :pep:`427` compliant too. It does not need to use the same tags as ``build_wheel`` but it must be tagged as compatible with the system. @@ -135,7 +137,7 @@ get_requires_for_build_editable def get_requires_for_build_editable(config_settings=None): ... -This hook MUST return an additional list of strings containing PEP 508 +This hook MUST return an additional list of strings containing :pep:`508` dependency specifications, above and beyond those specified in the ``pyproject.toml`` file, to be installed when calling the ``build_editable`` hooks. @@ -206,7 +208,7 @@ Frontends must install "editable" wheels in the same way as regular wheels. This also means uninstallation of editables does not require any special treatment. Frontends must create a ``direct_url.json`` file in the ``.dist-info`` -directory of the installed distribution, in compliance with PEP 610. The +directory of the installed distribution, in compliance with :pep:`610`. The ``url`` value must be a ``file://`` url pointing to the project directory (i.e. the directory containing ``pyproject.toml``), and the ``dir_info`` value must be ``{'editable': true}``. @@ -269,14 +271,14 @@ identifier. In other words ``pkg==1.0+local`` is not satisfied by version Virtual wheel ------------- -Another approach was proposed in PEP 662, where +Another approach was proposed in :pep:`662`, where the build backend returns a mapping from source files and directories to the installed layout. It is then up to the installer frontend to realize the editable installation by whatever means it deems adequate for its users. In terms of capabilities, both proposals provide the core "editable" feature. -The key difference is that PEP 662 leaves it to the frontend to decide how the +The key difference is that :pep:`662` leaves it to the frontend to decide how the editable installation will be realized, while with this PEP, the choice must be made by the backend. Both approaches can in principle provide several editable installation methods for a given project, and let the developer choose one at @@ -287,14 +289,14 @@ range of theoretical and practical expectations about editable installs. The reality is that the only one there is wide experience with is path insertion via .pth (i.e. what setup.py develop does). -We believe that PEP 660 better addresses these "unknown unknowns" today in the +We believe that :pep:`660` better addresses these "unknown unknowns" today in the most reliable way, by letting project authors select the backend or implement the method that provides the editable mechanism that best suit their requirements, and test it works correctly. Since the frontend has no latitude in *how* to install the "editable" wheel, in case of issue, there is only one place to investigate: the build backend. -With PEP 662, issues need to be investigated in the frontend, +With :pep:`662`, issues need to be investigated in the frontend, the backend and possiblty the specification. There is also a high probability that different frontends, implementing the specification in different ways, will produce installations that behave differently than project authors @@ -308,7 +310,7 @@ A `prototype `_ was made that created an unpacked wheel in a temporary directory, to be copied to the target environment by the frontend. This approach was not pursued because a wheel archive is easy to create for the backend, and using a wheel as communication -mechanism is a better fit with the PEP 517 philosophy, and therefore keeps +mechanism is a better fit with the :pep:`517` philosophy, and therefore keeps things simpler for the frontend. References diff --git a/peps/pep-0661.rst b/peps/pep-0661.rst new file mode 100644 index 000000000..e999168ac --- /dev/null +++ b/peps/pep-0661.rst @@ -0,0 +1,400 @@ +PEP: 661 +Title: Sentinel Values +Author: Tal Einat +Discussions-To: https://discuss.python.org/t/pep-661-sentinel-values/9126 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 06-Jun-2021 +Post-History: 06-Jun-2021 + + +TL;DR: See the `Specification`_ and `Reference Implementation`_. + + +Abstract +======== + +Unique placeholder values, commonly known as "sentinel values", are common in +programming. They have many uses, such as for: + +* Default values for function arguments, for when a value was not given:: + + def foo(value=None): + ... + +* Return values from functions when something is not found or unavailable:: + + >>> "abc".find("d") + -1 + +* Missing data, such as NULL in relational databases or "N/A" ("not + available") in spreadsheets + +Python has the special value ``None``, which is intended to be used as such +a sentinel value in most cases. However, sometimes an alternative sentinel +value is needed, usually when it needs to be distinct from ``None``. These +cases are common enough that several idioms for implementing such sentinels +have arisen over the years, but uncommon enough that there hasn't been a +clear need for standardization. However, the common implementations, +including some in the stdlib, suffer from several significant drawbacks. + +This PEP proposes adding a utility for defining sentinel values, to be used +in the stdlib and made publicly available as part of the stdlib. + +Note: Changing all existing sentinels in the stdlib to be implemented this +way is not deemed necessary, and whether to do so is left to the discretion +of the maintainers. + + +Motivation +========== + +In May 2021, a question was brought up on the python-dev mailing list +[1]_ about how to better implement a sentinel value for +``traceback.print_exception``. The existing implementation used the +following common idiom:: + + _sentinel = object() + +However, this object has an uninformative and overly verbose repr, causing the +function's signature to be overly long and hard to read:: + + >>> help(traceback.print_exception) + Help on function print_exception in module traceback: + + print_exception(exc, /, value=, tb=, + limit=None, file=None, chain=True) + +Additionally, two other drawbacks of many existing sentinels were brought up +in the discussion: + +1. Not having a distinct type, hence it being impossible to define clear + type signatures for functions with sentinels as default values +2. Incorrect behavior after being copied or unpickled, due to a separate + instance being created and thus comparisons using ``is`` failing + +In the ensuing discussion, Victor Stinner supplied a list of currently used +sentinel values in the Python standard library [2]_. This showed that the +need for sentinels is fairly common, that there are various implementation +methods used even within the stdlib, and that many of these suffer from at +least one of the three aforementioned drawbacks. + +The discussion did not lead to any clear consensus on whether a standard +implementation method is needed or desirable, whether the drawbacks mentioned +are significant, nor which kind of implementation would be good. The author +of this PEP created an issue on bugs.python.org [3]_ suggesting options for +improvement, but that focused on only a single problematic aspect of a few +cases, and failed to gather any support. + +A poll [4]_ was created on discuss.python.org to get a clearer sense of +the community's opinions. The poll's results were not conclusive, with 40% +voting for "The status-quo is fine / there’s no need for consistency in +this", but most voters voting for one or more standardized solutions. +Specifically, 37% of the voters chose "Consistent use of a new, dedicated +sentinel factory / class / meta-class, also made publicly available in the +stdlib". + +With such mixed opinions, this PEP was created to facilitate making a decision +on the subject. + +While working on this PEP, iterating on various options and implementations +and continuing discussions, the author has come to the opinion that a simple, +good implementation available in the standard library would be worth having, +both for use in the standard library itself and elsewhere. + + +Rationale +========= + +The criteria guiding the chosen implementation were: + +1. The sentinel objects should behave as expected by a sentinel object: When + compared using the ``is`` operator, it should always be considered + identical to itself but never to any other object. +2. Creating a sentinel object should be a simple, straightforward one-liner. +3. It should be simple to define as many distinct sentinel values as needed. +4. The sentinel objects should have a clear and short repr. +5. It should be possible to use clear type signatures for sentinels. +6. The sentinel objects should behave correctly after copying and/or + unpickling. +7. Such sentinels should work when using CPython 3.x and PyPy3, and ideally + also with other implementations of Python. +8. As simple and straightforward as possible, in implementation and especially + in use. Avoid this becoming one more special thing to learn when learning + Python. It should be easy to find and use when needed, and obvious enough + when reading code that one would normally not feel a need to look up its + documentation. + +With so many uses in the Python standard library [2]_, it would be useful to +have an implementation in the standard library, since the stdlib cannot use +implementations of sentinel objects available elsewhere (such as the +``sentinels`` [5]_ or ``sentinel`` [6]_ PyPI packages). + +After researching existing idioms and implementations, and going through many +different possible implementations, an implementation was written which meets +all of these criteria (see `Reference Implementation`_). + + +Specification +============= + +A new ``Sentinel`` class will be added to a new ``sentinels`` module. +Its initializer will accept a single required argument, the name of the +sentinel object, and two optional arguments: the repr of the object, and the +name of its module:: + + >>> from sentinel import Sentinel + >>> NotGiven = Sentinel('NotGiven') + >>> NotGiven + + >>> MISSING = Sentinel('MISSING', repr='mymodule.MISSING') + >>> MISSING + mymodule.MISSING + >>> MEGA = Sentinel('MEGA', repr='', module_name='mymodule') + + +Checking if a value is such a sentinel *should* be done using the ``is`` +operator, as is recommended for ``None``. Equality checks using ``==`` will +also work as expected, returning ``True`` only when the object is compared +with itself. Identity checks such as ``if value is MISSING:`` should usually +be used rather than boolean checks such as ``if value:`` or ``if not value:``. +Sentinel instances are truthy by default. + +The names of sentinels are unique within each module. When calling +``Sentinel()`` in a module where a sentinel with that name was already +defined, the existing sentinel with that name will be returned. Sentinels +with the same name in different modules will be distinct from each other. + +Creating a copy of a sentinel object, such as by using ``copy.copy()`` or by +pickling and unpickling, will return the same object. + +Type annotations for sentinel values should use ``Sentinel``. For example:: + + def foo(value: int | Sentinel = MISSING) -> int: + ... + +The ``module_name`` optional argument should normally not need to be supplied, +as ``Sentinel()`` will usually be able to recognize the module in which it was +called. ``module_name`` should be supplied only in unusual cases when this +automatic recognition does not work as intended, such as perhaps when using +Jython or IronPython. This parallels the designs of ``Enum`` and +``namedtuple``. For more details, see :pep:`435`. + +The ``Sentinel`` class may be sub-classed. Instances of each sub-class will +be unique, even if using the same name and module. This allows for +customizing the behavior of sentinels, such as controlling their truthiness. + + +Reference Implementation +======================== + +The reference implementation is found in a dedicated GitHub repo [7]_. A +simplified version follows:: + + _registry = {} + + class Sentinel: + """Unique sentinel values.""" + + def __new__(cls, name, repr=None, module_name=None): + name = str(name) + repr = str(repr) if repr else f'<{name.split(".")[-1]}>' + if module_name is None: + try: + module_name = \ + sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + module_name = __name__ + + registry_key = f'{module_name}-{name}' + + sentinel = _registry.get(registry_key, None) + if sentinel is not None: + return sentinel + + sentinel = super().__new__(cls) + sentinel._name = name + sentinel._repr = repr + sentinel._module_name = module_name + + return _registry.setdefault(registry_key, sentinel) + + def __repr__(self): + return self._repr + + def __reduce__(self): + return ( + self.__class__, + ( + self._name, + self._repr, + self._module_name, + ), + ) + + +Rejected Ideas +============== + + +Use ``NotGiven = object()`` +--------------------------- + +This suffers from all of the drawbacks mentioned in the `Rationale`_ section. + + +Add a single new sentinel value, such as ``MISSING`` or ``Sentinel`` +-------------------------------------------------------------------- + +Since such a value could be used for various things in various places, one +could not always be confident that it would never be a valid value in some use +cases. On the other hand, a dedicated and distinct sentinel value can be used +with confidence without needing to consider potential edge-cases. + +Additionally, it is useful to be able to provide a meaningful name and repr +for a sentinel value, specific to the context where it is used. + +Finally, this was a very unpopular option in the poll [4]_, with only 12% +of the votes voting for it. + + +Use the existing ``Ellipsis`` sentinel value +-------------------------------------------- + +This is not the original intended use of Ellipsis, though it has become +increasingly common to use it to define empty class or function blocks instead +of using ``pass``. + +Also, similar to a potential new single sentinel value, ``Ellipsis`` can't be +as confidently used in all cases, unlike a dedicated, distinct value. + + +Use a single-valued enum +------------------------ + +The suggested idiom is:: + + class NotGivenType(Enum): + NotGiven = 'NotGiven' + NotGiven = NotGivenType.NotGiven + +Besides the excessive repetition, the repr is overly long: +````. A shorter repr can be defined, at +the expense of a bit more code and yet more repetition. + +Finally, this option was the least popular among the nine options in the +poll [4]_, being the only option to receive no votes. + + +A sentinel class decorator +-------------------------- + +The suggested idiom is:: + + @sentinel(repr='') + class NotGivenType: pass + NotGiven = NotGivenType() + +While this allows for a very simple and clear implementation of the decorator, +the idiom is too verbose, repetitive, and difficult to remember. + + +Using class objects +------------------- + +Since classes are inherently singletons, using a class as a sentinel value +makes sense and allows for a simple implementation. + +The simplest version of this is:: + + class NotGiven: pass + +To have a clear repr, one would need to use a meta-class:: + + class NotGiven(metaclass=SentinelMeta): pass + +... or a class decorator:: + + @Sentinel + class NotGiven: pass + +Using classes this way is unusual and could be confusing. The intention of +code would be hard to understand without comments. It would also cause +such sentinels to have some unexpected and undesirable behavior, such as +being callable. + + +Define a recommended "standard" idiom, without supplying an implementation +-------------------------------------------------------------------------- + +Most common existing idioms have significant drawbacks. So far, no idiom +has been found that is clear and concise while avoiding these drawbacks. + +Also, in the poll [4]_ on this subject, the options for recommending an +idiom were unpopular, with the highest-voted option being voted for by only +25% of the voters. + + +Specific type signatures for each sentinel value +------------------------------------------------ + +For a long time, the author of this PEP strove to have type signatures for +such sentinels that were specific to each value. A leading proposal +(supported by Guido and others) was to expand the use of ``Literal``, e.g. +``Literal[MISSING]``. After much thought and discussion, especially on the +typing-sig mailing list [8]_, it seems that all such solutions would require +special-casing and/or added complexity in the implementations of static type +checkers, while also constraining the implementation of sentinels. + +Therefore, this PEP no longer proposes such signatures. Instead, this PEP +suggests using ``Sentinel`` as the type signature for sentinel values. + +It is somewhat unfortunate that static type checkers will sometimes not be +able to deduce more specific types due to this, such as inside a conditional +block like ``if value is not MISSING: ...``. However, this is a minor issue +in practice, as type checkers can be easily made to understand these cases +using ``typing.cast()``. + + +Additional Notes +================ + +* This PEP and the initial implementation are drafted in a dedicated GitHub + repo [7]_. + +* For sentinels defined in a class scope, to avoid potential name clashes, + one should use the fully-qualified name of the variable in the module. Only + the part of the name after the last period will be used for the default + repr. For example:: + + >>> class MyClass: + ... NotGiven = sentinel('MyClass.NotGiven') + >>> MyClass.NotGiven + + +* One should be careful when creating sentinels in a function or method, since + sentinels with the same name created by code in the same module will be + identical. If distinct sentinel objects are needed, make sure to use + distinct names. + + +References +========== + +.. [1] Python-Dev mailing list: `The repr of a sentinel `_ +.. [2] Python-Dev mailing list: `"The stdlib contains tons of sentinels" `_ +.. [3] `bpo-44123: Make function parameter sentinel values true singletons `_ +.. [4] discuss.python.org Poll: `Sentinel Values in the Stdlib `_ +.. [5] `The "sentinels" package on PyPI `_ +.. [6] `The "sentinel" package on PyPI `_ +.. [7] `Reference implementation at the taleinat/python-stdlib-sentinels GitHub repo `_ +.. [8] `Discussion thread about type signatures for these sentinels on the typing-sig mailing list `_ + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0662.rst b/peps/pep-0662.rst similarity index 98% rename from pep-0662.rst rename to peps/pep-0662.rst index 1567ec77a..54b83157d 100644 --- a/pep-0662.rst +++ b/peps/pep-0662.rst @@ -5,10 +5,12 @@ Sponsor: Brett Cannon Discussions-To: https://discuss.python.org/t/discuss-tbd-editable-installs-by-gaborbernat/9071 Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 28-May-2021 Post-History: -Resolution: https://discuss.python.org/t/pronouncement-on-peps-660-and-662-editable-installs +Resolution: https://discuss.python.org/t/pronouncement-on-peps-660-and-662-editable-installs/9450 + Abstract ======== @@ -138,7 +140,7 @@ For reference, a non-editable installation works as follows: :pep:`518` environment. Once invoked, the backend returns a wheel. #. The frontend takes the wheel and feeds it to an **installer** - (e.g.,`installer`_) to install the wheel into the target Python interpreter. + (e.g., installer_) to install the wheel into the target Python interpreter. The Mechanism ============= @@ -268,7 +270,7 @@ the frontend, and is encouraged for the frontend to communicate with the user exactly the method chosen, and what limitations that solution will have. The frontend must create a ``direct_url.json`` file in the ``.dist-info`` -directory of the installed distribution, in compliance with PEP 610. The ``url`` +directory of the installed distribution, in compliance with :pep:`610`. The ``url`` value must be a ``file://`` URL pointing to the project directory (i.e., the directory containing ``pyproject.toml``), and the ``dir_info`` value must be ``{'editable': true}``. diff --git a/pep-0662/pep-0662-editable.json b/peps/pep-0662/pep-0662-editable.json similarity index 100% rename from pep-0662/pep-0662-editable.json rename to peps/pep-0662/pep-0662-editable.json diff --git a/pep-0663.txt b/peps/pep-0663.rst similarity index 99% rename from pep-0663.txt rename to peps/pep-0663.rst index 1850c3a07..dfcc4bc21 100644 --- a/pep-0663.txt +++ b/peps/pep-0663.rst @@ -4,13 +4,13 @@ Version: $Revision$ Last-Modified: $Date$ Author: Ethan Furman Discussions-To: python-dev@python.org -Status: Draft +Status: Rejected Type: Informational Content-Type: text/x-rst Created: 30-Jun-2021 Python-Version: 3.11 Post-History: 20-Jul-2021, 02-Nov-2021 -Resolution: +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/RN3WCRZSTQR55DOHJTZ2KIO6CZPJPCU7/ Abstract diff --git a/peps/pep-0664.rst b/peps/pep-0664.rst new file mode 100644 index 000000000..6530677bc --- /dev/null +++ b/peps/pep-0664.rst @@ -0,0 +1,131 @@ +PEP: 664 +Title: Python 3.11 Release Schedule +Version: $Revision$ +Last-Modified: $Date$ +Author: Pablo Galindo Salgado +Status: Active +Type: Informational +Topic: Release +Content-Type: text/x-rst +Created: 12-Jul-2021 +Python-Version: 3.11 + + +Abstract +======== + +This document describes the development and release schedule for +Python 3.11. The schedule primarily concerns itself with PEP-sized +items. + +.. Small features may be added up to the first beta + release. Bugs may be fixed until the final release, + which is planned for October 2022. + +Release Manager and Crew +======================== + +- 3.11 Release Manager: Pablo Galindo Salgado +- Windows installers: Steve Dower +- Mac installers: Ned Deily +- Documentation: Julien Palard + + +Release Schedule +================ + +3.11.0 schedule +--------------- + +Note: the dates below use a 17-month development period that results +in a 12-month release cadence between feature versions, as defined by +:pep:`602`. + +Actual: + +- 3.11 development begins: Monday, 2021-05-03 +- 3.11.0 alpha 1: Tuesday, 2021-10-05 +- 3.11.0 alpha 2: Tuesday, 2021-11-02 +- 3.11.0 alpha 3: Wednesday, 2021-12-08 +- 3.11.0 alpha 4: Friday, 2022-01-14 +- 3.11.0 alpha 5: Thursday, 2022-02-03 +- 3.11.0 alpha 6: Monday, 2022-03-07 +- 3.11.0 alpha 7: Tuesday, 2022-04-05 +- 3.11.0 beta 1: Sunday, 2022-05-08 + (No new features beyond this point.) +- 3.11.0 beta 2: Tuesday, 2022-05-31 +- 3.11.0 beta 3: Wednesday, 2022-06-01 +- 3.11.0 beta 4: Monday, 2022-07-11 +- 3.11.0 beta 5: Tuesday, 2022-07-26 +- 3.11.0 candidate 1: Monday, 2022-08-08 +- 3.11.0 candidate 2: Monday, 2022-09-12 +- 3.11.0 final: Monday, 2022-10-24 + +Bugfix releases +--------------- + +Actual: + +- 3.11.1: Tuesday, 2022-12-06 +- 3.11.2: Wednesday, 2023-02-08 +- 3.11.3: Wednesday, 2023-04-05 +- 3.11.4: Tuesday, 2023-06-06 +- 3.11.5: Thursday, 2023-08-24 + +Expected: + +- 3.11.6: Monday, 2023-10-02 +- 3.11.7: Monday, 2023-12-04 +- 3.11.8: Monday, 2024-02-05 + +Final regular bugfix release with binary installers: + +- 3.11.9: Monday, 2024-04-01 + + +3.11 Lifespan +------------- + +3.11 will receive bugfix updates approximately every 2 months for +approximately 18 months. Some time after the release of 3.12.0 final, +the ninth and final 3.11 bugfix update will be released. After that, +it is expected that security updates (source only) will be released +until 5 years after the release of 3.11.0 final, so until approximately +October 2027. + + +Features for 3.11 +================= + +Some of the notable features of Python 3.11 include: + +* :pep:`654`, Exception Groups and ``except*``. +* :pep:`657`, Enhanced error locations in tracebacks. +* :pep:`680`, Support for parsing TOML in the standard library +* Python 3.11 is up to 10-60% faster than Python 3.10. On average, we measured + a 1.25x speedup on the standard benchmark suite. See `Faster CPython + `__ for + details. + +Typing features: + +* :pep:`646`, Variadic generics. +* :pep:`655`, Marking individual TypedDict items as required or potentially-missing. +* :pep:`673`, Self type. +* :pep:`675`, Arbitrary literal string type. +* :pep:`681`, Dataclass transforms + +Copyright +========= + +This document has been placed in the public domain. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 72 + coding: utf-8 + End: diff --git a/pep-0665.rst b/peps/pep-0665.rst similarity index 59% rename from pep-0665.rst rename to peps/pep-0665.rst index 8f39bbab6..923dc83d9 100644 --- a/pep-0665.rst +++ b/peps/pep-0665.rst @@ -5,26 +5,32 @@ Author: Brett Cannon , Tzu-ping Chung PEP-Delegate: Paul Moore Discussions-To: https://discuss.python.org/t/9911 -Status: Draft +Status: Rejected Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 29-Jul-2021 -Post-History: 29-Jul-2021, 03-Nov-2021 -Resolution: +Post-History: 29-Jul-2021, 03-Nov-2021, 25-Nov-2021 +Resolution: https://discuss.python.org/t/pep-665-take-2-a-file-format-to-list-python-dependencies-for-reproducibility-of-an-application/11736/140 + +.. note:: + This PEP was rejected due to lukewarm reception from the community + from the lack of source distribution support. ======== Abstract ======== This PEP specifies a file format to specify the list of Python package -installation requirements for an application, and the relation between the -specified requirements. The list of requirements is considered +installation requirements for an application, and the relation between +the specified requirements. The list of requirements is considered exhaustive for the installation target, and thus not requiring any information beyond the platform being installed for, and the file itself. The file format is flexible enough to allow installing the -requirements across different platforms, which guarantees +requirements across different platforms, which allows for reproducibility on multiple platforms from the same file. + =========== Terminology =========== @@ -38,18 +44,18 @@ import system. The packages on PyPI are an example of this. An *application* or *app* is an end product that other external code does not directly rely on via the import system (i.e. they are standalone). Desktop applications, command-line tools, etc. are -examples. +examples of applications. A *lock file* records the packages that are to be installed for an app. Traditionally, the exact version of the package to be installed -is specified by a lock file, but all specified packages are not always +is specified by a lock file, but specified packages are not always installed on a given platform (according a filtering logic described in a later section), which enables the lock file to describe reproducibility across multiple platforms. Examples of this are -``package-lock.json`` from npm_, ``Poetry.lock`` from Poetry, etc. +``package-lock.json`` from npm_, ``Poetry.lock`` from Poetry_, etc. *Locking* is the act of taking the input of the packages an app -depends on and producting a lock file from that. +depends on and producing a lock file from that. A *locker* is a tool which produces a lock file. @@ -85,9 +91,20 @@ and reproducibility on any one specific platform. Three, reproducibility is more secure. When you control exactly what files are installed, you can make sure no malicious actor is attempting to slip nefarious code into your application (i.e. some -supply chain attacks). By making a lock file which always leads to +supply chain attacks). By using a lock file which always leads to reproducible installs, we can avoid certain risks entirely. +Four, relying on the `wheel file`_ format provides reproducibility +without requiring build tools to support reproducibility themselves. +Thanks to wheels being static and not executing code as part of +installation, wheels always lead to a reproducible result. Compare +this to source distributions (aka sdists) or source trees which only +lead to a reproducible install if their build tool supports +reproducibility due to inherent code execution. Unfortunately the vast +majority of build tools do not support reproducible builds, so this +PEP helps alleviate that issue by only supporting wheels as a package +format. + This PEP proposes a standard for a lock file, as the current solutions don't meet the outlined goals. Today, the closest we come to a lock file standard is the `requirements file format`_ from pip. @@ -106,10 +123,10 @@ file formats: #. Pyflow_ Unfortunately, those tools all use differing lock file formats. This -means tooling around these tools much be unique. This impacts tooling +means tooling around these tools must be unique. This impacts tooling such as code editors and hosting providers, which want to be as flexible as possible when it comes to accepting a user's application -code, but also have a limit as to how much development resource they +code, but also have a limit as to how much development resources they can spend to add support for yet another lock file format. A standardized format would allow tools to focus their work on a single target, and make sure that workflow decisions made by developers @@ -138,7 +155,7 @@ File Format ----------- We wanted the file format to be easy to read as a diff when auditing -a change to the lock file. As such, and thanks to PEP 518 and +a change to the lock file. As such, and thanks to :pep:`518` and ``pyproject.toml``, we decided to go with the TOML_ file format. @@ -149,23 +166,27 @@ Secure by Design Viewing the `requirements file format`_ as the closest we have to a lock file standard, there are a few issues with the file format when it comes to security. First is that the file format simply does not -require you specify the exact version of a package. This is why -tools like `pip-tools`_ exist to help manage that for the user. +require you to specify the exact version of a package. This is why +tools like `pip-tools`_ exist to help manage that users of +requirements files. + +Second, you must opt into specifying what files are acceptable to be +installed by using the ``--hash`` argument for a specific dependency. +This is also optional with pip-tools as it requires specifying the +``--generate-hashes`` CLI argument. This requires ``--require-hashes`` +for pip to make sure no dependencies lack a hash to check. -Second, you must opt into specifying what files may be installed by -using the ``--hash`` argument for a specific dependency. This is also -optional with pip-tools as it requires specifying the -``--generate-hashes`` CLI argument. Third, even when you control what files may be installed, it does not prevent other packages from being installed. If a dependency is not listed in the requirements file, pip will happily go searching for a -file to meet that need, unless you specify ``--no-deps`` as an -argument. +file to meet that need. You must specify ``--no-deps`` as an +argument to pip to prevent unintended dependency resolution outside +of the requirements file. Fourth, the format allows for installing a `source distribution file`_ (aka "sdist"). By its very nature, -installing an sdist may imply executing arbitrary Python code, meaning +installing an sdist requires executing arbitrary Python code, meaning that there is no control over what files may be installed. Only by specifying ``--only-binary :all:`` can you guarantee pip to only use a `wheel file`_ for each package. @@ -175,17 +196,20 @@ being proposed, a user should always do the following steps: #. Use pip-tools and its command ``pip-compile --generate-hashes`` #. Install the requirements file using - ``pip install --no-deps --only-binary :all:`` + ``pip install --require-hashes --no-deps --only-binary :all:`` -Critically, all of those flags, and both specificity and exhaustion of -what to install that pip-tools provides, are optional. +Critically, all of those flags, and both the specificity and +exhaustion of what to install that pip-tools provides, are optional +for requirements files. -As such, the proposal raised in this PEP is secure by design to combat -some supply chain attacks. Hashes for files which would be used to -install from are **required**. You can **only** install from wheels -to unambiguously define what files will be placed in the file system. -Installers **must** have an unambiguous installation from a lock file -for a given platform. +As such, the proposal raised in this PEP is secure by design which +combats some supply chain attacks. Hashes for files which would be +used to install from are **required**. You can **only** install from +wheels to unambiguously define what files will be placed in the file +system. Installers **must** lead to an deterministic installation +from a lock file for a given platform. All of this leads to a +reproducible installation which you can deem trustworthy (when you +have audited the lock file and what it lists). -------------- @@ -195,17 +219,17 @@ Cross-Platform Various projects which already have a lock file, like PDM_ and Poetry_, provide a lock file which is *cross-platform*. This allows for a single lock file to work on multiple platforms while still -leading to exact same top-level requirements to be installed -everywhere while the installation being consistent/unambiguous on +leading to the exact same top-level requirements to be installed +everywhere with the installation being consistent/unambiguous on each platform. As to why this is useful, let's use an example involving PyWeek_ -(a week-long game development competition). We assume you are -developing on Linux, while someone you choose to partner with is -using macOS. Now assume the judges are using Windows. How do you make -sure everyone is using the same top-level dependencies, while allowing -for any platform-specific requirements (e.g. a package requires a -helper package under Windows)? +(a week-long game development competition). Assume you are developing +on Linux, while someone you choose to partner with is using macOS. +Now assume the judges are using Windows. How do you make sure everyone +is using the same top-level dependencies, while allowing for any +platform-specific requirements (e.g. a package requires a helper +package under Windows)? With a cross-platform lock file, you can make sure that the key requirements are met consistently across all platforms. You can then @@ -221,7 +245,7 @@ The separation of concerns between a locker and an installer allows for an installer to have a much simpler operation to perform. As such, it not only allows for installers to be easier to write, but facilitates in making sure installers create unambiguous, reproducible -installations. +installations correctly. The installer can also expend less computation/energy in creating the installation. This is beneficial not only for faster installs, but @@ -229,8 +253,9 @@ also from an energy consumption perspective, as installers are expected to be run more often than lockers. This has led to a design where the locker must do more work upfront -to benefit installers. It also means the complexity of package -dependencies is simpler and easier to comprehend to avoid ambiguity. +to the benefit installers. It also means the complexity of package +dependencies is simpler and easier to comprehend in a lock files to +avoid ambiguity. ============= @@ -242,19 +267,19 @@ Details ------- Lock files MUST use the TOML_ file format. This not only prevents the -need to have another file format in the Python packaging ecosystem, -thanks to its adoption by PEP 518 for ``pyproject.toml``, but also +need to have another file format in the Python packaging ecosystem +thanks to its adoption by :pep:`518` for ``pyproject.toml``, but also assists in making lock files more human-readable. Lock files MUST end their file names with ``.pylock.toml``. The ``.toml`` part unambiguously distinguishes the format of the file, and helps tools like code editors support the file appropriately. The ``.pylock`` part distinguishes the file from other TOML files the user -has, to make logic easier for tools to create functionalities specific -to Python lock files, instead of TOML files in general. +has, to make the logic easier for tools to create functionality +specific to Python lock files, instead of TOML files in general. The following sections are the top-level keys of the TOML file data -format. Any field not listed as required is considered optional. +format. Any field not listed as **required** is considered optional. ``version`` @@ -264,15 +289,16 @@ This field is **required**. The version of the lock file being used. The key MUST be a string consisting of a number that follows the same formatting as the -``Metadata-Version`` key in the `core metadata spec`_. The value MUST -be set to ``"1.0"`` until a future PEP allows for a different value. -The introduction of a new *optional* key SHOULD increase the minor -version. The introduction of a new required key or changing the -format MUST increase the major version. How to handle other scenarios -is left as a per-PEP decision. +``Metadata-Version`` key in the `core metadata spec`_. + +The value MUST be set to ``"1.0"`` until a future PEP allows for a +different value. The introduction of a new *optional* key to the file +format SHOULD increase the minor version. The introduction of a new +required key or changing the format MUST increase the major version. +How to handle other scenarios is left as a per-PEP decision. Installers MUST warn the user if the lock file specifies a version -whose major version is support but whose minor version is +whose major version is supported but whose minor version is unsupported/unrecognized (e.g. the installer supports ``"1.0"``, but the lock file specifies ``"1.1"``). @@ -291,9 +317,8 @@ native timestamp type). It MUST be recorded using the UTC time zone to avoid ambiguity. If the SOURCE_DATE_EPOCH_ environment variable is set, it MUST be used -as the timestamp by the locker. This faciliates reproducibility of the -lock file itself. - +as the timestamp by the locker. This facilitates reproducibility of +the lock file itself. ``[tool]`` @@ -318,13 +343,13 @@ A table containing data applying to the overall lock file. A key storing a string containing an environment marker as specified in the `dependency specifier spec`_. - The locker MAY specify an environment marker which specifies any restrictions the lock file was generated under. If the installer is installing for an environment which does not satisfy the specified environment marker, the installer MUST raise an -error as the lock file does not support the environment. +error as the lock file does not support the target installation +environment. ``metadata.tag`` @@ -333,12 +358,10 @@ error as the lock file does not support the environment. A key storing a string specifying `platform compatibility tags`_ (i.e. wheel tags). The tag MAY be a compressed tag set. -The locker MAY specify a tag (set) which specify which platform(s) -the lock file supports. - If the installer is installing for an environment which does not satisfy the specified tag (set), the installer MUST raise an error -as the lock file does not support the environment. +as the lock file does not support the targeted installation +environment. ``metadata.requires`` @@ -354,7 +377,7 @@ and thus the root of the dependency graph. ``metadata.requires-python`` ---------------------------- -A string specifying the support version(s) of Python for this lock +A string specifying the supported version(s) of Python for this lock file. It follows the same format as that specified for the ``Requires-Python`` field in the `core metadata spec`_. @@ -364,11 +387,11 @@ file. It follows the same format as that specified for the This array is **required**. -An array per package and version containing details for the potential +An array per package and version containing entries for the potential (wheel) files to install (as represented by ``_name_`` and ``_version_``, respectively). -Lockers must MUST normalize a project's name according to the +Lockers MUST normalize a project's name according to the `simple repository API`_. If extras are specified as part of the project to install, the extras are to be included in the key name and are to be sorted in lexicographic order. @@ -379,23 +402,11 @@ Within the file, the tables for the projects SHOULD be sorted by: #. Package version, newest/highest to older/lowest according to the `version specifiers spec`_ #. Optional dependencies (extras) via lexicographic order -#. File name based on the ``filename`` or ``url`` field (discussed +#. File name based on the ``filename`` field (discussed below) -All of this is to help minimize diff changes between tool executions. - - -``package._name_._version_.url`` --------------------------------- - -A string representing a URL where to get the file. - -The installer MAY support any schemes it wants for URLs -(e.g. ``file:`` as well as ``https:``). - -An installer MAY choose to not use the URL to retrieve a file -if a file matching the specified hash can be found using some -alternative means (e.g. on the file system in a cache directory). +These recommendations are to help minimize diff changes between tool +executions. ``package._name_._version_.filename`` @@ -403,11 +414,45 @@ alternative means (e.g. on the file system in a cache directory). This field is **required**. -A string representing the name of the file as represented by an entry -in the array. This field is required to simplify installers as the -file name is required to resolve wheel tags derived from the file -name. It also guarantees that the association of the array entry to -the file it is meant for is always clear. +A string representing the base name of the file as represented by an +entry in the array (i.e. what +``os.path.basename()``/``pathlib.PurePath.name`` represents). This +field is required to simplify installers as the file name is required +to resolve wheel tags derived from the file name. It also guarantees +that the association of the array entry to the file it is meant for is +always clear. + + +``[package._name_._version_.hashes]`` +------------------------------------- + +This table is **required**. + +A table with keys specifying a hash algorithm and values as the hash +for the file represented by this entry in the +``package._name_._version_`` table. + +Lockers SHOULD list hashes in lexicographic order. This is to help +minimize diff sizes and the potential to overlook hash value changes. + +An installer MUST only install a file which matches one of the +specified hashes. + + +``package._name_._version_.url`` +-------------------------------- + +A string representing a URL where to get the file. + +The installer MAY support any schemes it wants for URLs. A URL with no +scheme MUST be assumed to be a local file path (both relative paths to +the lock file and absolute paths). Installers MUST support, at +minimum, HTTPS URLs as well as local file paths. + +An installer MAY choose to not use the URL to retrieve a file +if a file matching the specified hash can be found using alternative +means (e.g. on the file system in a cache directory). + ``package._name_._version_.direct`` ----------------------------------- @@ -421,20 +466,12 @@ If the key is true, then the installer MUST follow the the installation as "direct". -``[package._name_._version_.hashes]`` -------------------------------------- +``package._name_._version_.requires-python`` +-------------------------------------------- -This table is **required**. - -A table with keys specifying hash algorithms and values as the hash -for the file represented by this entry in the -``package._name_._version_`` table. - -Lockers SHOULD list hashes in lexicographic order. This is to help -minimize diff sizes and the potential to overlook hash value changes. - -An installer MUST only install a file which matches one of the -specified hashes. +A string specifying the support version(s) of Python for this file. It +follows the same format as that specified for the +``Requires-Python`` field in the `core metadata spec`_. ``package._name_._version_.requires`` @@ -444,14 +481,6 @@ An array of strings following the `dependency specifier spec`_ which represent the dependencies of this file. -``package._name_._version_.requires-python`` --------------------------------------------- - -A string specifying the support version(s) of Python for this file. It -follows the same format as that specified for the -``Requires-Python`` field in the `core metadata spec`_. - - ------- Example ------- @@ -462,35 +491,67 @@ Example created-at = 2021-10-19T22:33:45.520739+00:00 [tool] - # Tool-specific table ala PEP 518's `[tool]` table. - + # Tool-specific table. [metadata] - requires = ["mousebender"] - requires-python = ">=3.6" + requires = ["mousebender", "coveragepy[toml]"] + marker = "sys_platform == 'linux'" # As an example for coverage. + requires-python = ">=3.7" [[package.attrs."21.2.0"]] - url = "https://files.pythonhosted.org/packages/20/a9/ba6f1cd1a1517ff022b35acd6a7e4246371dfab08b8e42b829b6d07913cc/attrs-21.2.0-py2.py3-none-any.whl" filename = "attrs-21.2.0-py2.py3-none-any.whl" hashes.sha256 = "149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1" + url = "https://files.pythonhosted.org/packages/20/a9/ba6f1cd1a1517ff022b35acd6a7e4246371dfab08b8e42b829b6d07913cc/attrs-21.2.0-py2.py3-none-any.whl" + requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + + [[package.attrs."21.2.0"]] + # If attrs had another wheel file (e.g. that was platform-specific), + # it could be listed here. + + [[package."coveragepy[toml]"."6.2.0"]] + filename = "coverage-6.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + hashes.sha256 = "c7912d1526299cb04c88288e148c6c87c0df600eca76efd99d84396cfe00ef1d" + url = "https://files.pythonhosted.org/packages/da/64/468ca923e837285bd0b0a60bd9a287945d6b68e325705b66b368c07518b1/coverage-6.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl" + requires-python = ">=3.6" + requires = ["tomli"] + + [[package."coveragepy[toml]"."6.2.0"]] + filename = "coverage-6.2-cp310-cp310-musllinux_1_1_x86_64.whl " + hashes.sha256 = "276651978c94a8c5672ea60a2656e95a3cce2a3f31e9fb2d5ebd4c215d095840" + url = "https://files.pythonhosted.org/packages/17/d6/a29f2cccacf2315150c31d8685b4842a6e7609279939a478725219794355/coverage-6.2-cp310-cp310-musllinux_1_1_x86_64.whl" + requires-python = ">=3.6" + requires = ["tomli"] + + # More wheel files for `coverage` could be listed for more + # extensive support (i.e. all Linux-based wheels). [[package.mousebender."2.0.0"]] - url = "https://files.pythonhosted.org/packages/f4/b3/f6fdbff6395e9b77b5619160180489410fb2f42f41272994353e7ecf5bdf/mousebender-2.0.0-py3-none-any.whl" filename = "mousebender-2.0.0-py3-none-any.whl" hashes.sha256 = "a6f9adfbd17bfb0e6bb5de9a27083e01dfb86ed9c3861e04143d9fd6db373f7c" + url = "https://files.pythonhosted.org/packages/f4/b3/f6fdbff6395e9b77b5619160180489410fb2f42f41272994353e7ecf5bdf/mousebender-2.0.0-py3-none-any.whl" + requires-python = ">=3.6" requires = ["attrs", "packaging"] [[package.packaging."20.9"]] - url = "https://files.pythonhosted.org/packages/3e/89/7ea760b4daa42653ece2380531c90f64788d979110a2ab51049d92f408af/packaging-20.9-py2.py3-none-any.whl" filename = "packaging-20.9-py2.py3-none-any.whl" hashes.blake-256 = "3e897ea760b4daa42653ece2380531c90f64788d979110a2ab51049d92f408af" hashes.sha256 = "67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" + url = "https://files.pythonhosted.org/packages/3e/89/7ea760b4daa42653ece2380531c90f64788d979110a2ab51049d92f408af/packaging-20.9-py2.py3-none-any.whl" + requires-python = ">=3.6" requires = ["pyparsing"] [[package.pyparsing."2.4.7"]] - url = "https://files.pythonhosted.org/packages/8a/bb/488841f56197b13700afd5658fc279a2025a39e22449b7cf29864669b15d/pyparsing-2.4.7-py2.py3-none-any.whl" filename = "pyparsing-2.4.7-py2.py3-none-any.whl" - hashes.sha256="ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + hashes.sha256 = "ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + url = "https://files.pythonhosted.org/packages/8a/bb/488841f56197b13700afd5658fc279a2025a39e22449b7cf29864669b15d/pyparsing-2.4.7-py2.py3-none-any.whl" + direct = true # For demonstration purposes. + requires-python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + + [[package.tomli."2.0.0"]] + filename = "tomli-2.0.0-py3-none-any.whl" + hashes.sha256 = "b5bde28da1fed24b9bd1d4d2b8cba62300bfb4ec9a6187a957e8ddb9434c5224" + url = "https://files.pythonhosted.org/packages/e2/9f/5e1557a57a7282f066351086e78f87289a3446c47b2cb5b8b2f614d8fe99/tomli-2.0.0-py3-none-any.whl" + requires-python = ">=3.7" ------------------------ @@ -500,22 +561,19 @@ Expectations for Lockers Lockers MUST create lock files for which a topological sort of the packages which qualify for installation on the specified platform results in a graph for which only a single version of any package -is possible and there is at least one compatible file to install for -those packages. This equates to a lock file that which is acceptable -based on ``metadata.marker``, ``metadata.tag``, and -``metadata.requires-python`` will have a list of package versions -after evaluating environment markers and eliminating unsupported -files for which the only decision the installer will need to make is -which file to use for the package (which is outlined below). +qualifies for installation and there is at least one compatible file +to install for each package. This leads to a lock file for any +supported platform where the only decision an installer can make +is what the "best-fitting" wheel is to install (which is discussed +below). -This means that lockers are expected to utilize ``metadata.marker``, -``metadata.tag``, and ``metadata.requires-python`` as appropriate -as well as environment markers specified via ``requires`` and Python -version requirements via ``requires-python`` to enforce this result -for installers. Put another way, the information used in the lock -file is not expected to be pristine/raw from the locker's input and -instead is to be changed as necessary to the benefit of the locker's -goals. +Lockers are expected to utilize ``metadata.marker``, ``metadata.tag``, +and ``metadata.requires-python`` as appropriate as well as environment +markers specified via ``requires`` and Python version requirements via +``requires-python`` to enforce this result for installers. Put another +way, the information used in the lock file is not expected to be +pristine/raw from the locker's input and instead is to be changed as +necessary to the benefit of the locker's goals. --------------------------- @@ -526,23 +584,25 @@ The expected algorithm for resolving what to install is: #. Construct a dependency graph based on the data in the lock file with ``metadata.requires`` as the starting/root point. -#. Eliminate all (wheel) files that are unsupported by the specified - platform. +#. Eliminate all files that are unsupported by the specified platform. #. Eliminate all irrelevant edges between packages based on marker - evaluation. + evaluation for ``requires``. #. Raise an error if a package version is still reachable from the - root of the dependency graph but lacks any compatible (wheel) - file. + root of the dependency graph but lacks any compatible file. #. Verify that all packages left only have one version to install, raising an error otherwise. #. Install the best-fitting wheel file for each package which remains. -What constitues the "best-fitting wheel file" is an open issue. +Installers MUST follow a deterministic algorithm determine what the +"best-fitting wheel file" is. A simple solution for this is to +rely upon the `packaging project `__ +and its ``packaging.tags`` module to determine wheel file precedence. Installers MUST support installing into an empty environment. Installers MAY support installing into an environment that already -conatins installed packages (and whatever that would entail). +contains installed packages (and whatever that would entail to be +supported). ======================== @@ -563,6 +623,17 @@ Pyflow_ has said they `"like the idea" `__ of the PEP. +Poetry_ has said they would **not** support the PEP as-is because +`"Poetry supports sdists files, directory and VCS dependencies which are not supported" `__. +Recording requirements at the file level, which is on purpose to +better reflect what can occur when it comes to dependencies, +`"is contradictory to the design of Poetry" `__. +This also excludes export support to a this PEP's lock file as +`"Poetry exports the information present in the poetry.lock file into another format" `__ +and sdists and source trees are included in ``Poetry.lock`` files. +Thus it is not a clean translation from Poetry's lock file to this +PEP's lock file format. + ======================= Backwards Compatibility @@ -582,11 +653,96 @@ For projects which do document their lock file format like pipenv_, they will very likely need a major version release which changes the lock file format. -Specifically for Poetry_, it has an -`export command `_ which -should allow Poetry to support this lock file format even if the -project chooses not to adopt this PEP as Poetry's primary lock file -format. + +=============== +Transition Plan +=============== + +In general, this PEP could be considered successful if: + +#. Two pre-existing tools became lockers (e.g. `pip-tools`_, PDM_, + pip_ via ``pip freeze``). +#. Pip became an installer. +#. One major, non-Python-specific platform supported the file format + (e.g. a cloud provider). + +This would show interoperability, usability, and programming +community/business acceptance. + +In terms of a transition plan, there are potentially multiple steps +that could lead to this desired outcome. Below is a somewhat idealized +plan that would see this PEP being broadly used. + + +--------- +Usability +--------- + +First, a ``pip freeze`` equivalent tool could be developed which +creates a lock file. While installed packages do not by themselves +provide enough information to statically create a lock file, a user +could provide local directories and index URLs to construct one. This +would then lead to lock files that are stricter than a requirements +file by limiting the lock file to the current platform. This would +also allow people to see whether their environment would be +reproducible. + +Second, a stand-alone installer should be developed. As the +requirements on an installer are much simpler than what pip provides, +it should be reasonable to have an installer that is independently +developed. + +Third, a tool to convert a pinned requirements file as emitted by +pip-tools could be developed. Much like the ``pip freeze`` equivalent +outlined above, some input from the user may be needed. But this tool +could act as a transitioning step for anyone who has an appropriate +requirements file. This could also act as a test before potentially +having pip-tools grow some ``--lockfile`` flag to use this PEP. + +All of this could be required before the PEP transitions from +conditional acceptance to full acceptance (and give the community a +chance to test if this PEP is potentially useful). + + +---------------- +Interoperability +---------------- + +At this point, the goal would be to increase interoperability between +tools. + +First, pip would become an installer. By having the most widely used +installer support the format, people can innovate on the locker side +while knowing people will have the tools necessary to actually consume +a lock file. + +Second, pip becomes a locker. Once again, pip's reach would make the +format accessible for the vast majority of Python users very quickly. + +Third, a project with a pre-existing lock file format supports at +least exporting to the lock file format (e.g. PDM or Pyflow). This +would show that the format meets the needs of other projects. + + +---------- +Acceptance +---------- + +With the tooling available throughout the community, acceptance would +be shown via those not exclusively tied to the Python community +supporting the file format based on what they believe their users +want. + +First, tools that operate on requirements files like code editors +having equivalent support for lock files. + +Second, consumers of requirements files like cloud providers would +also accept lock files. + +At this point the PEP would have permeated out far enough to be on +par with requirements files in terms of general acceptance and +potentially more if projects had dropped their own lock files for this +PEP. ===================== @@ -596,8 +752,10 @@ Security Implications A lock file should not introduce security issues but instead help solve them. By requiring the recording of hashes for files, a lock file is able to help prevent tampering with code since the hash -details were recorded. A lock file also helps prevent unexpected -package updates being installed which may be malicious. +details were recorded. Relying on only wheel files means what files +will be installed can be known ahead of time and is reproducible. A +lock file also helps prevent unexpected package updates being +installed which may in turn be malicious. ================= @@ -616,10 +774,10 @@ files. Reference Implementation ======================== -No proof-of-concept or reference implementation currently exists. An -example locker and installer will be provided before this PEP is -fully accepted (although this is not a necessarily a requirement for -conditional acceptance). +A proof-of-concept locker can be found at +https://github.com/frostming/pep665_poc . No installer has been +implemented yet, but the design of this PEP suggests the locker is the +more difficult aspect to implement. ============== @@ -642,7 +800,7 @@ tools already use a TOML parser thanks to ``pyproject.toml`` so this issue did not seem to be a showstopper. Some have also argued against this concern in the past by the fact that if packaging tools abhor installing dependencies and feel they can't vendor a package then the -packaging ecosystem has much bigger issues to rectify than needing to +packaging ecosystem has much bigger issues to rectify than the need to depend on a third-party TOML parser. @@ -687,7 +845,7 @@ installers did not have to make any decisions about *what* to install, only validating that the lock file would work for the target platform. This idea was eventually rejected due to the number of combinations -of potential PEP 508 environment markers. The decision was made that +of potential :pep:`508` environment markers. The decision was made that trying to have lockers generate all possible combinations as individual lock files when a project wants to be cross-platform would be too much. @@ -718,18 +876,18 @@ as the term. Accepting PEP 650 ----------------- -PEP 650 was an earlier attempt at trying to tackle this problem by -specifying an API for installers instead of standardizing on a lock file -format (ala PEP 517). The +:pep:`650` was an earlier attempt at trying to tackle this problem by +specifying an API for installers instead of standardizing on a lock +file format (ala :pep:`517`). The `initial response `__ -to PEP 650 could be considered mild/lukewarm. People seemed to be -consistently confused over which tools should provide what functionality -to implement the PEP. It also potentially incurred more overhead as -it would require executing Python APIs to perform any actions involving -packaging. +to :pep:`650` could be considered mild/lukewarm. People seemed to be +consistently confused over which tools should provide what +functionality to implement the PEP. It also potentially incurred more +overhead as it would require executing Python APIs to perform any +actions involving packaging. This PEP chooses to standardize around an artifact instead of an API -(ala PEP 621). This would allow for more tool integrations as it +(ala :pep:`621`). This would allow for more tool integrations as it removes the need to specifically use Python to do things such as create a lock file, update it, or even install packages listed in a lock file. It also allows for easier introspection by forcing @@ -737,8 +895,10 @@ dependency graph details to be written in a human-readable format. It also allows for easier sharing of knowledge by standardizing what people need to know more (e.g. tutorials become more portable between tools when it comes to understanding the artifact they produce). It's -also simply the approach other language communities have taken and seem -to be happy with. +also simply the approach other language communities have taken and +seem to be happy with. + +Acceptance of this PEP would mean :pep:`650` gets rejected. ------------------------------------------------------- @@ -746,93 +906,57 @@ Specifying Requirements per Package Instead of per File ------------------------------------------------------- An earlier draft of this PEP specified dependencies at the package -level instead of per (wheel) file. While this has traditionally been -how packaging systems work, it actually did not reflect accurately -how things are specified. As such, this PEP was subsequently updated -to reflect the granularity that dependencies can truly be specified -at. - - -=========== -Open Issues -=========== - - -------------------------------------------------------------------------------------- -Allowing Source Distributions and Source Trees to be an Opt-In, Supported File Format -------------------------------------------------------------------------------------- - -For security reproducibility reasons this PEP only considers -supporting installation from wheel files. Installing from either an -sdist or source tree requires arbitrary code execution during -installation, unknown files to be installed, and an unknown set of -dependencies. Those issues all run counter to guaranteeing users get -the same files for the same platform as well as making sure they are -receiving the expected files. - -To deal with this issue, people would need to build their own wheels -from sdists and cache them. Then the lockers would record the hashes -of those wheels and the installers would then be expected to use -those wheels. - -Another option is to allow sdists (and potentially source trees) be -listed as support file formats, but have them marked as insecure in -the lock file and require the installer force the user to opt into -using insecure file formats. Unfortunately because sdists which don't -necessarily follow version 2.2 of the `core metadata spec`_ for their -``PKG-INFO`` file will have unknown dependencies, breaking the -guarantee that results will be reproducible thanks to potential -arbitrary calculations of those dependencies. And even if an sdist did -follow the latest spec, they could still list their requirements as -dynamic, still making it impossible to statically know what should be -installed. As such, installers would either have to have a full -resolver to handle these dynamic cases or only sdists which follow -version 2.2 of the core metadata spec **and** statically specify -their dependencies could be listed. But at that point the project is -probably capable of providing wheels, making support for sdists that -much less important/useful. +level instead of per file. While this has traditionally been how +packaging systems work, it actually did not reflect accurately how +things are specified. As such, this PEP was subsequently updated to +reflect the granularity that dependencies can truly be specified at. ---------------------------------- Specify Where Lockers Gather Input ---------------------------------- -This PEP currently does not specify how a locker gets its input. It -could be possible to support a subset of PEP 621 such that -``project.requires-python`` and ``project.dependencies`` are read -from ``pyproject.toml`` and automatically used as input if provided. -But this or some other practice could also be left as something to -grow organically in the community and making that the standard at a -later date. +This PEP does not specify how a locker gets its input. An initial +suggestion was to partially reuse :pep:`621`, but due to disagreements +on how flexible the potential input should be in terms of specifying +things such as indexes, etc., it was decided this would best be left +to a separate PEP. ------------------------------------- -What is a "best-fitting wheel file"? ------------------------------------- +------------------------------------------------------------------------------------- +Allowing Source Distributions and Source Trees to be an Opt-In, Supported File Format +------------------------------------------------------------------------------------- -The expected steps of installing a package much include decided which -wheel file to install as a package may have a universal wheel on top -of very specific wheels. But as `platform compatibility tags`_ do not -specify how to determine priority and there is no way to use -environment markers to specify an exact wheel, there's no defined way -for an installer to deterministically determine what wheel file to -select. +After `extensive discussion `__, +it was decided that this PEP would not support source distributions +(aka sdists) or source trees as an acceptable format for code. +Introducing sdists and source trees to this PEP would immediately undo +the reproducibility and security goals due to needing to execute code +to build the sdist or source tree. It would also greatly increase +the complexity for (at least) installers as the dynamic build nature +of sdists and source trees means the installer would need to handle +fully resolving whatever requirements the sdists produced dynamically, +both from a building and installation perspective. -There are two possible solutions. One is for the locker to specify a -ranking/priority order to the wheel files. That way the installer -simply sorts to the supported wheel files by that order and installs -the the top rated/ranked wheel file. This puts the priority order -under the control of the locker. +Due to all of this, it was decided it was best to have a separate +discussion about what supporting sdists and source trees **after** +this PEP is accepted/rejected. As the proposed file format is +versioned, introducing sdists and source tree support in a later PEP +is doable. -The other option is to specify in this PEP how to calculate the -priority/ranking of wheel files. This is currently tool-based and -seems to have been acceptable overall by the community, but having a -specification for this would probably still be welcome. It may be -somewhat disruptive, though, as it could change what files get -installed by tools which implement the ordering outside of the -context of this PEP. And if this PEP gains traction, it is reasonable -to assume that users will expect the ordering to be consistent across -tools. +It should be noted, though, that this PEP is **not** stop an +out-of-band solution from being developed to be used in conjunction +with this PEP. Building wheel files from sdists and shipping them with +code upon deployment so they can be included in the lock file is one +option. Another is to use a requirements file *just* for sdists and +source trees, then relying on a lock file for all wheels. + + +=========== +Open Issues +=========== + +None. =============== @@ -840,15 +964,15 @@ Acknowledgments =============== Thanks to Frost Ming of PDM_ and SĂ©bastien Eustace of Poetry_ for -providing input around dynamic install-time resolution of PEP 508 +providing input around dynamic install-time resolution of :pep:`508` requirements. Thanks to Kushal Das for making sure reproducible builds stayed a concern for this PEP. Thanks to Andrea McInnes for initially settling the bikeshedding and -choosing the paint colour of ``needs`` (at which point that caused -people to rally around the ``requires`` colour). +choosing the paint colour of ``needs`` (at which point people ralled +around the ``requires`` colour instead). ========= diff --git a/pep-0666.txt b/peps/pep-0666.rst similarity index 90% rename from pep-0666.txt rename to peps/pep-0666.rst index 8c30fb09b..dd3bdce23 100644 --- a/pep-0666.txt +++ b/peps/pep-0666.rst @@ -1,14 +1,12 @@ PEP: 666 Title: Reject Foolish Indentation -Version: $Revision$ -Last-Modified: $Date$ -Author: lac@strakt.com (Laura Creighton) +Author: Laura Creighton Status: Rejected Type: Standards Track Content-Type: text/x-rst Created: 03-Dec-2001 Python-Version: 2.2 -Post-History: 5-Dec-2001 +Post-History: 05-Dec-2001 Abstract @@ -75,7 +73,7 @@ than whether it is too hostile for newcomers. Possibly somebody could get around to explaining to me what is the difference between ``__getattr__`` and ``__getattribute__`` in non-Classic classes in 2.2, a question I have foolishly posted in the middle of the current tab -thread. I would like to know the answer to that question [2]_. +thread. I would like to know the answer to that question [1]_. This proposal, if accepted, will probably mean a heck of a lot of work for somebody. But since I don't want it accepted, I don't @@ -85,10 +83,7 @@ care. References ========== -.. [1] PEP 1, PEP Purpose and Guidelines - http://www.python.org/dev/peps/pep-0001/ - -.. [2] Tim Peters already has (private correspondence). My early 2.2 +.. [1] Tim Peters already has (private correspondence). My early 2.2 didn't have a ``__getattribute__``, and ``__getattr__`` was implemented like ``__getattribute__`` now is. This has been fixed. The important conclusion is that my Decorator Pattern @@ -99,12 +94,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - fill-column: 70 - End: diff --git a/pep-0667.rst b/peps/pep-0667.rst similarity index 75% rename from pep-0667.rst rename to peps/pep-0667.rst index 82ef24db8..abf8d7386 100644 --- a/pep-0667.rst +++ b/peps/pep-0667.rst @@ -5,6 +5,7 @@ Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 30-Jul-2021 +Python-Version: 3.13 Post-History: 20-Aug-2021 @@ -118,33 +119,57 @@ For example:: y print(locals(), x) -``test()`` will print ``{'x': 2, 'y': 4, 'z': 5} 2`` +``test()`` will print ``{'x': 2, 'y': 4, 'z': 5} 2``. -In Python 3.10, the above will fail with a ``NameError``, +In Python 3.10, the above will fail with an ``UnboundLocalError``, as the definition of ``y`` by ``l()['y'] = 4`` is lost. +If the second-to-last line were changed from ``y`` to ``z``, this would be a +``NameError``, as it is today. Keys added to ``frame.f_locals`` that are not +lexically local variables remain visible in ``frame.f_locals``, but do not +dynamically become local variables. + C-API ----- Extensions to the API ''''''''''''''''''''' -Two new C-API functions will be added:: +Four new C-API functions will be added:: - PyObject *PyEval_Locals(void) + PyObject *PyEval_GetFrameLocals(void) + PyObject *PyEval_GetFrameGlobals(void) + PyObject *PyEval_GetFrameBuiltins(void) PyObject *PyFrame_GetLocals(PyFrameObject *f) -``PyEval_Locals()`` is equivalent to: ``locals()``. +``PyEval_GetFrameLocals()`` is equivalent to: ``locals()``. +``PyEval_GetFrameGlobals()`` is equivalent to: ``globals()``. ``PyFrame_GetLocals(f)`` is equivalent to: ``f.f_locals``. -Both functions will return a new reference. +All these functions will return a new reference. Changes to existing APIs '''''''''''''''''''''''' -The C-API function ``PyEval_GetLocals()`` will be deprecated. -``PyEval_Locals()`` should be used instead. +The following C-API functions will be deprecated, as they return borrowed references:: + + PyEval_GetLocals() + PyEval_GetGlobals() + PyEval_GetBuiltins() + +They will be removed in 3.15. + +The following functions should be used instead:: + + PyEval_GetFrameLocals() + PyEval_GetFrameGlobals() + PyEval_GetFrameBuiltins() + +which return new references. + +The semantics of ``PyEval_GetLocals()`` is changed as it now returns a +view of the frame locals, not a dictionary. The following three functions will become no-ops, and will be deprecated:: @@ -152,7 +177,7 @@ The following three functions will become no-ops, and will be deprecated:: PyFrame_FastToLocals() PyFrame_LocalsToFast() -The above four deprecated functions will be removed in 3.13. +They will be removed in 3.15. Behavior of f_locals for optimized functions -------------------------------------------- @@ -186,10 +211,7 @@ PyEval_GetLocals Because ``PyEval_GetLocals()`` returns a borrowed reference, it requires the dictionary to be cached on the frame, extending its lifetime and -forces memory to be allocated for the frame object on the heap as well. - -Using ``PyEval_Locals()`` will be much more efficient -than ``PyEval_GetLocals()``. +creating a cycle. ``PyEval_GetFrameLocals()`` should be used instead. This code:: @@ -201,7 +223,7 @@ This code:: should be replaced with:: - locals = PyEval_Locals(); + locals = PyEval_GetFrameLocals(); if (locals == NULL) { goto error_handler; } @@ -209,7 +231,7 @@ should be replaced with:: PyFrame_FastToLocals, etc. '''''''''''''''''''''''''' -These functions were designed to convert the internal "fast" representation +These functions were designed to convert the internal "fast" representation of the locals variables of a function to a dictionary, and vice versa. Calls to them are no longer required. C code that directly accesses the @@ -268,9 +290,11 @@ They serve only to illustrate the proposed design. _locals : array[Object] # The values of the local variables, items may be NULL. _extra_locals: dict | NULL # Dictionary for storing extra locals not in _locals. + _locals_cache: FrameLocalsProxy | NULL # required to support PyEval_GetLocals() def __init__(self, ...): self._extra_locals = NULL + self._locals_cache = NULL ... @property @@ -278,6 +302,7 @@ They serve only to illustrate the proposed design. return FrameLocalsProxy(self) class FrameLocalsProxy: + "Implements collections.MutableMapping." __slots__ "_frame" @@ -332,24 +357,11 @@ They serve only to illustrate the proposed design. continue yield name - def pop(self): + def __contains__(self, item): f = self._frame - co = f.f_code - if f._extra_locals: - return f._extra_locals.pop() - for index, _ in enumerate(co._variable_names): - val = f._locals[index] - if val is NULL: - continue - if index in co._cells: - cell = val - val = cell.cell_contents - if val is NULL: - continue - cell.cell_contents = NULL - else: - f._locals[index] = NULL - return val + if item in f._extra_locals: + return True + return item in co._variable_names def __len__(self): f = self._frame @@ -372,76 +384,79 @@ C API PyObject *PyEval_GetLocals(void) { PyFrameObject * = ...; // Get the current frame. - Py_CLEAR(frame->_locals_cache); - frame->_locals_cache = PyEval_Locals(); + if (frame->_locals_cache == NULL) { + frame->_locals_cache = PyEval_GetFrameLocals(); + } return frame->_locals_cache; } As with all functions that return a borrowed reference, care must be taken to ensure that the reference is not used beyond the lifetime of the object. +Impact on PEP 709 inlined comprehensions +======================================== + +For inlined comprehensions within a function, ``locals()`` currently behaves the +same inside or outside of the comprehension, and this will not change. The +behavior of ``locals()`` inside functions will generally change as specified in +the rest of this PEP. + +For inlined comprehensions at module or class scope, currently calling +``locals()`` within the inlined comprehension returns a new dictionary for each +call. This PEP will make ``locals()`` within a function also always return a new +dictionary for each call, improving consistency; class or module scope inlined +comprehensions will appear to behave as if the inlined comprehension is still a +distinct function. + Comparison with PEP 558 ======================= -This PEP and PEP 558 [2]_ share a common goal: +This PEP and :pep:`558` share a common goal: to make the semantics of ``locals()`` and ``frame.f_locals()`` intelligible, and their operation reliable. -In the author's opinion, PEP 558 fails to do that as it is too -complex, and has many corner cases which will lead to bugs. -The key difference between this PEP and PEP 558 is that -PEP 558 requires an internal copy of the local variables, +The key difference between this PEP and :pep:`558` is that +:pep:`558` keeps an internal copy of the local variables, whereas this PEP does not. -Maintaining a copy adds considerably to the complexity of both -the specification and implementation, and brings no real benefits. -The semantics of ``frame.f_locals`` ------------------------------------ - -In this PEP, ``frame.f_locals`` is a view onto the underlying frame. -It is always synchronized with the underlying frame. -In PEP 558, there is an additional copy of the local variables present -in the frame which is updated whenever ``frame.f_locals`` is accessed. -PEP 558 does not make it clear whether calls to ``locals()`` -update ``frame.f_locals`` or not. - -For example consider:: - - def foo(): - x = sys._getframe().f_locals - y = locals() - print(tuple(x)) - print(tuple(y)) - -It is not clear from PEP 558 (at time of writing) what would be printed. -Does the call to ``locals()`` update ``x``? -Would ``"y"`` be present in either ``x`` or ``y``? - -With this PEP it should be clear that the above would print:: - - ('x', 'y') - ('x',) +:pep:`558` does not specify exactly when the internal copy is +updated, making the behavior of :pep:`558` impossible to reason about. Open Issues =========== +Have locals() return a mapping proxy +------------------------------------ + An alternative way to define ``locals()`` would be simply as:: def locals(): return sys._getframe(1).f_locals This would be simpler and easier to understand. However, -there would be backwards compatibility issues when ``locals`` is assigned -to a local variable or when passed to ``eval`` or ``exec``. +there would be backwards compatibility issues when ``locals()`` is assigned +to a local variable or passed to ``eval`` or ``exec``. + +Lifetime of the mapping proxy +----------------------------- + +Each read of the ``f_locals`` attributes creates a new mapping proxy. +This is done to avoid creating a reference cycle. + +An alternative would be to cache the proxy on the frame, so that +``frame.f_locals is frame.f_locals`` would be true. +The downside of this is that the reference cycle would delay collection +of both the frame and mapping proxy until the next cycle collection. + +``PyEval_GetLocals()`` already creates a cycle, as it returns a borrowed reference. + References ========== .. [1] https://bugs.python.org/issue30744 -.. [2] https://www.python.org/dev/peps/pep-0558/ - Copyright ========= diff --git a/pep-0668.rst b/peps/pep-0668.rst similarity index 96% rename from pep-0668.rst rename to peps/pep-0668.rst index 6361a6ed0..45f0968bd 100644 --- a/pep-0668.rst +++ b/peps/pep-0668.rst @@ -1,19 +1,22 @@ PEP: 668 -Title: Graceful cooperation between external and Python package managers +Title: Marking Python base environments as “externally managed” Author: Geoffrey Thomas , Matthias Klose , Filipe LaĂ­ns , Donald Stufft , - Tzu-Ping Chung , + Tzu-ping Chung , Stefano Rivera , Elana Hashman , Pradyun Gedam -Discussions-To: https://discuss.python.org/t/graceful-cooperation-between-external-and-python-package-managers-pep-668/10302 -Status: Draft -Type: Informational +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/10302 +Status: Accepted +Type: Standards Track +Topic: Packaging Content-Type: text/x-rst Created: 18-May-2021 Post-History: 28-May-2021 +Resolution: https://discuss.python.org/t/10302/44 Abstract ======== @@ -98,7 +101,7 @@ package Python-specific package manager A tool for installing, upgrading, and/or removing Python packages in a manner that conforms to Python packaging standards (such as - PEP 376 and PEP 427). The most popular Python-specific package + :pep:`376` and :pep:`427`). The most popular Python-specific package manager is pip [#pip]_; other examples include the old Easy Install command [#easy-install]_ as well as direct usage of a ``setup.py`` command. @@ -114,7 +117,7 @@ distro package manager packages in an installed instance of that distro, which is capable of installing Python packages as well as non-Python packages, and therefore generally has its own database of installed software - unrelated to PEP 376. Examples include ``apt``, ``dpkg``, ``dnf``, + unrelated to :pep:`376`. Examples include ``apt``, ``dpkg``, ``dnf``, ``rpm``, ``pacman``, and ``brew``. The salient feature is that if a package was installed by a distro package manager, removing or upgrading it in a way that would satisfy a Python-specific package @@ -291,9 +294,9 @@ Case Description ``pip install`` permitted Deleting exte 3 Distro Python in venv Currently yes; stays yes There are no externally-installed packages 4 Distro Python in venv Currently yes; stays yes Currently no; stays no with ``--system-site-packages`` -5 Distro Python in Docker Currently yes; stays yes Currently yes; becomes no - (assuming the Docker image - removes the marker file) +5 Distro Python in Docker Currently yes; becomes no Currently yes; becomes no + (assuming the distro + adds a marker file) 6 Conda environment Currently yes; stays yes Currently yes; stays yes 7 Dev-facing distro Currently yes; becomes no Currently often yes; becomes no (assuming they add a (assuming they configure ``sysconfig`` as needed) @@ -312,9 +315,9 @@ In more detail, the use cases above are: does not change its behavior. Such a CPython should (regardless of this PEP) not be installed in - a way that that overlaps any distro-installed Python on the same - system. For instance, on an OS that ships Python in ``/usr/bin``, - you should not install a custom CPython built with ``./configure + a way that overlaps any distro-installed Python on the same system. + For instance, on an OS that ships Python in ``/usr/bin``, you + should not install a custom CPython built with ``./configure --prefix=/usr``, or it will overwrite some files from the distro and the distro will eventually overwrite some files from your installation. Instead, your installation should be in a separate @@ -374,8 +377,8 @@ In more detail, the use cases above are: worth calling out explicitly, because anything on the global ``sys.path`` is visible. - Currently, the answer to "Will ``pip` delete externally-installed - packages`` is no, because pip has a special case for running in a + Currently, the answer to "Will ``pip`` delete externally-installed + packages" is no, because pip has a special case for running in a virtual environment and attempting to delete packages outside it. After this PEP, the answer remains no, but the reasoning becomes more general: system site packages will be outside any of the @@ -659,18 +662,14 @@ on Debian) depends on pipx. .. _pipx: https://github.com/pypa/pipx -Remove the marker file in container images ------------------------------------------- +Keep the marker file in container images +---------------------------------------- Distros that produce official images for single-application containers -(e.g., Docker container images) should remove the +(e.g., Docker container images) should keep the ``EXTERNALLY-MANAGED`` file, preferably in a way that makes it not -come back if a user of that image installs package updates inside -their image (think ``RUN apt-get dist-upgrade``). On dpkg-based -systems, using ``dpkg-divert --local`` to persistently rename the file -would work. On other systems, there may need to be some configuration -flag available to a post-install script to re-remove the -``EXTERNALLY-MANAGED`` file. +go away if a user of that image installs package updates inside +their image (think ``RUN apt-get dist-upgrade``). Create separate distro and local directories -------------------------------------------- @@ -707,14 +706,15 @@ Filesystem Hierarchy Standard`__. There are two ways you could do this. One is, if you are building and packaging Python libraries directly (e.g., your packaging helpers -unpack a PEP 517-built wheel or call ``setup.py install``), arrange +unpack a :pep:`517`-built wheel or call ``setup.py install``), arrange for those tools to use a directory that is not in a ``sysconfig`` scheme but is still on ``sys.path``. The other is to arrange for the default ``sysconfig`` scheme to change when running inside a package build versus when running on an installed system. The ``sysconfig`` customization hooks from -bpo-43976_ should make this easy: make your packaging tool set an +bpo-43976_ should make this easy (once accepted and implemented): +make your packaging tool set an environment variable or some other detectable configuration, and define a ``get_preferred_schemes`` function to return a different scheme when called from inside a package build. Then you can use ``pip @@ -886,7 +886,7 @@ non-virtual-environment installs in any Python installation, but that is outside the scope of this PEP.) Should the file be TOML? TOML is gaining popularity for packaging (see -e.g. PEP 517) but does not yet have an implementation in the standard +e.g. :pep:`517`) but does not yet have an implementation in the standard library. Strictly speaking, this isn't a blocker - distros need only write the file, not read it, so they don't need a TOML library (the file will probably be written by hand, regardless of format), and @@ -924,16 +924,16 @@ distinction seems difficult. We think it's a good idea for Python-specific package managers to print a warning if they shadow a package, but we think it's not worth disabling it by default. -Why not use the ``INSTALLER`` file from PEP 376 to determine who +Why not use the ``INSTALLER`` file from :pep:`376` to determine who installed a package and whether it can be removed? First, it's specific to a particular package (it's in the package's ``dist-info`` directory), so like some of the alternatives above, it doesn't provide information on an entire environment and whether package installations -are permissible. PEP 627 also updates PEP 376 to prevent programmatic +are permissible. :pep:`627` also updates :pep:`376` to prevent programmatic use of ``INSTALLER``, specifying that the file is "to be used for informational purposes only. [...] Our goal is supporting interoperating tools, and basing any action on which tool happened to -install a package runs counter to that goal." Finally, as PEP 627 +install a package runs counter to that goal." Finally, as :pep:`627` envisions, there are legitimate use cases for one tool knowing how to handle packages installed by another tool; for instance, ``conda`` can safely remove a package installed by ``pip`` into a Conda environment. @@ -1060,7 +1060,7 @@ References For additional background on these problems and previous attempts to solve them, see `Debian bug 771794`_ "pip silently removes/updates -system provided python packages` from 2014, Fedora's 2018 article +system provided python packages" from 2014, Fedora's 2018 article `Making sudo pip safe`_ about pointing ``sudo pip`` at /usr/local (which acknowledges that the changes still do not make ``sudo pip`` completely safe), pip issues 5605_ ("Disable upgrades to existing diff --git a/peps/pep-0669.rst b/peps/pep-0669.rst new file mode 100644 index 000000000..a12c30161 --- /dev/null +++ b/peps/pep-0669.rst @@ -0,0 +1,597 @@ +PEP: 669 +Title: Low Impact Monitoring for CPython +Author: Mark Shannon +Discussions-To: https://discuss.python.org/t/pep-669-low-impact-monitoring-for-cpython/13018/ +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 18-Aug-2021 +Python-Version: 3.12 +Post-History: `07-Dec-2021 `__, + `10-Jan-2022 `__, +Resolution: https://discuss.python.org/t/pep-669-low-impact-monitoring-for-cpython/13018/42 + +Abstract +======== + +Using a profiler or debugger in CPython can have a severe impact on +performance. Slowdowns by an order of magnitude are common. + +This PEP proposes an API for monitoring Python programs running +on CPython that will enable monitoring at low cost. + +Although this PEP does not specify an implementation, it is expected that +it will be implemented using the quickening step of +:pep:`659`. + +A ``sys.monitoring`` namespace will be added, which will contain +the relevant functions and constants. + + +Motivation +========== + +Developers should not have to pay an unreasonable cost to use debuggers, +profilers and other similar tools. + +C++ and Java developers expect to be able to run a program at full speed +(or very close to it) under a debugger. +Python developers should expect that too. + +Rationale +========= + +The quickening mechanism provided by :pep:`659` provides a way to dynamically +modify executing Python bytecode. These modifications have little cost beyond +the parts of the code that are modified and a relatively low cost to those +parts that are modified. We can leverage this to provide an efficient +mechanism for monitoring that was not possible in 3.10 or earlier. + +By using quickening, we expect that code run under a debugger on 3.12 +should outperform code run without a debugger on 3.11. +Profiling will still slow down execution, but by much less than in 3.11. + + +Specification +============= + +Monitoring of Python programs is done by registering callback functions +for events and by activating a set of events. + +Activating events and registering callback functions are independent of each other. + +Both registering callbacks and activating events are done on a per-tool basis. +It is possible to have multiple tools that respond to different sets of events. + +Note that, unlike ``sys.settrace()``, events and callbacks are per interpreter, not per thread. + +Events +------ + +As a code object executes various events occur that might be of interest +to tools. By activating events and by registering callback functions +tools can respond to these events in any way that suits them. +Events can be set globally, or for individual code objects. + +For 3.12, CPython will support the following events: + +* PY_START: Start of a Python function (occurs immediately after the call, the callee's frame will be on the stack) +* PY_RESUME: Resumption of a Python function (for generator and coroutine functions), except for throw() calls. +* PY_THROW: A Python function is resumed by a throw() call. +* PY_RETURN: Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). +* PY_YIELD: Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). +* PY_UNWIND: Exit from a Python function during exception unwinding. +* CALL: A call in Python code (event occurs before the call). +* C_RETURN: Return from any callable, except Python functions (event occurs after the return). +* C_RAISE: Exception raised from any callable, except Python functions (event occurs after the exit). +* RAISE: An exception is raised, except those that cause a ``STOP_ITERATION`` event. +* EXCEPTION_HANDLED: An exception is handled. +* LINE: An instruction is about to be executed that has a different line number from the preceding instruction. +* INSTRUCTION -- A VM instruction is about to be executed. +* JUMP -- An unconditional jump in the control flow graph is made. +* BRANCH -- A conditional branch is taken (or not). +* STOP_ITERATION -- An artificial ``StopIteration`` is raised; + see `the STOP_ITERATION event`_. + +More events may be added in the future. + +All events will be attributes of the ``events`` namespace in ``sys.monitoring``. +All events will represented by a power of two integer, so that they can be combined +with the ``|`` operator. + +Events are divided into three groups: + +Local events +'''''''''''' + +Local events are associated with normal execution of the program and happen +at clearly defined locations. All local events can be disabled. +The local events are: + +* PY_START +* PY_RESUME +* PY_RETURN +* PY_YIELD +* CALL +* LINE +* INSTRUCTION +* JUMP +* BRANCH +* STOP_ITERATION + +Ancilliary events +''''''''''''''''' + +Ancillary events can be monitored like other events, but are controlled +by another event: + +* C_RAISE +* C_RETURN + +The ``C_RETURN`` and ``C_RAISE`` events are are controlled by the ``CALL`` +event. ``C_RETURN`` and ``C_RAISE`` events will only be seen if the +corresponding ``CALL`` event is being monitored. + +Other events +'''''''''''' + +Other events are not necessarily tied to a specific location in the +program and cannot be individually disabled. + +The other events that can be monitored are: + +* PY_THROW +* PY_UNWIND +* RAISE +* EXCEPTION_HANDLED + + +The STOP_ITERATION event +'''''''''''''''''''''''' + +:pep:`PEP 380 <380#use-of-stopiteration-to-return-values>` +specifies that a ``StopIteration`` exception is raised when returning a value +from a generator or coroutine. However, this is a very inefficient way to +return a value, so some Python implementations, notably CPython 3.12+, do not +raise an exception unless it would be visible to other code. + +To allow tools to monitor for real exceptions without slowing down generators +and coroutines, the ``STOP_ITERATION`` event is provided. +``STOP_ITERATION`` can be locally disabled, unlike ``RAISE``. + +Tool identifiers +---------------- + +The VM can support up to 6 tools at once. +Before registering or activating events, a tool should choose an identifier. +Identifiers are integers in the range 0 to 5. + +:: + + sys.monitoring.use_tool_id(id, name:str) -> None + sys.monitoring.free_tool_id(id) -> None + sys.monitoring.get_tool(id) -> str | None + +``sys.monitoring.use_tool_id`` raises a ``ValueError`` if ``id`` is in use. +``sys.monitoring.get_tool`` returns the name of the tool if ``id`` is in use, +otherwise it returns ``None``. + +All IDs are treated the same by the VM with regard to events, but the +following IDs are pre-defined to make co-operation of tools easier:: + + sys.monitoring.DEBUGGER_ID = 0 + sys.monitoring.COVERAGE_ID = 1 + sys.monitoring.PROFILER_ID = 2 + sys.monitoring.OPTIMIZER_ID = 5 + +There is no obligation to set an ID, nor is there anything preventing a tool +from using an ID even it is already in use. +However, tools are encouraged to use a unique ID and respect other tools. + +For example, if a debugger were attached and ``DEBUGGER_ID`` were in use, it +should report an error, rather than carrying on regardless. + +The ``OPTIMIZER_ID`` is provided for tools like Cinder or PyTorch +that want to optimize Python code, but need to decide what to +optimize in a way that depends on some wider context. + +Setting events globally +----------------------- + +Events can be controlled globally by modifying the set of events being monitored: + +* ``sys.monitoring.get_events(tool_id:int)->int`` + Returns the ``int`` representing all the active events. + +* ``sys.monitoring.set_events(tool_id:int, event_set: int)`` + Activates all events which are set in ``event_set``. + Raises a ``ValueError`` if ``tool_id`` is not in use. + +No events are active by default. + +Per code object events +---------------------- + +Events can also be controlled on a per code object basis: + +* ``sys.monitoring.get_local_events(tool_id:int, code: CodeType)->int`` + Returns all the local events for ``code`` + +* ``sys.monitoring.set_local_events(tool_id:int, code: CodeType, event_set: int)`` + Activates all the local events for ``code`` which are set in ``event_set``. + Raises a ``ValueError`` if ``tool_id`` is not in use. + +Local events add to global events, but do not mask them. +In other words, all global events will trigger for a code object, +regardless of the local events. + +Register callback functions +--------------------------- + +To register a callable for events call:: + + sys.monitoring.register_callback(tool_id:int, event: int, func: Callable | None) -> Callable | None + +If another callback was registered for the given ``tool_id`` and ``event``, +it is unregistered and returned. +Otherwise ``register_callback`` returns ``None``. + +Functions can be unregistered by calling +``sys.monitoring.register_callback(tool_id, event, None)``. + +Callback functions can be registered and unregistered at any time. + +Registering or unregistering a callback function will generate a ``sys.audit`` event. + +Callback function arguments +''''''''''''''''''''''''''' + +When an active event occurs, the registered callback function is called. +Different events will provide the callback function with different arguments, as follows: + +* ``PY_START`` and ``PY_RESUME``:: + + func(code: CodeType, instruction_offset: int) -> DISABLE | Any + +* ``PY_RETURN`` and ``PY_YIELD``: + + ``func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any`` + +* ``CALL``, ``C_RAISE`` and ``C_RETURN``: + + ``func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any`` + + If there are no arguments, ``arg0`` is set to ``MISSING``. + +* ``RAISE`` and ``EXCEPTION_HANDLED``: + + ``func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any`` + +* ``LINE``: + + ``func(code: CodeType, line_number: int) -> DISABLE | Any`` + +* ``BRANCH``: + + ``func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any`` + + Note that the ``destination_offset`` is where the code will next execute. + For an untaken branch this will be the offset of the instruction following + the branch. + +* ``INSTRUCTION``: + + ``func(code: CodeType, instruction_offset: int) -> DISABLE | Any`` + + +If a callback function returns ``DISABLE``, then that function will no longer +be called for that ``(code, instruction_offset)`` until +``sys.monitoring.restart_events()`` is called. +This feature is provided for coverage and other tools that are only interested +seeing an event once. + +Note that ``sys.monitoring.restart_events()`` is not specific to one tool, +so tools must be prepared to receive events that they have chosen to DISABLE. + +Events in callback functions +---------------------------- + +Events are suspended in callback functions and their callees for the tool +that registered that callback. + +That means that other tools will see events in the callback functions for other +tools. This could be useful for debugging a profiling tool, but would produce +misleading profiles, as the debugger tool would show up in the profile. + +Order of events +--------------- + +If an instructions triggers several events they occur in the following order: + +* LINE +* INSTRUCTION +* All other events (only one of these events can occur per instruction) + +Each event is delivered to tools in ascending order of ID. + +The "call" event group +---------------------- + +Most events are independent; setting or disabling one event has no effect on the others. +However, the ``CALL``, ``C_RAISE`` and ``C_RETURN`` events form a group. +If any of those events are set or disabled, then all events in the group are. +Disabling a ``CALL`` event will not disable the matching ``C_RAISE`` or ``C_RETURN``, +but will disable all subsequent events. + + +Attributes of the ``sys.monitoring`` namespace +---------------------------------------------- + +* ``def use_tool_id(id)->None`` +* ``def free_tool_id(id)->None`` +* ``def get_events(tool_id: int)->int`` +* ``def set_events(tool_id: int, event_set: int)->None`` +* ``def get_local_events(tool_id: int, code: CodeType)->int`` +* ``def set_local_events(tool_id: int, code: CodeType, event_set: int)->None`` +* ``def register_callback(tool_id: int, event: int, func: Callable)->Optional[Callable]`` +* ``def restart_events()->None`` +* ``DISABLE: object`` +* ``MISSING: object`` + +Access to "debug only" features +------------------------------- + +Some features of the standard library are not accessible to normal code, +but are accessible to debuggers. For example, setting local variables, or +the line number. + +These features will be available to callback functions. + +Backwards Compatibility +======================= + +This PEP is mostly backwards compatible. + +There are some compatibility issues with :pep:`523`, as the behavior +of :pep:`523` plugins is outside of the VM's control. +It is up to :pep:`523` plugins to ensure that they respect the semantics +of this PEP. Simple plugins that do not change the state of the VM, and +defer execution to ``_PyEval_EvalFrameDefault()`` should continue to work. + +:func:`sys.settrace` and :func:`sys.setprofile` will act as if they were tools +6 and 7 respectively, so can be used alongside this PEP. + +This means that :func:`sys.settrace` and :func:`sys.setprofile` may not work +correctly with all :pep:`523` plugins. Although, simple :pep:`523` +plugins, as described above, should be fine. + +Performance +----------- + +If no events are active, this PEP should have a small positive impact on +performance. Experiments show between 1 and 2% speedup from not supporting +:func:`sys.settrace` directly. + +The performance of :func:`sys.settrace` will be about the same. +The performance of :func:`sys.setprofile` should be better. +However, tools relying on :func:`sys.settrace` and +:func:`sys.setprofile` can be made a lot faster by using the +API provided by this PEP. + +If a small set of events are active, e.g. for a debugger, then the overhead +of callbacks will be orders of magnitudes less than for :func:`sys.settrace` +and much cheaper than using :pep:`523`. + +Coverage tools can be implemented at very low cost, +by returning ``DISABLE`` in all callbacks. + +For heavily instrumented code, e.g. using ``LINE``, performance should be +better than ``sys.settrace``, but not by that much as performance will be +dominated by the time spent in callbacks. + +For optimizing virtual machines, such as future versions of CPython +(and ``PyPy`` should they choose to support this API), changes to the set +active events in the midst of a long running program could be quite +expensive, possibly taking hundreds of milliseconds as it triggers +de-optimizations. Once such de-optimization has occurred, performance should +recover as the VM can re-optimize the instrumented code. + +In general these operations can be considered to be fast: + +* ``def get_events(tool_id: int)->int`` +* ``def get_local_events(tool_id: int, code: CodeType)->int`` +* ``def register_callback(tool_id: int, event: int, func: Callable)->Optional[Callable]`` +* ``def get_tool(tool_id) -> str | None`` + +These operations are slower, but not especially so: + +* ``def set_local_events(tool_id: int, code: CodeType, event_set: int)->None`` + +And these operations should be regarded as slow: + +* ``def use_tool_id(id, name:str)->None`` +* ``def free_tool_id(id)->None`` +* ``def set_events(tool_id: int, event_set: int)->None`` +* ``def restart_events()->None`` + +How slow the slow operations are depends on when they happen. +If done early in the program, before modules are loaded, +they should be fairly inexpensive. + +Memory Consumption +'''''''''''''''''' + +When not in use, this PEP will have a negligible change on memory consumption. + +How memory is used is very much an implementation detail. +However, we expect that for 3.12 the additional memory consumption per +code object will be **roughly** as follows: + ++-------------+--------+--------+-------------+ +| | Events | ++-------------+--------+--------+-------------+ +| Tools | Others | LINE | INSTRUCTION | ++=============+========+========+=============+ +| One | None | ≈40% | ≈80% | ++-------------+--------+--------+-------------+ ++ Two or more | ≈40% | ≈120% | ≈200% | ++-------------+--------+--------+-------------+ + + +Security Implications +===================== + +Allowing modification of running code has some security implications, +but no more than the ability to generate and call new code. + +All the new functions listed above will trigger audit hooks. + +Implementation +============== + +This outlines the proposed implementation for CPython 3.12. The actual +implementation for later versions of CPython and other Python implementations +may differ considerably. + +The proposed implementation of this PEP will be built on top of the quickening +step of CPython 3.11, as described in :pep:`PEP 659 <659#quickening>`. +Instrumentation works in much the same way as quickening, bytecodes are +replaced with instrumented ones as needed. + +For example, if the ``CALL`` event is turned on, +then all call instructions will be +replaced with a ``INSTRUMENTED_CALL`` instruction. + +Note that this will interfere with specialization, which will result in some +performance degradation in addition to the overhead of calling the +registered callable. + +When the set of active events changes, the VM will immediately update +all code objects present on the call stack of any thread. It will also set in +place traps to ensure that all code objects are correctly instrumented when +called. Consequently changing the set of active events should be done as +infrequently as possible, as it could be quite an expensive operation. + +Other events, such as ``RAISE`` can be turned on or off cheaply, +as they do not rely on code instrumentation, but runtime checks when the +underlying event occurs. + +The exact set of events that require instrumentation is an implementation detail, +but for the current design, the following events will require instrumentation: + +* PY_START +* PY_RESUME +* PY_RETURN +* PY_YIELD +* CALL +* LINE +* INSTRUCTION +* JUMP +* BRANCH + +Each instrumented bytecode will require an additional 8 bits of information to +note which tool the instrumentation applies to. +``LINE`` and ``INSTRUCTION`` events require additional information, as they +need to store the original instruction, or even the instrumented instruction +if they overlap other instrumentation. + + +Implementing tools +================== + +It is the philosophy of this PEP that it should be possible for third-party monitoring +tools to achieve high-performance, not that it should be easy for them to do so. + +Converting events into data that is meaningful to the users is +the responsibility of the tool. + +All events have a cost, and tools should attempt to the use set of events +that trigger the least often and still provide the necessary information. + +Debuggers +--------- + +Inserting breakpoints +''''''''''''''''''''' + +Breakpoints can be inserted setting per code object events, either ``LINE`` or ``INSTRUCTION``, +and returning ``DISABLE`` for any events not matching a breakpoint. + +Stepping +'''''''' + +Debuggers usually offer the ability to step execution by a +single instruction or line. + +Like breakpoints, stepping can be implemented by setting per code object events. +As soon as normal execution is to be resumed, the local events can be unset. + +Attaching +''''''''' + +Debuggers can use the ``PY_START`` and ``PY_RESUME`` events to be informed +when a code object is first encountered, so that any necessary breakpoints +can be inserted. + +Coverage Tools +-------------- + +Coverage tools need to track which parts of the control graph have been +executed. To do this, they need to register for the ``PY_`` events, +plus ``JUMP`` and ``BRANCH``. + +This information can be then be converted back into a line based report +after execution has completed. + +Profilers +--------- + +Simple profilers need to gather information about calls. +To do this profilers should register for the following events: + +* PY_START +* PY_RESUME +* PY_THROW +* PY_RETURN +* PY_YIELD +* PY_UNWIND +* CALL +* C_RAISE +* C_RETURN + + +Line based profilers +'''''''''''''''''''' + +Line based profilers can use the ``LINE`` and ``JUMP`` events. +Implementers of profilers should be aware that instrumenting ``LINE`` +events will have a large impact on performance. + +.. note:: + + Instrumenting profilers have significant overhead and will distort + the results of profiling. Unless you need exact call counts, + consider using a statistical profiler. + + +Rejected ideas +============== + +A draft version of this PEP proposed making the user responsible +for inserting the monitoring instructions, rather than have VM do it. +However, that puts too much of a burden on the tools, and would make +attaching a debugger nearly impossible. + +An earlier version of this PEP, proposed storing events as ``enums``:: + + class Event(enum.IntFlag): + PY_START = ... + +However, that would prevent monitoring of code before the ``enum`` module was +loaded and could cause unnecessary overhead. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0670.rst b/peps/pep-0670.rst new file mode 100644 index 000000000..a9298a682 --- /dev/null +++ b/peps/pep-0670.rst @@ -0,0 +1,670 @@ +PEP: 670 +Title: Convert macros to functions in the Python C API +Author: Erlend Egeberg Aasland , + Victor Stinner +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Created: 19-Oct-2021 +Python-Version: 3.11 +Post-History: `20-Oct-2021 `__, + `08-Feb-2022 `__, + `22-Feb-2022 `__ +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/QQFCJ7LR36RUZSC3WI6WZZMQVQ3ZI4MS/ + + +Abstract +======== + +Macros in the C API will be converted to static inline functions or +regular functions. This will help avoid macro pitfalls in C/C++, and +make the functions usable from other programming languages. + +To avoid compiler warnings, function arguments of pointer types +will be cast to appropriate types using additional macros. +The cast will not be done in the limited C API version 3.11: +users who opt in to the new limited API may need to add casts to +the exact expected type. + +To avoid introducing incompatible changes, macros which can be used as +l-value in an assignment will not be converted. + + +Rationale +========= + +The use of macros may have unintended adverse effects that are hard to +avoid, even for experienced C developers. Some issues have been known +for years, while others have been discovered recently in Python. +Working around macro pitfalls makes the macro code harder to read and +to maintain. + +Converting macros to functions has multiple advantages: + +* Functions don't suffer from macro pitfalls, for example the following + ones described in `GCC documentation + `_: + + - Misnesting + - Operator precedence problems + - Swallowing the semicolon + - Duplication of side effects + - Self-referential macros + - Argument prescan + - Newlines in arguments + + Functions don't need the following workarounds for macro + pitfalls, making them usually easier to read and to maintain than + similar macro code: + + - Adding parentheses around arguments. + - Using line continuation characters if the function is written on + multiple lines. + - Adding commas to execute multiple expressions. + - Using ``do { ... } while (0)`` to write multiple statements. + +* Argument types and the return type of functions are well defined. +* Debuggers and profilers can retrieve the name of inlined functions. +* Debuggers can put breakpoints on inlined functions. +* Variables have a well-defined scope. + +Converting macros and static inline functions to regular functions makes +these regular functions accessible to projects which use Python but +cannot use macros and static inline functions. + + +Specification +============= + +Convert macros to static inline functions +----------------------------------------- + +Most macros will be converted to static inline functions. + +The following macros will not be converted: + +* Object-like macros (i.e. those which don't need parentheses and + arguments). For example: + + * Empty macros. Example: ``#define Py_HAVE_CONDVAR``. + * Macros only defining a value, even if a constant with a well defined + type would be better. Example: ``#define METH_VARARGS 0x0001``. + +* Compatibility layer for different C compilers, C language extensions, + or recent C features. + Example: ``Py_GCC_ATTRIBUTE()``, ``Py_ALWAYS_INLINE``, ``Py_MEMCPY()``. +* Macros used for definitions rather than behavior. + Example: ``PyAPI_FUNC``, ``Py_DEPRECATED``, ``Py_PYTHON_H``. +* Macros that need C preprocessor features, like stringification and + concatenation. Example: ``Py_STRINGIFY()``. +* Macros which cannot be converted to functions. Examples: + ``Py_BEGIN_ALLOW_THREADS`` (contains an unpaired ``}``), ``Py_VISIT`` + (relies on specific variable names), Py_RETURN_RICHCOMPARE (returns + from the calling function). +* Macros which can be used as an l-value in assignments. This would be + an incompatible change and is out of the scope of this PEP. + Example: ``PyBytes_AS_STRING()``. +* Macros which have different return types depending on the code path + or arguments. + + +Convert static inline functions to regular functions +---------------------------------------------------- + +Static inline functions in the public C API may be converted to regular +functions, but only if there is no measurable performance impact of +changing the function. +The performance impact should be measured with benchmarks. + + +Cast pointer arguments +---------------------- + +Currently, most macros accepting pointers cast pointer arguments to +their expected types. For example, in Python 3.6, the ``Py_TYPE()`` +macro casts its argument to ``PyObject*``: + +.. code-block:: c + + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + +The ``Py_TYPE()`` macro accepts the ``PyObject*`` type, but also any +pointer types, such as ``PyLongObject*`` and ``PyDictObject*``. + +Functions are strongly typed, and can only accept one type of argument. + +To avoid compiler errors and warnings in existing code, when a macro is +converted to a function and the macro casts at least one of its arguments +a new macro will be added to keep the cast. The new macro +and the function will have the same name. + +Example with the ``Py_TYPE()`` +macro converted to a static inline function: + +.. code-block:: c + + static inline PyTypeObject* Py_TYPE(PyObject *ob) { + return ob->ob_type; + } + #define Py_TYPE(ob) Py_TYPE((PyObject*)(ob)) + +The cast is kept for all pointer types, not only ``PyObject*``. +This includes casts to ``void*``: removing a cast to ``void*`` would emit +a new warning if the function is called with a ``const void*`` variable. +For example, the ``PyUnicode_WRITE()`` macro casts its *data* argument to +``void*``, and so it currently accepts ``const void*`` type, even though +it writes into *data*. This PEP will not change this. + + +Avoid the cast in the limited C API version 3.11 +'''''''''''''''''''''''''''''''''''''''''''''''' + +The casts will be excluded from the limited C API version 3.11 and newer. +When an API user opts into the new limited API, they must pass the expected +type or perform the cast. + +As an example, ``Py_TYPE()`` will be defined like this: + +.. code-block:: c + + static inline PyTypeObject* Py_TYPE(PyObject *ob) { + return ob->ob_type; + } + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_TYPE(ob) Py_TYPE((PyObject*)(ob)) + #endif + + +Return type is not changed +-------------------------- + +When a macro is converted to a function, its return type must not change +to prevent emitting new compiler warnings. + +For example, Python 3.7 changed the return type of ``PyUnicode_AsUTF8()`` +from ``char*`` to ``const char*`` (`commit +`__). +The change emitted new compiler warnings when building C extensions +expecting ``char*``. This PEP doesn't change the return type to prevent +this issue. + + +Backwards Compatibility +======================= + +The PEP is designed to avoid C API incompatible changes. + +Only C extensions explicitly targeting the limited C API version 3.11 +must now pass the expected types to functions: pointer arguments are no +longer cast to the expected types. + +Function arguments of pointer types are still cast and return types are +not changed to prevent emitting new compiler warnings. + +Macros which can be used as l-value in an assignment are not modified by +this PEP to avoid incompatible changes. + + +Examples of Macro Pitfalls +========================== + +Duplication of side effects +--------------------------- + +Macros: + +.. code-block:: c + + #define PySet_Check(ob) \ + (Py_IS_TYPE(ob, &PySet_Type) \ + || PyType_IsSubtype(Py_TYPE(ob), &PySet_Type)) + + #define Py_IS_NAN(X) ((X) != (X)) + +If the *op* or the *X* argument has a side effect, the side effect is +duplicated: it executed twice by ``PySet_Check()`` and ``Py_IS_NAN()``. + +For example, the ``pos++`` argument in the +``PyUnicode_WRITE(kind, data, pos++, ch)`` code has a side effect. +This code is safe because the ``PyUnicode_WRITE()`` macro only uses its +3rd argument once and so does not duplicate ``pos++`` side effect. + +Misnesting +---------- + +Example of the `bpo-43181: Python macros don't shield arguments +`_. The ``PyObject_TypeCheck()`` +macro before it has been fixed: + +.. code-block:: c + + #define PyObject_TypeCheck(ob, tp) \ + (Py_IS_TYPE(ob, tp) || PyType_IsSubtype(Py_TYPE(ob), (tp))) + +C++ usage example: + +.. code-block:: c + + PyObject_TypeCheck(ob, U(f(c))) + +The preprocessor first expands it: + +.. code-block:: c + + (Py_IS_TYPE(ob, f(c)) || ...) + +C++ ``"<"`` and ``">"`` characters are not treated as brackets by the +preprocessor, so the ``Py_IS_TYPE()`` macro is invoked with 3 arguments: + +* ``ob`` +* ``f(c)`` + +The compilation fails with an error on ``Py_IS_TYPE()`` which only takes +2 arguments. + +The bug is that the *op* and *tp* arguments of ``PyObject_TypeCheck()`` +must be put between parentheses: replace ``Py_IS_TYPE(ob, tp)`` with +``Py_IS_TYPE((ob), (tp))``. In regular C code, these parentheses are +redundant, can be seen as a bug, and so are often forgotten when writing +macros. + +To avoid Macro Pitfalls, the ``PyObject_TypeCheck()`` macro has been +converted to a static inline function: +`commit `__. + + +Examples of hard to read macros +=============================== + +PyObject_INIT() +--------------- + +Example showing the usage of commas in a macro which has a return value. + +Python 3.7 macro: + +.. code-block:: c + + #define PyObject_INIT(op, typeobj) \ + ( Py_TYPE(op) = (typeobj), _Py_NewReference((PyObject *)(op)), (op) ) + +Python 3.8 function (simplified code): + +.. code-block:: c + + static inline PyObject* + _PyObject_INIT(PyObject *op, PyTypeObject *typeobj) + { + Py_TYPE(op) = typeobj; + _Py_NewReference(op); + return op; + } + + #define PyObject_INIT(op, typeobj) \ + _PyObject_INIT(_PyObject_CAST(op), (typeobj)) + +* The function doesn't need the line continuation character ``"\"``. +* It has an explicit ``"return op;"`` rather than the surprising + ``", (op)"`` syntax at the end of the macro. +* It uses short statements on multiple lines, rather than being written + as a single long line. +* Inside the function, the *op* argument has the well defined type + ``PyObject*`` and so doesn't need casts like ``(PyObject *)(op)``. +* Arguments don't need to be put inside parentheses: use ``typeobj``, + rather than ``(typeobj)``. + +_Py_NewReference() +------------------ + +Example showing the usage of an ``#ifdef`` inside a macro. + +Python 3.7 macro (simplified code): + +.. code-block:: c + + #ifdef COUNT_ALLOCS + # define _Py_INC_TPALLOCS(OP) inc_count(Py_TYPE(OP)) + # define _Py_COUNT_ALLOCS_COMMA , + #else + # define _Py_INC_TPALLOCS(OP) + # define _Py_COUNT_ALLOCS_COMMA + #endif /* COUNT_ALLOCS */ + + #define _Py_NewReference(op) ( \ + _Py_INC_TPALLOCS(op) _Py_COUNT_ALLOCS_COMMA \ + Py_REFCNT(op) = 1) + +Python 3.8 function (simplified code): + +.. code-block:: c + + static inline void _Py_NewReference(PyObject *op) + { + _Py_INC_TPALLOCS(op); + Py_REFCNT(op) = 1; + } + + +PyUnicode_READ_CHAR() +--------------------- + +This macro reuses arguments, and possibly calls ``PyUnicode_KIND`` multiple +times: + +.. code-block:: c + + #define PyUnicode_READ_CHAR(unicode, index) \ + (assert(PyUnicode_Check(unicode)), \ + assert(PyUnicode_IS_READY(unicode)), \ + (Py_UCS4) \ + (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \ + ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \ + (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \ + ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \ + ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \ + ) \ + )) + +Possible implementation as a static inlined function: + +.. code-block:: c + + static inline Py_UCS4 + PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) + { + assert(PyUnicode_Check(unicode)); + assert(PyUnicode_IS_READY(unicode)); + + switch (PyUnicode_KIND(unicode)) { + case PyUnicode_1BYTE_KIND: + return (Py_UCS4)((const Py_UCS1 *)(PyUnicode_DATA(unicode)))[index]; + case PyUnicode_2BYTE_KIND: + return (Py_UCS4)((const Py_UCS2 *)(PyUnicode_DATA(unicode)))[index]; + case PyUnicode_4BYTE_KIND: + default: + return (Py_UCS4)((const Py_UCS4 *)(PyUnicode_DATA(unicode)))[index]; + } + } + + +Macros converted to functions since Python 3.8 +============================================== + +This is a list of macros already converted to functions between +Python 3.8 and Python 3.11. +Even though some converted macros (like ``Py_INCREF()``) are very +commonly used by C extensions, these conversions did not significantly +impact Python performance and most of them didn't break backward +compatibility. + +Macros converted to static inline functions +------------------------------------------- + +Python 3.8: + +* ``Py_DECREF()`` +* ``Py_INCREF()`` +* ``Py_XDECREF()`` +* ``Py_XINCREF()`` +* ``PyObject_INIT()`` +* ``PyObject_INIT_VAR()`` +* ``_PyObject_GC_UNTRACK()`` +* ``_Py_Dealloc()`` + +Macros converted to regular functions +------------------------------------- + +Python 3.9: + +* ``PyIndex_Check()`` +* ``PyObject_CheckBuffer()`` +* ``PyObject_GET_WEAKREFS_LISTPTR()`` +* ``PyObject_IS_GC()`` +* ``PyObject_NEW()``: alias to ``PyObject_New()`` +* ``PyObject_NEW_VAR()``: alias to ``PyObjectVar_New()`` + +To avoid performance slowdown on Python built without LTO, +private static inline functions have been added to the internal C API: + +* ``_PyIndex_Check()`` +* ``_PyObject_IS_GC()`` +* ``_PyType_HasFeature()`` +* ``_PyType_IS_GC()`` + + +Static inline functions converted to regular functions +------------------------------------------------------- + +Python 3.11: + +* ``PyObject_CallOneArg()`` +* ``PyObject_Vectorcall()`` +* ``PyVectorcall_Function()`` +* ``_PyObject_FastCall()`` + +To avoid performance slowdown on Python built without LTO, a +private static inline function has been added to the internal C API: + +* ``_PyVectorcall_FunctionInline()`` + + +Incompatible changes +-------------------- + +While other converted macros didn't break the backward compatibility, +there is an exception. + +The 3 macros ``Py_REFCNT()``, ``Py_TYPE()`` and ``Py_SIZE()`` have been +converted to static inline functions in Python 3.10 and 3.11 to disallow +using them as l-value in assignment. It is an incompatible change made +on purpose: see `bpo-39573 `_ for +the rationale. + +This PEP does not propose converting macros which can be used as l-value +to avoid introducing new incompatible changes. + + +Performance concerns and benchmarks +=================================== + +There have been concerns that converting macros to functions can degrade +performance. + +This section explains performance concerns and shows benchmark results +using `PR 29728 `_, which +replaces the following static inline functions with macros: + +* ``PyObject_TypeCheck()`` +* ``PyType_Check()``, ``PyType_CheckExact()`` +* ``PyType_HasFeature()`` +* ``PyVectorcall_NARGS()`` +* ``Py_DECREF()``, ``Py_XDECREF()`` +* ``Py_INCREF()``, ``Py_XINCREF()`` +* ``Py_IS_TYPE()`` +* ``Py_NewRef()`` +* ``Py_REFCNT()``, ``Py_TYPE()``, ``Py_SIZE()`` + + +The benchmarks were run on Fedora 35 (Linux) with GCC 11 on a laptop with 8 +logical CPUs (4 physical CPU cores). + + +Static inline functions +----------------------- + +First of all, converting macros to *static inline* functions has +negligible impact on performance: the measured differences are consistent +with noise due to unrelated factors. + +Static inline functions are a new feature in the C99 standard. Modern C +compilers have efficient heuristics to decide if a function should be +inlined or not. + +When a C compiler decides to not inline, there is likely a good reason. +For example, inlining would reuse a register which requires to +save/restore the register value on the stack and so increases the stack +memory usage, or be less efficient. + +Benchmark of the ``./python -m test -j5`` command on Python built in +release mode with ``gcc -O3``, LTO and PGO: + +* Macros (PR 29728): 361 sec +- 1 sec +* Static inline functions (reference): 361 sec +- 1 sec + +There is **no significant performance difference** between macros and +static inline functions when static inline functions **are inlined**. + + +Debug build +----------- + +Performance in debug builds *can* suffer when macros are converted to +functions. This is compensated by better debuggability: debuggers can +retrieve function names, set breakpoints inside functions, etc. + +On Windows, when Python is built in debug mode by Visual Studio, static +inline functions are not inlined. + +On other platforms, ``./configure --with-pydebug`` uses the ``-Og`` compiler +option on compilers that support it (including GCC and LLVM Clang). +``-Og`` means “optimize debugging experience”. +Otherwise, the ``-O0`` compiler option is used. +``-O0`` means “disable most optimizations”. + +With GCC 11, ``gcc -Og`` can inline static inline functions, whereas +``gcc -O0`` does not inline static inline functions. + +Benchmark of the ``./python -m test -j10`` command on Python built in +debug mode with ``gcc -O0`` (that is, compiler optimizations, +including inlining, are explicitly disabled): + +* Macros (PR 29728): 345 sec ± 5 sec +* Static inline functions (reference): 360 sec ± 6 sec + +Replacing macros with static inline functions makes Python +**1.04x slower** when the compiler **does not inline** static inline +functions. + +Note that benchmarks should not be run on a Python debug build. +Moreover, using link-time optimization (LTO) and profile-guided optimization +(PGO) is recommended for best performance and reliable benchmarks. +PGO helps the compiler to decide if functions should be inlined or not. + + +Force inlining +-------------- + +The ``Py_ALWAYS_INLINE`` macro can be used to force inlining. This macro +uses ``__attribute__((always_inline))`` with GCC and Clang, and +``__forceinline`` with MSC. + +Previous attempts to use ``Py_ALWAYS_INLINE`` didn't show any benefit, and were +abandoned. See for example `bpo-45094 `_ +"Consider using ``__forceinline`` and ``__attribute__((always_inline))`` on +static inline functions (``Py_INCREF``, ``Py_TYPE``) for debug build". + +When the ``Py_INCREF()`` macro was converted to a static inline +function in 2018 (`commit +`__), +it was decided not to force inlining. The machine code was analyzed with +multiple C compilers and compiler options, and ``Py_INCREF()`` was always +inlined without having to force inlining. The only case where it was not +inlined was the debug build. See discussion in `bpo-35059 +`_ "Convert ``Py_INCREF()`` and +``PyObject_INIT()`` to inlined functions". + + +Disabling inlining +------------------ + +On the other side, the ``Py_NO_INLINE`` macro can be used to disable +inlining. It can be used to reduce the stack memory usage, or to prevent +inlining on LTO+PGO builds, which generally inline code more aggressively: +see `bpo-33720 `_. The +``Py_NO_INLINE`` macro uses ``__attribute__ ((noinline))`` with GCC and +Clang, and ``__declspec(noinline)`` with MSC. + +This technique is available, though we currently don't know a concrete +function for which it would be useful. +Note that with macros, it is not possible to disable inlining at all. + + +Rejected Ideas +============== + +Keep macros, but fix some macro issues +-------------------------------------- + +Macros are always "inlined" with any C compiler. + +The duplication of side effects can be worked around in the caller of +the macro. + +People using macros should be considered "consenting adults". People who +feel unsafe with macros should simply not use them. + +These ideas are rejected because macros *are* error prone, and it is too easy +to miss a macro pitfall when writing and reviewing macro code. Moreover, macros +are harder to read and maintain than functions. + + +Post History +============ + +python-dev mailing list threads: + +* `Version 2 of PEP 670 - Convert macros to functions in the Python C API + `_ + (February 2022) +* `Steering Council reply to PEP 670 -- Convert macros to + functions in the Python C API + `_ + (February 2022) +* `PEP 670: Convert macros to functions in the Python C API + `_ + (October 2021) + + +References +========== + + +* `bpo-45490 `_: + [C API] PEP 670: Convert macros to functions in the Python C API + (October 2021). +* `What to do with unsafe macros + `_ + (March 2021). +* `bpo-43502 `_: + [C-API] Convert obvious unsafe macros to static inline functions + (March 2021). + + +Version History +=============== + +* Version 2: + + * Stricter policy on not changing argument types and return type. + * Better explain why pointer arguments require a cast to not emit new + compiler warnings. + * Macros which can be used as l-values are no longer modified by the + PEP. + * Macros having multiple return types are no longer modified by the + PEP. + * Limited C API version 3.11 no longer casts pointer arguments. + * No longer remove return values of macros "which should not have a + return value". + * Add "Macros converted to functions since Python 3.8" section. + * Add "Benchmark comparing macros and static inline functions" + section. + +* Version 1: First public version + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0671.rst b/peps/pep-0671.rst similarity index 62% rename from pep-0671.rst rename to peps/pep-0671.rst index cba06c83f..6febb920f 100644 --- a/pep-0671.rst +++ b/peps/pep-0671.rst @@ -1,13 +1,14 @@ PEP: 671 Title: Syntax for late-bound function argument defaults Author: Chris Angelico +Discussions-To: https://mail.python.org/archives/list/python-ideas@python.org/thread/UVOQEK7IRFSCBOH734T5GFJOEJXFCR6A/ Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 24-Oct-2021 -Python-Version: 3.11 -Post-History: 24-Oct-2021 - +Python-Version: 3.12 +Post-History: `24-Oct-2021 `__, + `01-Dec-2021 `__ Abstract ======== @@ -71,38 +72,41 @@ allows the expression to refer to other arguments. Multiple late-bound arguments are evaluated from left to right, and can refer to previously-defined values. Order is defined by the function, regardless of -the order in which keyword arguments may be passed. Using names of later -arguments should not be relied upon, and while this MAY work in some Python -implementations, it should be considered dubious:: +the order in which keyword arguments may be passed. def prevref(word="foo", a=>len(word), b=>a//2): # Valid - def selfref(spam=>spam): # Highly likely to give an error - def spaminate(sausage=>eggs + 1, eggs=>sausage - 1): # Confusing, may fail - def frob(n=>len(items), items=[]): # May fail, may succeed + def selfref(spam=>spam): # UnboundLocalError + def spaminate(sausage=>eggs + 1, eggs=>sausage - 1): # Confusing, don't do this + def frob(n=>len(items), items=[]): # See below +Evaluation order is left-to-right; however, implementations MAY choose to do so +in two separate passes, first for all passed arguments and early-bound defaults, +and then a second pass for late-bound defaults. Otherwise, all arguments will be +assigned strictly left-to-right. -Choice of spelling ------------------- +Rejected choices of spelling +---------------------------- -Our chief syntax proposal is ``name=>expression`` -- our two syntax proposals -... ahem. Amongst our potential syntaxes are:: +While this document specifies a single syntax ``name=>expression``, alternate +spellings are similarly plausible. The following spellings were considered:: - # Preferred options: adorn the equals sign (approximate preference order) def bisect(a, hi=>len(a)): - def bisect(a, hi=:len(a)): def bisect(a, hi:=len(a)): def bisect(a, hi?=len(a)): - def bisect(a, hi!=len(a)): - def bisect(a, hi=\len(a)): - def bisect(a, hi=@len(a)): - # Less preferred option: adorn the variable name def bisect(a, @hi=len(a)): - # Less preferred option: adorn the expression - def bisect(a, hi=`len(a)`): Since default arguments behave largely the same whether they're early or late -bound, the preferred syntax is very similar to the existing early-bind syntax. -The alternatives offer little advantage over the preferred one. +bound, the chosen syntax ``hi=>len(a)`` is deliberately similar to the existing +early-bind syntax. + +One reason for rejection of the ``:=`` syntax is its behaviour with annotations. +Annotations go before the default, so in all syntax options, it must be +unambiguous (both to the human and the parser) whether this is an annotation, +a default, or both. The alternate syntax ``target:=expr`` runs the risk of +being misinterpreted as ``target:int=expr`` with the annotation omitted in +error, and may thus mask bugs. The chosen syntax ``target=>expr`` does not +have this problem. + How to Teach This ================= @@ -117,28 +121,35 @@ bound arguments are broadly equivalent to code at the top of the function:: def add_item(item, target=): if target was omitted: target = [] +A simple rule of thumb is: "target=expression" is evaluated when the function +is defined, and "target=>expression" is evaluated when the function is called. +Either way, if the argument is provided at call time, the default is ignored. +While this does not completely explain all the subtleties, it is sufficient to +cover the important distinction here (and the fact that they are similar). -Interaction with other open PEPs + +Interaction with other proposals ================================ -PEP 661 attempts to solve one of the same problems as this does. It seeks to +:pep:`661` attempts to solve one of the same problems as this does. It seeks to improve the documentation of sentinel values in default arguments, where this -proposal seeks to remove the need for sentinels in many common cases. PEP 661 +proposal seeks to remove the need for sentinels in many common cases. :pep:`661` is able to improve documentation in arbitrarily complicated functions (it cites ``traceback.print_exception`` as its primary motivation, which has two arguments which must both-or-neither be specified); on the other hand, many of the common cases would no longer need sentinels if the true default could be defined by the function. Additionally, dedicated sentinel objects can be -used as dictionary lookup keys, where PEP 671 does not apply. +used as dictionary lookup keys, where :pep:`671` does not apply. - -Open Issues -=========== - -- Annotations go before the default, so in all syntax options, it must be - unambiguous (both to the human and the parser) whether this is an annotation, - a default, or both. The worst offender is the ``:=`` notation, as ``:int=`` - would be a valid annotation and early-bound default. +A generic system for deferred evaluation has been proposed at times (not to be +confused with :pep:`563` and :pep:`649` which are specific to annotations). +While it may seem, on the surface, that late-bound argument defaults are of a +similar nature, they are in fact unrelated and orthogonal ideas, and both could +be of value to the language. The acceptance or rejection of this proposal would +not affect the viability of a deferred evaluation proposal, and vice versa. (A +key difference between generalized deferred evaluation and argument defaults is +that argument defaults will always and only be evaluated as the function begins +executing, whereas deferred expressions would only be realized upon reference.) Implementation details @@ -159,10 +170,13 @@ needs to be queried. If it is ``None``, then the default is indeed the value ``Ellipsis``; otherwise, it is a descriptive string and the true value is calculated as the function begins. -When a parameter with a late-bound default is omitted, the -function will begin with the parameter unbound. The function begins by testing -for each parameter with a late-bound default, and if unbound, evaluates the -original expression. +When a parameter with a late-bound default is omitted, the function will begin +with the parameter unbound. The function begins by testing for each parameter +with a late-bound default using a new opcode QUERY_FAST/QUERY_DEREF, and if +unbound, evaluates the original expression. This opcode (available only for +fast locals and closure variables) pushes True onto the stack if the given +local has a value, and False if not - meaning that it pushes False if LOAD_FAST +or LOAD_DEREF would raise UnboundLocalError, and True if it would succeed. Out-of-order variable references are permitted as long as the referent has a value from an argument or early-bound default. @@ -200,14 +214,3 @@ Copyright This document is placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-0672.rst b/peps/pep-0672.rst similarity index 98% rename from pep-0672.rst rename to peps/pep-0672.rst index fa0b311a7..147a3c5bf 100644 --- a/pep-0672.rst +++ b/peps/pep-0672.rst @@ -32,7 +32,7 @@ accepting malicious code. The possible issues generally can't be solved in Python itself without excessive restrictions of the language. -They should be solved in code edirors and review tools +They should be solved in code editors and review tools (such as *diff* displays), by enforcing project-specific policies, and by raising awareness of individual programmers. @@ -145,7 +145,7 @@ This allows identifiers that look the same to humans, but not to Python. For example, all of the following are distinct identifiers: * ``scope`` (Latin, ASCII-only) -* ``scĐŸpe`` (wih a Cyrillic ``ĐŸ``) +* ``scĐŸpe`` (with a Cyrillic ``ĐŸ``) * ``scÎżpe`` (with a Greek ``Îż``) * ``Ń•ŃĐŸŃ€Đ”`` (all Cyrillic letters) @@ -259,7 +259,7 @@ Python strings are collections of *Unicode codepoints*, not “characters”. For reasons like compatibility with earlier encodings, Unicode often has several ways to encode what is essentially a single “character”. -For example, all are these different ways of writing ``Å`` as a Python string, +For example, all these are different ways of writing ``Å`` as a Python string, each of which is unequal to the others. * ``"\N{LATIN CAPITAL LETTER A WITH RING ABOVE}"`` (1 codepoint) diff --git a/pep-0673.rst b/peps/pep-0673.rst similarity index 93% rename from pep-0673.rst rename to peps/pep-0673.rst index 7069fc193..6a1546ead 100644 --- a/pep-0673.rst +++ b/peps/pep-0673.rst @@ -5,20 +5,22 @@ Last-Modified: $Date$ Author: Pradeep Kumar Srinivasan , James Hilton-Balfe Sponsor: Jelle Zijlstra -Discussions-To: Typing-Sig -Status: Draft +Discussions-To: typing-sig@python.org +Status: Accepted Type: Standards Track +Topic: Typing Content-Type: text/x-rst Created: 10-Nov-2021 Python-Version: 3.11 -Post-History: +Post-History: 17-Nov-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/J7BWL5KWOPQQK5KFWKENVLXW6UGSPTGI/ Abstract ======== This PEP introduces a simple and intuitive way to annotate methods that return an instance of their class. This behaves the same as the ``TypeVar``-based -approach specified in `PEP 484 `_ +approach specified in :pep:`484` but is more concise and easier to follow. Motivation @@ -142,6 +144,11 @@ of ``dict`` and 1314 uses of ``Callable`` `as of October 2021 This suggests that a ``Self`` type will be used quite often and users will benefit a lot from the simpler approach above. +Users of Python types have also frequently requested this feature, +both on the `proposal doc +`_ +and on `GitHub `_. + Specification ============= @@ -221,7 +228,8 @@ value of type ``Shape``, when in fact it should be ``Circle``: :: - class Circle(Shape): ... + class Circle(Shape): + def circumference(self) -> float: ... shape = Shape.from_config({"scale": 7.0}) # => Shape @@ -285,7 +293,7 @@ We propose using ``Self`` directly to achieve the same behavior: class Shape: def difference(self, other: Self) -> float: ... - def apply(self, f: Callable[[Self], None]) -> None: 
 + def apply(self, f: Callable[[Self], None]) -> None: ... Note that specifying ``self: Self`` is harmless, so some users may find it more readable to write the above as: @@ -316,7 +324,7 @@ have a ``LinkedList`` whose elements must be subclasses of the current class. # OK LinkedList[int](value=1, next=LinkedList[int](value=2)) # Not OK - LinkedList[int](value=1, next=LinkedList[str](value=”hello”)) + LinkedList[int](value=1, next=LinkedList[str](value="hello")) However, annotating the ``next`` attribute as ``LinkedList[T]`` allows invalid @@ -345,9 +353,8 @@ We propose expressing this constraint using ``next: Self | None``: @dataclass class LinkedList(Generic[T]): - next: Self | None = None value: T - + next: Self | None = None @dataclass class OrdinalLinkedList(LinkedList[int]): @@ -406,7 +413,7 @@ This is equivalent to writing: :: - Self = TypeVar(“Self”, bound=”Container[Any]”) + Self = TypeVar("Self", bound="Container[Any]") class Container(Generic[T]): value: T @@ -425,7 +432,7 @@ an object of generic type ``Container[T]``, ``Self`` is bound to int_container: Container[int] str_container: Container[str] reveal_type(int_container.set_value(42)) # => Container[int] - reveal_type(str_container.set_value(“hello”)) # => Container[str] + reveal_type(str_container.set_value("hello")) # => Container[str] def object_with_generic_type( container: Container[T], value: T, @@ -474,7 +481,7 @@ Use in Protocols from typing import Protocol, Self - class Shape(Protocol): + class ShapeProtocol(Protocol): scale: float def set_scale(self, scale: float) -> Self: @@ -489,7 +496,7 @@ is treated equivalently to: SelfShape = TypeVar("SelfShape", bound="ShapeProtocol") - class Shape(Protocol): + class ShapeProtocol(Protocol): scale: float def set_scale(self: SelfShape, scale: float) -> SelfShape: @@ -497,8 +504,8 @@ is treated equivalently to: return self -See `PEP 544 -`_ for +See :pep:`PEP 544 +<544#self-types-in-protocols>` for details on the behavior of TypeVars bound to protocols. Checking a class for compatibility with a protocol: If a protocol uses @@ -605,11 +612,6 @@ The following uses of ``Self`` are accepted: # Accepted (treated as an @property returning the Callable type) bar: Callable[[Self], int] = foo - TupleSelf = Tuple[Self, Self] - class Alias: - def return_tuple(self) -> TupleSelf: - return (self, self) - class HasNestedFunction: x: int = 42 @@ -658,7 +660,21 @@ The following uses of ``Self`` are rejected. class Bar(Generic[T]): def bar(self) -> T: ... - class Baz(Foo[Self]): ... # Rejected + class Baz(Bar[Self]): ... # Rejected + +We reject type aliases containing ``Self``. Supporting ``Self`` +outside class definitions can require a lot of special-handling in +type checkers. Given that it also goes against the rest of the PEP to +use ``Self`` outside a class definition, we believe the added +convenience of aliases is not worth it: + +:: + + TupleSelf = Tuple[Self, Self] # Rejected + + class Alias: + def return_tuple(self) -> TupleSelf: # Rejected + return (self, self) Note that we reject ``Self`` in staticmethods. ``Self`` does not add much value since there is no ``self`` or ``cls`` to return. The only possible use @@ -781,8 +797,8 @@ Other languages have similar ways to express the type of the enclosing class: Thanks to the following people for their feedback on the PEP: -Jia Chen, Rebecca Chen, Sergei Lebedev, Kaylynn Morgan, Tuomas Suutari, Alex -Waygood, Shannon Zhu, and НоĐșота ĐĄĐŸĐ±ĐŸĐ»Đ”ĐČ +Jia Chen, Rebecca Chen, Sergei Lebedev, Kaylynn Morgan, Tuomas +Suutari, Eric Traut, Alex Waygood, Shannon Zhu, and НоĐșота ĐĄĐŸĐ±ĐŸĐ»Đ”ĐČ Copyright ========= diff --git a/peps/pep-0674.rst b/peps/pep-0674.rst new file mode 100644 index 000000000..c87574a4d --- /dev/null +++ b/peps/pep-0674.rst @@ -0,0 +1,553 @@ +PEP: 674 +Title: Disallow using macros as l-values +Author: Victor Stinner +Status: Deferred +Type: Standards Track +Content-Type: text/x-rst +Created: 30-Nov-2021 +Python-Version: 3.12 + + +Abstract +======== + +Disallow using macros as l-values. For example, +``Py_TYPE(obj) = new_type`` now fails with a compiler error. + +In practice, the majority of affected projects only have to make two +changes: + +* Replace ``Py_TYPE(obj) = new_type`` + with ``Py_SET_TYPE(obj, new_type)``. +* Replace ``Py_SIZE(obj) = new_size`` + with ``Py_SET_SIZE(obj, new_size)``. + + +PEP Deferral +============ + +See `SC reply to PEP 674 -- Disallow using macros as l-values +`_ +(February 2022). + + +Rationale +========= + +Using a macro as a an l-value +----------------------------- + +In the Python C API, some functions are implemented as macro because +writing a macro is simpler than writing a regular function. If a macro +exposes directly a structure member, it is technically possible to use +this macro to not only get the structure member but also set it. + +Example with the Python 3.10 ``Py_TYPE()`` macro:: + + #define Py_TYPE(ob) (((PyObject *)(ob))->ob_type) + +This macro can be used as a **r-value** to **get** an object type:: + + type = Py_TYPE(object); + +It can also be used as an **l-value** to **set** an object type:: + + Py_TYPE(object) = new_type; + +It is also possible to set an object reference count and an object size +using ``Py_REFCNT()`` and ``Py_SIZE()`` macros. + +Setting directly an object attribute relies on the current exact CPython +implementation. Implementing this feature in other Python +implementations can make their C API implementation less efficient. + +CPython nogil fork +------------------ + +Sam Gross forked Python 3.9 to remove the GIL: the `nogil branch +`_. This fork has no +``PyObject.ob_refcnt`` member, but a more elaborated implementation for +reference counting, and so the ``Py_REFCNT(obj) = new_refcnt;`` code +fails with a compiler error. + +Merging the nogil fork into the upstream CPython main branch requires +first to fix this C API compatibility issue. It is a concrete example of +a Python optimization blocked indirectly by the C API. + +This issue was already fixed in Python 3.10: the ``Py_REFCNT()`` macro +has been already modified to disallow using it as an l-value. + +These statements are endorsed by Sam Gross (nogil developer). + +HPy project +----------- + +The `HPy project `_ is a brand new C API for +Python using only handles and function calls: handles are opaque, +structure members cannot be accessed directly, and pointers cannot be +dereferenced. + +Searching and replacing ``Py_SET_SIZE()`` is easier and safer than +searching and replacing some strange macro uses of ``Py_SIZE()``. +``Py_SIZE()`` can be semi-mechanically replaced by ``HPy_Length()``, +whereas seeing ``Py_SET_SIZE()`` would immediately make clear that the +code needs bigger changes in order to be ported to HPy (for example by +using ``HPyTupleBuilder`` or ``HPyListBuilder``). + +The fewer internal details exposed via macros, the easier it will be for +HPy to provide direct equivalents. Any macro that references +"non-public" interfaces effectively exposes those interfaces publicly. + +These statements are endorsed by Antonio Cuni (HPy developer). + +GraalVM Python +-------------- + +In GraalVM, when a Python object is accessed by the Python C API, the C API +emulation layer has to wrap the GraalVM objects into wrappers that expose +the internal structure of the CPython structures (PyObject, PyLongObject, +PyTypeObject, etc). This is because when the C code accesses it directly or via +macros, all GraalVM can intercept is a read at the struct offset, which has +to be mapped back to the representation in GraalVM. The smaller the +"effective" number of exposed struct members (by replacing macros with +functions), the simpler GraalVM wrappers can be. + +This PEP alone is not enough to get rid of the wrappers in GraalVM, but it +is a step towards this long term goal. GraalVM already supports HPy which is a better +solution in the long term. + +These statements are endorsed by Tim Felgentreff (GraalVM Python developer). + +Specification +============= + +Disallow using macros as l-values +---------------------------------- + +The following 65 macros are modified to disallow using them as l-values. + +PyObject and PyVarObject macros +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``Py_TYPE()``: ``Py_SET_TYPE()`` must be used instead +* ``Py_SIZE()``: ``Py_SET_SIZE()`` must be used instead + +GET macros +^^^^^^^^^^ + +* ``PyByteArray_GET_SIZE()`` +* ``PyBytes_GET_SIZE()`` +* ``PyCFunction_GET_CLASS()`` +* ``PyCFunction_GET_FLAGS()`` +* ``PyCFunction_GET_FUNCTION()`` +* ``PyCFunction_GET_SELF()`` +* ``PyCell_GET()`` +* ``PyCode_GetNumFree()`` +* ``PyDict_GET_SIZE()`` +* ``PyFunction_GET_ANNOTATIONS()`` +* ``PyFunction_GET_CLOSURE()`` +* ``PyFunction_GET_CODE()`` +* ``PyFunction_GET_DEFAULTS()`` +* ``PyFunction_GET_GLOBALS()`` +* ``PyFunction_GET_KW_DEFAULTS()`` +* ``PyFunction_GET_MODULE()`` +* ``PyHeapType_GET_MEMBERS()`` +* ``PyInstanceMethod_GET_FUNCTION()`` +* ``PyList_GET_SIZE()`` +* ``PyMemoryView_GET_BASE()`` +* ``PyMemoryView_GET_BUFFER()`` +* ``PyMethod_GET_FUNCTION()`` +* ``PyMethod_GET_SELF()`` +* ``PySet_GET_SIZE()`` +* ``PyTuple_GET_SIZE()`` +* ``PyUnicode_GET_DATA_SIZE()`` +* ``PyUnicode_GET_LENGTH()`` +* ``PyUnicode_GET_LENGTH()`` +* ``PyUnicode_GET_SIZE()`` +* ``PyWeakref_GET_OBJECT()`` + +AS macros +^^^^^^^^^ + +* ``PyByteArray_AS_STRING()`` +* ``PyBytes_AS_STRING()`` +* ``PyFloat_AS_DOUBLE()`` +* ``PyUnicode_AS_DATA()`` +* ``PyUnicode_AS_UNICODE()`` + +PyUnicode macros +^^^^^^^^^^^^^^^^ + +* ``PyUnicode_1BYTE_DATA()`` +* ``PyUnicode_2BYTE_DATA()`` +* ``PyUnicode_4BYTE_DATA()`` +* ``PyUnicode_DATA()`` +* ``PyUnicode_IS_ASCII()`` +* ``PyUnicode_IS_COMPACT()`` +* ``PyUnicode_IS_READY()`` +* ``PyUnicode_KIND()`` +* ``PyUnicode_READ()`` +* ``PyUnicode_READ_CHAR()`` + +PyDateTime GET macros +^^^^^^^^^^^^^^^^^^^^^ + +* ``PyDateTime_DATE_GET_FOLD()`` +* ``PyDateTime_DATE_GET_HOUR()`` +* ``PyDateTime_DATE_GET_MICROSECOND()`` +* ``PyDateTime_DATE_GET_MINUTE()`` +* ``PyDateTime_DATE_GET_SECOND()`` +* ``PyDateTime_DATE_GET_TZINFO()`` +* ``PyDateTime_DELTA_GET_DAYS()`` +* ``PyDateTime_DELTA_GET_MICROSECONDS()`` +* ``PyDateTime_DELTA_GET_SECONDS()`` +* ``PyDateTime_GET_DAY()`` +* ``PyDateTime_GET_MONTH()`` +* ``PyDateTime_GET_YEAR()`` +* ``PyDateTime_TIME_GET_FOLD()`` +* ``PyDateTime_TIME_GET_HOUR()`` +* ``PyDateTime_TIME_GET_MICROSECOND()`` +* ``PyDateTime_TIME_GET_MINUTE()`` +* ``PyDateTime_TIME_GET_SECOND()`` +* ``PyDateTime_TIME_GET_TZINFO()`` + +Port C extensions to Python 3.11 +-------------------------------- + +In practice, the majority of projects affected by these PEP only have to +make two changes: + +* Replace ``Py_TYPE(obj) = new_type`` + with ``Py_SET_TYPE(obj, new_type)``. +* Replace ``Py_SIZE(obj) = new_size`` + with ``Py_SET_SIZE(obj, new_size)``. + +The `pythoncapi_compat project +`_ can be used to +update automatically C extensions: add Python 3.11 support without +losing support with older Python versions. The project provides a header +file which provides ``Py_SET_REFCNT()``, ``Py_SET_TYPE()`` and +``Py_SET_SIZE()`` functions to Python 3.8 and older. + +PyTuple_GET_ITEM() and PyList_GET_ITEM() are left unchanged +----------------------------------------------------------- + +The ``PyTuple_GET_ITEM()`` and ``PyList_GET_ITEM()`` macros are left +unchanged. + +The code patterns ``&PyTuple_GET_ITEM(tuple, 0)`` and +``&PyList_GET_ITEM(list, 0)`` are still commonly used to get access to +the inner ``PyObject**`` array. + +Changing these macros is out of the scope of this PEP. + +PyDescr_NAME() and PyDescr_TYPE() are left unchanged +---------------------------------------------------- + +The ``PyDescr_NAME()`` and ``PyDescr_TYPE()`` macros are left unchanged. + +These macros give access to ``PyDescrObject.d_name`` and +``PyDescrObject.d_type`` members. They can be used as l-values to set +these members. + +The SWIG project uses these macros as l-values to set these members. It +would be possible to modify SWIG to prevent setting ``PyDescrObject`` +structure members directly, but it is not really worth it since the +``PyDescrObject`` structure is not performance critical and is unlikely +to change soon. + +See the `bpo-46538 `_ "[C API] Make +the PyDescrObject structure opaque: PyDescr_NAME() and PyDescr_TYPE()" +issue for more details. + + +Implementation +============== + +The implementation is tracked by `bpo-45476: [C API] PEP 674: Disallow +using macros as l-values `_. + +Py_TYPE() and Py_SIZE() macros +------------------------------ + +In May 2020, the ``Py_TYPE()`` and ``Py_SIZE()`` macros have been +modified to disallow using them as l-values (`Py_TYPE +`_, +`Py_SIZE +`_). + +In November 2020, the change was `reverted +`__, +since it broke too many third party projects. + +In June 2021, once most third party projects were updated, a `second +attempt +`_ +was done, but had to be `reverted again +`__ +, since it broke test_exceptions on Windows. + +In September 2021, once `test_exceptions has been fixed +`_, +Py_TYPE() and Py_SIZE() were finally `changed +`_. + +In November 2021, this backward incompatible change got a +`Steering Council exception +`_. + +In October 2022, Python 3.11 got released with Py_TYPE() and Py_SIZE() +incompatible changes. + +Backwards Compatibility +======================= + +The proposed C API changes are backward incompatible on purpose. + +In practice, only ``Py_TYPE()`` and ``Py_SIZE()`` macros are used as +l-values. + +This change does not follow the :pep:`387` deprecation process. There is +no known way to emit a deprecation warning only when a macro is used as +an l-value, but not when it's used differently (ex: as a r-value). + +The following 4 macros are left unchanged to reduce the number of +affected projects: ``PyDescr_NAME()``, ``PyDescr_TYPE()``, +``PyList_GET_ITEM()`` and ``PyTuple_GET_ITEM()``. + +Statistics +---------- + +In total (projects on PyPI and not on PyPI), 34 projects are known to be +affected by this PEP: + +* 16 projects (47%) are already fixed +* 18 projects (53%) are not fixed yet + (pending fix or have to regenerate their Cython code) + +On September 1, 2022, the PEP affects 18 projects (0.4%) of the top 5000 +PyPI projects: + +* 15 projects (0.3%) have to regenerate their Cython code +* 3 projects (0.1%) have a pending fix + +Top 5000 PyPI +------------- + +Projects with a pending fix (3): + +* datatable (1.0.0): + `fixed `__ +* guppy3 (3.1.2): + `fixed `__ +* scipy (1.9.3): need to update boost python + +Moreover, 15 projects have to regenerate their Cython code. + +Projects released with a fix (12): + +* bitarray (1.6.2): + `commit `__ +* Cython (0.29.20): `commit `__ +* immutables (0.15): + `commit `__ +* mercurial (5.7): + `commit `__, + `bug report `__ +* mypy (v0.930): + `commit `__ +* numpy (1.22.1): + `commit `__, + `commit 2 `__ +* pycurl (7.44.1): + `commit `__ +* PyGObject (3.42.0) +* pyside2 (5.15.1): + `bug report `__ +* python-snappy (0.6.1): + `fixed `__ +* recordclass (0.17.2): + `fixed `__ +* zstd (1.5.0.3): + `commit `__ + +There are also two backport projects which are affected by this PEP: + +* pickle5 (0.0.12): backport for Python <= 3.7 +* pysha3 (1.0.2): backport for Python <= 3.5 + +They must not be used and cannot be used on Python 3.11. + +Other affected projects +----------------------- + +Other projects released with a fix (4): + +* boost (1.78.0): + `commit `__ +* breezy (3.2.1): + `bug report `__ +* duplicity (0.8.18): + `commit `__ +* gobject-introspection (1.70.0): + `MR `__ + + +Relationship with the HPy project +================================= + +The HPy project +--------------- + +The hope with the HPy project is to provide a C API that is close +to the original API—to make porting easy—and have it perform as close to +the existing API as possible. At the same time, HPy is sufficiently +removed to be a good "C extension API" (as opposed to a stable subset of +the CPython implementation API) that does not leak implementation +details. To ensure this latter property, the HPy project tries to +develop everything in parallel for CPython, PyPy, and GraalVM Python. + +HPy is still evolving very fast. Issues are still being solved while +migrating NumPy, and work has begun on adding support for HPy to Cython. Work on +pybind11 is starting soon. Tim Felgentreff believes by the time HPy has +these users of the existing C API working, HPy should be in a state +where it is generally useful and can be deemed stable enough that +further development can follow a more stable process. + +In the long run the HPy project would like to become a promoted API to +write Python C extensions. + +The HPy project is a good solution for the long term. It has the +advantage of being developed outside Python and it doesn't require any C +API change. + +The C API is here is stay for a few more years +---------------------------------------------- + +The first concern about HPy is that right now, HPy is not mature nor +widely used, and CPython still has to continue supporting a large amount +of C extensions which are not likely to be ported to HPy soon. + +The second concern is the inability to evolve CPython internals to +implement new optimizations, and the inefficient implementation of the +current C API in PyPy, GraalPython, etc. Sadly, HPy will only solve +these problems when most C extensions will be fully ported to HPy: +when it will become reasonable to consider dropping the "legacy" Python +C API. + +While porting a C extension to HPy can be done incrementally on CPython, +it requires to modify a lot of code and takes time. Porting most C +extensions to HPy is expected to take a few years. + +This PEP proposes to make the C API "less bad" by fixing one problem +which is clearily identified as causing practical issues: macros used as +l-values. This PEP only requires updating a minority of C +extensions, and usually only a few lines need to be changed in impacted +extensions. + +For example, NumPy 1.22 is made of 307,300 lines of C code, and adapting +NumPy to the this PEP only modified 11 lines (use Py_SET_TYPE and +Py_SET_SIZE) and adding 4 lines (to define Py_SET_TYPE and Py_SET_SIZE +for Python 3.8 and older). The beginnings of the NumPy port to HPy +already required modifying more lines than that. + +Right now, it's hard to bet which approach is the best: fixing the +current C API, or focusing on HPy. It would be risky to only focus on +HPy. + + +Rejected Idea: Leave the macros as they are +=========================================== + +The documentation of each function can discourage developers to use +macros to modify Python objects. + +If these is a need to make an assignment, a setter function can be added +and the macro documentation can require to use the setter function. For +example, a ``Py_SET_TYPE()`` function has been added to Python 3.9 and +the ``Py_TYPE()`` documentation now requires to use the +``Py_SET_TYPE()`` function to set an object type. + +If developers use macros as an l-value, it's their responsibility when +their code breaks, not Python's responsibility. We are operating under +the consenting adults principle: we expect users of the Python C API to +use it as documented and expect them to take care of the fallout, if +things break when they don't. + +This idea was rejected because only few developers read the +documentation, and only a minority is tracking changes of the Python C +API documentation. The majority of developers are only using CPython and +so are not aware of compatibility issues with other Python +implementations. + +Moreover, continuing to allow using macros as an l-value does not help +the HPy project, and leaves the burden of emulating them on GraalVM's +Python implementation. + + +Macros already modified +======================= + +The following C API macros have already been modified to disallow using +them as l-value: + +* ``PyCell_SET()`` +* ``PyList_SET_ITEM()`` +* ``PyTuple_SET_ITEM()`` +* ``Py_REFCNT()`` (Python 3.10): ``Py_SET_REFCNT()`` must be used +* ``_PyGCHead_SET_FINALIZED()`` +* ``_PyGCHead_SET_NEXT()`` +* ``asdl_seq_GET()`` +* ``asdl_seq_GET_UNTYPED()`` +* ``asdl_seq_LEN()`` +* ``asdl_seq_SET()`` +* ``asdl_seq_SET_UNTYPED()`` + +For example, ``PyList_SET_ITEM(list, 0, item) < 0`` now fails with a +compiler error as expected. + + +Post History +============ + +* `PEP 674 "Disallow using macros as l-values" and Python 3.11 + `__ (August 18, 2022) +* `SC reply to PEP 674 -- Disallow using macros as l-values + `__ (February 22, 2022) +* `PEP 674: Disallow using macros as l-value (version 2) + `_ + (Jan 18, 2022) +* `PEP 674: Disallow using macros as l-value + `_ + (Nov 30, 2021) + + +References +========== + +* `Python C API: Add functions to access PyObject + `_ (October + 2021) article by Victor Stinner +* `[capi-sig] Py_TYPE() and Py_SIZE() become static inline functions + `_ + (September 2021) +* `[C API] Avoid accessing PyObject and PyVarObject members directly: add Py_SET_TYPE() and Py_IS_TYPE(), disallow Py_TYPE(obj)=type + `__ (February 2020) +* `bpo-30459: PyList_SET_ITEM could be safer + `_ (May 2017) + + +Version History +=============== + +* Version 3: No longer change PyDescr_TYPE() and PyDescr_NAME() macros +* Version 2: Add "Relationship with the HPy project" section, remove + the PyPy section +* Version 1: First public version + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0675.rst b/peps/pep-0675.rst new file mode 100644 index 000000000..505b89f33 --- /dev/null +++ b/peps/pep-0675.rst @@ -0,0 +1,1311 @@ +PEP: 675 +Title: Arbitrary Literal String Type +Version: $Revision$ +Last-Modified: $Date$ +Author: Pradeep Kumar Srinivasan , Graham Bleaney +Sponsor: Jelle Zijlstra +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/VB74EHNM4RODDFM64NEEEBJQVAUAWIAW/ +Status: Accepted +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 30-Nov-2021 +Python-Version: 3.11 +Post-History: 07-Feb-2022 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/XEOOSSPNYPGZ5NXOJFPLXG2BTN7EVRT5/ + +Abstract +======== + +There is currently no way to specify, using type annotations, that a +function parameter can be of any literal string type. We have to +specify a precise literal string type, such as +``Literal["foo"]``. This PEP introduces a supertype of literal string +types: ``LiteralString``. This allows a function to accept arbitrary +literal string types, such as ``Literal["foo"]`` or +``Literal["bar"]``. + + +Motivation +========== + +Powerful APIs that execute SQL or shell commands often recommend that +they be invoked with literal strings, rather than arbitrary user +controlled strings. There is no way to express this recommendation in +the type system, however, meaning security vulnerabilities sometimes +occur when developers fail to follow it. For example, a naive way to +look up a user record from a database is to accept a user id and +insert it into a predefined SQL query: + +:: + + def query_user(conn: Connection, user_id: str) -> User: + query = f"SELECT * FROM data WHERE user_id = {user_id}" + conn.execute(query) + ... # Transform data to a User object and return it + + query_user(conn, "user123") # OK. + +However, the user-controlled data ``user_id`` is being mixed with the +SQL command string, which means a malicious user could run arbitrary +SQL commands: + +:: + + # Delete the table. + query_user(conn, "user123; DROP TABLE data;") + + # Fetch all users (since 1 = 1 is always true). + query_user(conn, "user123 OR 1 = 1") + + +To prevent such SQL injection attacks, SQL APIs offer parameterized +queries, which separate the executed query from user-controlled data +and make it impossible to run arbitrary queries. For example, with +`sqlite3 `_, our +original function would be written safely as a query with parameters: + +:: + + def query_user(conn: Connection, user_id: str) -> User: + query = "SELECT * FROM data WHERE user_id = ?" + conn.execute(query, (user_id,)) + ... + +The problem is that there is no way to enforce this +discipline. sqlite3's own `documentation +`_ can only admonish +the reader to not dynamically build the ``sql`` argument from external +input; the API's authors cannot express that through the type +system. Users can (and often do) still use a convenient f-string as +before and leave their code vulnerable to SQL injection. + +Existing tools, such as the popular security linter `Bandit +`_, +attempt to detect unsafe external data used in SQL APIs, by inspecting +the AST or by other semantic pattern-matching. These tools, however, +preclude common idioms like storing a large multi-line query in a +variable before executing it, adding literal string modifiers to the +query based on some conditions, or transforming the query string using +a function. (We survey existing tools in the `Rejected Alternatives`_ +section.) For example, many tools will detect a false positive issue +in this benign snippet: + + +:: + + def query_data(conn: Connection, user_id: str, limit: bool) -> None: + query = """ + SELECT + user.name, + user.age + FROM data + WHERE user_id = ? + """ + if limit: + query += " LIMIT 1" + + conn.execute(query, (user_id,)) + +We want to forbid harmful execution of user-controlled data while +still allowing benign idioms like the above and not requiring extra +user work. + +To meet this goal, we introduce the ``LiteralString`` type, which only +accepts string values that are known to be made of literals. This is a +generalization of the ``Literal["foo"]`` type from :pep:`586`. +A string of type +``LiteralString`` cannot contain user-controlled data. Thus, any API +that only accepts ``LiteralString`` will be immune to injection +vulnerabilities (with `pragmatic limitations `_). + +Since we want the ``sqlite3`` ``execute`` method to disallow strings +built with user input, we would make its `typeshed stub +`_ +accept a ``sql`` query that is of type ``LiteralString``: + +:: + + from typing import LiteralString + + def execute(self, sql: LiteralString, parameters: Iterable[str] = ...) -> Cursor: ... + + +This successfully forbids our unsafe SQL example. The variable +``query`` below is inferred to have type ``str``, since it is created +from a format string using ``user_id``, and cannot be passed to +``execute``: + +:: + + def query_user(conn: Connection, user_id: str) -> User: + query = f"SELECT * FROM data WHERE user_id = {user_id}" + conn.execute(query) # Error: Expected LiteralString, got str. + ... + +The method remains flexible enough to allow our more complicated +example: + +:: + + def query_data(conn: Connection, user_id: str, limit: bool) -> None: + # This is a literal string. + query = """ + SELECT + user.name, + user.age + FROM data + WHERE user_id = ? + """ + + if limit: + # Still has type LiteralString because we added a literal string. + query += " LIMIT 1" + + conn.execute(query, (user_id,)) # OK + +Notice that the user did not have to change their SQL code at all. The +type checker was able to infer the literal string type and complain +only in case of violations. + +``LiteralString`` is also useful in other cases where we want strict +command-data separation, such as when building shell commands or when +rendering a string into an HTML response without escaping (see +`Appendix A: Other Uses`_). Overall, this combination of strictness +and flexibility makes it easy to enforce safer API usage in sensitive +code without burdening users. + +Usage statistics +---------------- + +In a sample of open-source projects using ``sqlite3``, we found that +``conn.execute`` was called `~67% of the time +`_ +with a safe string literal and `~33% of the time +`_ +with a potentially unsafe, local string variable. Using this PEP's +literal string type along with a type checker would prevent the unsafe +portion of that 33% of cases (ie. the ones where user controlled data +is incorporated into the query), while seamlessly allowing the safe +ones to remain. + +Rationale +========= + +Firstly, why use *types* to prevent security vulnerabilities? + +Warning users in documentation is insufficient - most users either +never see these warnings or ignore them. Using an existing dynamic or +static analysis approach is too restrictive - these prevent natural +idioms, as we saw in the `Motivation`_ section (and will discuss more +extensively in the `Rejected Alternatives`_ section). The typing-based +approach in this PEP strikes a user-friendly balance between +strictness and flexibility. + +Runtime approaches do not work because, at runtime, the query string +is a plain ``str``. While we could prevent some exploits using +heuristics, such as regex-filtering for obviously malicious payloads, +there will always be a way to work around them (perfectly +distinguishing good and bad queries reduces to the halting problem). + +Static approaches, such as checking the AST to see if the query string +is a literal string expression, cannot tell when a string is assigned +to an intermediate variable or when it is transformed by a benign +function. This makes them overly restrictive. + +The type checker, surprisingly, does better than both because it has +access to information not available in the runtime or static analysis +approaches. Specifically, the type checker can tell us whether an +expression has a literal string type, say ``Literal["foo"]``. The type +checker already propagates types across variable assignments or +function calls. + +In the current type system itself, if the SQL or shell command +execution function only accepted three possible input strings, our job +would be done. We would just say: + +:: + + def execute(query: Literal["foo", "bar", "baz"]) -> None: ... + +But, of course, ``execute`` can accept *any* possible query. How do we +ensure that the query does not contain an arbitrary, user-controlled +string? + +We want to specify that the value must be of some type +``Literal[<...>]`` where ``<...>`` is some string. This is what +``LiteralString`` represents. ``LiteralString`` is the "supertype" of +all literal string types. In effect, this PEP just introduces a type +in the type hierarchy between ``Literal["foo"]`` and ``str``. Any +particular literal string, such as ``Literal["foo"]`` or +``Literal["bar"]``, is compatible with ``LiteralString``, but not the +other way around. The "supertype" of ``LiteralString`` itself is +``str``. So, ``LiteralString`` is compatible with ``str``, but not the +other way around. + +Note that a ``Union`` of literal types is naturally compatible with +``LiteralString`` because each element of the ``Union`` is individually +compatible with ``LiteralString``. So, ``Literal["foo", "bar"]`` is +compatible with ``LiteralString``. + +However, recall that we don't just want to represent exact literal +queries. We also want to support composition of two literal strings, +such as ``query + " LIMIT 1"``. This too is possible with the above +concept. If ``x`` and ``y`` are two values of type ``LiteralString``, +then ``x + y`` will also be of type compatible with +``LiteralString``. We can reason about this by looking at specific +instances such as ``Literal["foo"]`` and ``Literal["bar"]``; the value +of the added string ``x + y`` can only be ``"foobar"``, which has type +``Literal["foobar"]`` and is thus compatible with +``LiteralString``. The same reasoning applies when ``x`` and ``y`` are +unions of literal types; the result of pairwise adding any two literal +types from ``x`` and ``y`` respectively is a literal type, which means +that the overall result is a ``Union`` of literal types and is thus +compatible with ``LiteralString``. + +In this way, we are able to leverage Python's concept of a ``Literal`` +string type to specify that our API can only accept strings that are +known to be constructed from literals. More specific details follow in +the remaining sections. + +Specification +============= + + +Runtime Behavior +---------------- + +We propose adding ``LiteralString`` to ``typing.py``, with an +implementation similar to ``typing.NoReturn``. + +Note that ``LiteralString`` is a special form used solely for type +checking. There is no expression for which ``type()`` will +produce ``LiteralString`` at runtime. So, we do not specify in the +implementation that it is a subclass of ``str``. + + +Valid Locations for ``LiteralString`` +----------------------------------------- + +``LiteralString`` can be used where any other type can be used: + +:: + + variable_annotation: LiteralString + + def my_function(literal_string: LiteralString) -> LiteralString: ... + + class Foo: + my_attribute: LiteralString + + type_argument: List[LiteralString] + + T = TypeVar("T", bound=LiteralString) + +It cannot be nested within unions of ``Literal`` types: + +:: + + bad_union: Literal["hello", LiteralString] # Not OK + bad_nesting: Literal[LiteralString] # Not OK + + +Type Inference +-------------- + +.. _inferring_literal_string: + + +Inferring ``LiteralString`` +''''''''''''''''''''''''''' + +Any literal string type is compatible with ``LiteralString``. For +example, ``x: LiteralString = "foo"`` is valid because ``"foo"`` is +inferred to be of type ``Literal["foo"]``. + +As per the `Rationale`_, we also infer ``LiteralString`` in the +following cases: + ++ Addition: ``x + y`` is of type ``LiteralString`` if both ``x`` and + ``y`` are compatible with ``LiteralString``. + ++ Joining: ``sep.join(xs)`` is of type ``LiteralString`` if ``sep``'s + type is compatible with ``LiteralString`` and ``xs``'s type is + compatible with ``Iterable[LiteralString]``. + ++ In-place addition: If ``s`` has type ``LiteralString`` and ``x`` has + type compatible with ``LiteralString``, then ``s += x`` preserves + ``s``'s type as ``LiteralString``. + ++ String formatting: An f-string has type ``LiteralString`` if and only + if its constituent expressions are literal strings. ``s.format(...)`` + has type ``LiteralString`` if and only if ``s`` and the arguments have + types compatible with ``LiteralString``. + ++ Literal-preserving methods: In `Appendix C `_, + we have provided an exhaustive list of ``str`` methods that preserve the + ``LiteralString`` type. + +In all other cases, if one or more of the composed values has a +non-literal type ``str``, the composition of types will have type +``str``. For example, if ``s`` has type ``str``, then ``"hello" + s`` +has type ``str``. This matches the pre-existing behavior of type +checkers. + +``LiteralString`` is compatible with the type ``str``. It inherits all +methods from ``str``. So, if we have a variable ``s`` of type +``LiteralString``, it is safe to write ``s.startswith("hello")``. + +Some type checkers refine the type of a string when doing an equality +check: + +:: + + def foo(s: str) -> None: + if s == "bar": + reveal_type(s) # => Literal["bar"] + +Such a refined type in the if-block is also compatible with +``LiteralString`` because its type is ``Literal["bar"]``. + + +Examples +'''''''' + +See the examples below to help clarify the above rules: + +:: + + + literal_string: LiteralString + s: str = literal_string # OK + + literal_string: LiteralString = s # Error: Expected LiteralString, got str. + literal_string: LiteralString = "hello" # OK + +Addition of literal strings: + +:: + + def expect_literal_string(s: LiteralString) -> None: ... + + expect_literal_string("foo" + "bar") # OK + expect_literal_string(literal_string + "bar") # OK + + literal_string2: LiteralString + expect_literal_string(literal_string + literal_string2) # OK + + plain_string: str + expect_literal_string(literal_string + plain_string) # Not OK. + +Join using literal strings: + +:: + + expect_literal_string(",".join(["foo", "bar"])) # OK + expect_literal_string(literal_string.join(["foo", "bar"])) # OK + expect_literal_string(literal_string.join([literal_string, literal_string2])) # OK + + xs: List[LiteralString] + expect_literal_string(literal_string.join(xs)) # OK + expect_literal_string(plain_string.join([literal_string, literal_string2])) + # Not OK because the separator has type 'str'. + +In-place addition using literal strings: + +:: + + literal_string += "foo" # OK + literal_string += literal_string2 # OK + literal_string += plain_string # Not OK + +Format strings using literal strings: + +:: + + literal_name: LiteralString + expect_literal_string(f"hello {literal_name}") + # OK because it is composed from literal strings. + + expect_literal_string("hello {}".format(literal_name)) # OK + + expect_literal_string(f"hello") # OK + + username: str + expect_literal_string(f"hello {username}") + # NOT OK. The format-string is constructed from 'username', + # which has type 'str'. + + expect_literal_string("hello {}".format(username)) # Not OK + +Other literal types, such as literal integers, are not compatible with ``LiteralString``: + +:: + + some_int: int + expect_literal_string(some_int) # Error: Expected LiteralString, got int. + + literal_one: Literal[1] = 1 + expect_literal_string(literal_one) # Error: Expected LiteralString, got Literal[1]. + + +We can call functions on literal strings: + +:: + + def add_limit(query: LiteralString) -> LiteralString: + return query + " LIMIT = 1" + + def my_query(query: LiteralString, user_id: str) -> None: + sql_connection().execute(add_limit(query), (user_id,)) # OK + +Conditional statements and expressions work as expected: + +:: + + def return_literal_string() -> LiteralString: + return "foo" if condition1() else "bar" # OK + + def return_literal_str2(literal_string: LiteralString) -> LiteralString: + return "foo" if condition1() else literal_string # OK + + def return_literal_str3() -> LiteralString: + if condition1(): + result: Literal["foo"] = "foo" + else: + result: LiteralString = "bar" + + return result # OK + + +Interaction with TypeVars and Generics +'''''''''''''''''''''''''''''''''''''' + +TypeVars can be bound to ``LiteralString``: + +:: + + from typing import Literal, LiteralString, TypeVar + + TLiteral = TypeVar("TLiteral", bound=LiteralString) + + def literal_identity(s: TLiteral) -> TLiteral: + return s + + hello: Literal["hello"] = "hello" + y = literal_identity(hello) + reveal_type(y) # => Literal["hello"] + + s: LiteralString + y2 = literal_identity(s) + reveal_type(y2) # => LiteralString + + s_error: str + literal_identity(s_error) + # Error: Expected TLiteral (bound to LiteralString), got str. + + +``LiteralString`` can be used as a type argument for generic classes: + +:: + + class Container(Generic[T]): + def __init__(self, value: T) -> None: + self.value = value + + literal_string: LiteralString = "hello" + x: Container[LiteralString] = Container(literal_string) # OK + + s: str + x_error: Container[LiteralString] = Container(s) # Not OK + +Standard containers like ``List`` work as expected: + +:: + + xs: List[LiteralString] = ["foo", "bar", "baz"] + + +Interactions with Overloads +''''''''''''''''''''''''''' + +Literal strings and overloads do not need to interact in a special +way: the existing rules work fine. ``LiteralString`` can be used as a +fallback overload where a specific ``Literal["foo"]`` type does not +match: + +:: + + @overload + def foo(x: Literal["foo"]) -> int: ... + @overload + def foo(x: LiteralString) -> bool: ... + @overload + def foo(x: str) -> str: ... + + x1: int = foo("foo") # First overload. + x2: bool = foo("bar") # Second overload. + s: str + x3: str = foo(s) # Third overload. + + +Backwards Compatibility +======================= + +We propose adding ``typing_extensions.LiteralString`` for use in +earlier Python versions. + +As :pep:`PEP 586 mentions +<586#backwards-compatibility>`, +type checkers "should feel free to experiment with more sophisticated +inference techniques". So, if the type checker infers a literal string +type for an unannotated variable that is initialized with a literal +string, the following example should be OK: + +:: + + x = "hello" + expect_literal_string(x) + # OK, because x is inferred to have type 'Literal["hello"]'. + +This enables precise type checking of idiomatic SQL query code without +annotating the code at all (as seen in the `Motivation`_ section +example). + +However, like :pep:`586`, this PEP does not mandate the above inference +strategy. In case the type checker doesn't infer ``x`` to have type +``Literal["hello"]``, users can aid the type checker by explicitly +annotating it as ``x: LiteralString``: + +:: + + x: LiteralString = "hello" + expect_literal_string(x) + + +Rejected Alternatives +===================== + +Why not use tool X? +------------------- + +Tools to catch issues such as SQL injection seem to come in three +flavors: AST based, function level analysis, and taint flow analysis. + +**AST-based tools**: `Bandit +`_ +has a plugin to warn when SQL queries are not literal +strings. The problem is that many perfectly safe SQL +queries are dynamically built out of string literals, as shown in the +`Motivation`_ section. At the +AST level, the resultant SQL query is not going to appear as a string +literal anymore and is thus indistinguishable from a potentially +malicious string. To use these tools would require significantly +restricting developers' ability to build SQL queries. ``LiteralString`` +can provide similar safety guarantees with fewer restrictions. + +**Semgrep and pyanalyze**: Semgrep supports a more sophisticated +function level analysis, including `constant propagation +`_ +within a function. This allows us to prevent injection attacks while +permitting some forms of safe dynamic SQL queries within a +function. `pyanalyze +`_ +has a similar extension. But neither handles function calls that +construct and return safe SQL queries. For example, in the code sample +below, ``build_insert_query`` is a helper function to create a query +that inserts multiple values into the corresponding columns. Semgrep +and pyanalyze forbid this natural usage whereas ``LiteralString`` +handles it with no burden on the programmer: + +:: + + def build_insert_query( + table: LiteralString + insert_columns: Iterable[LiteralString], + ) -> LiteralString: + sql = "INSERT INTO " + table + + column_clause = ", ".join(insert_columns) + value_clause = ", ".join(["?"] * len(insert_columns)) + + sql += f" ({column_clause}) VALUES ({value_clause})" + return sql + + def insert_data( + conn: Connection, + kvs_to_insert: Dict[LiteralString, str] + ) -> None: + query = build_insert_query("data", kvs_to_insert.keys()) + conn.execute(query, kvs_to_insert.values()) + + # Example usage + data_to_insert = { + "column_1": value_1, # Note: values are not literals + "column_2": value_2, + "column_3": value_3, + } + insert_data(conn, data_to_insert) + + +**Taint flow analysis**: Tools such as `Pysa +`_ or `CodeQL +`_ are capable of tracking data flowing +from a user controlled input into a SQL query. These tools are +powerful but involve considerable overhead in setting up the tool in +CI, defining "taint" sinks and sources, and teaching developers how to +use them. They also usually take longer to run than a type checker +(minutes instead of seconds), which means feedback is not +immediate. Finally, they move the burden of preventing vulnerabilities +on to library users instead of allowing the libraries themselves to +specify precisely how their APIs must be called (as is possible with +``LiteralString``). + +One final reason to prefer using a new type over a dedicated tool is +that type checkers are more widely used than dedicated security +tooling; for example, MyPy was downloaded `over 7 million times +`_ in Jan 2022 vs `less than +2 million times `_ for +Bandit. Having security protections built right into type checkers +will mean that more developers benefit from them. + + +Why not use a ``NewType`` for ``str``? +-------------------------------------- + +Any API for which ``LiteralString`` would be suitable could instead be +updated to accept a different type created within the Python type +system, such as ``NewType("SafeSQL", str)``: + +:: + + SafeSQL = NewType("SafeSQL", str) + + def execute(self, sql: SafeSQL, parameters: Iterable[str] = ...) -> Cursor: ... + + execute(SafeSQL("SELECT * FROM data WHERE user_id = ?"), user_id) # OK + + user_query: str + execute(user_query) # Error: Expected SafeSQL, got str. + + +Having to create a new type to call an API might give some developers +pause and encourage more caution, but it doesn't guarantee that +developers won't just turn a user controlled string into the new type, +and pass it into the modified API anyway: + +:: + + query = f"SELECT * FROM data WHERE user_id = f{user_id}" + execute(SafeSQL(query)) # No error! + +We are back to square one with the problem of preventing arbitrary +inputs to ``SafeSQL``. This is not a theoretical concern +either. Django uses the above approach with ``SafeString`` and +`mark_safe +`_. Issues +such as `CVE-2020-13596 +`_ +show how this technique can `fail +`_. + +Also note that this requires invasive changes to the source code +(wrapping the query with ``SafeSQL``) whereas ``LiteralString`` +requires no such changes. Users can remain oblivious to it as long as +they pass in literal strings to sensitive APIs. + +Why not try to emulate Trusted Types? +------------------------------------- + +`Trusted Types +`_ is a W3C +specification for preventing DOM-based Cross Site Scripting (XSS). XSS +occurs when dangerous browser APIs accept raw user-controlled +strings. The specification modifies these APIs to accept only the +"Trusted Types" returned by designated sanitizing functions. These +sanitizing functions must take in a potentially malicious string and +validate it or render it benign somehow, for example by verifying that +it is a valid URL or HTML-encoding it. + +It can be tempting to assume porting the concept of Trusted Types to +Python could solve the problem. The fundamental difference, however, +is that the output of a Trusted Types sanitizer is usually intended +*to not be executable code*. Thus it's easy to HTML encode the input, +strip out dangerous tags, or otherwise render it inert. With a SQL +query or shell command, the end result *still needs to be executable +code*. There is no way to write a sanitizer that can reliably figure +out which parts of an input string are benign and which ones are +potentially malicious. + +Runtime Checkable ``LiteralString`` +----------------------------------- + +The ``LiteralString`` concept could be extended beyond static type +checking to be a runtime checkable property of ``str`` objects. This +would provide some benefits, such as allowing frameworks to raise +errors on dynamic strings. Such runtime errors would be a more robust +defense mechanism than type errors, which can potentially be +suppressed, ignored, or never even seen if the author does not use a +type checker. + +This extension to the ``LiteralString`` concept would dramatically +increase the scope of the proposal by requiring changes to one of the +most fundamental types in Python. While runtime taint checking on +strings, similar to Perl's `taint `_, +has been `considered `_ and +`attempted `_ in the past, and +others may consider it in the future, such extensions are out of scope +for this PEP. + + +Rejected Names +-------------- + +We considered a variety of names for the literal string type and +solicited ideas on `typing-sig +`_. +Some notable alternatives were: + ++ ``Literal[str]``: This is a natural extension of the + ``Literal["foo"]`` type name, but typing-sig `objected + `_ + that users could mistake this for the literal type of the ``str`` + class. + ++ ``LiteralStr``: This is shorter than ``LiteralString`` but looks + weird to the PEP authors. + ++ ``LiteralDerivedString``: This (along with + ``MadeFromLiteralString``) best captures the technical meaning of + the type. It represents not just the type of literal expressions, + such as ``"foo"``, but also that of expressions composed from + literals, such as ``"foo" + "bar"``. However, both names seem wordy. + ++ ``StringLiteral``: Users might confuse this with the existing + concept of `"string literals" + `_ + where the string exists as a syntactic token in the source code, + whereas our concept is more general. + ++ ``SafeString``: While this comes close to our intended meaning, it + may mislead users into thinking that the string has been sanitized in + some way, perhaps by escaping HTML tags or shell-related special + characters. + ++ ``ConstantStr``: This does not capture the idea of composing literal + strings. + ++ ``StaticStr``: This suggests that the string is statically + computable, i.e., computable without running the program, which is + not true. The literal string may vary based on runtime flags, as + seen in the `Motivation`_ examples. + ++ ``LiteralOnly[str]``: This has the advantage of being extensible to + other literal types, such as ``bytes`` or ``int``. However, we did + not find the extensibility worth the loss of readability. + +Overall, there was no clear winner on typing-sig over a long period, +so we decided to tip the scales in favor of ``LiteralString``. + + +``LiteralBytes`` +---------------- + +We could generalize literal byte types, such as ``Literal[b"foo"]``, +to ``LiteralBytes``. However, literal byte types are used much less +frequently than literal string types and we did not find much user +demand for ``LiteralBytes``, so we decided not to include it in this +PEP. Others may, however, consider it in future PEPs. + + +Reference Implementation +======================== + +This is implemented in Pyre v0.9.8 and is actively being used. + +The implementation simply extends the type checker with +``LiteralString`` as a supertype of literal string types. + +To support composition via addition, join, etc., it was sufficient to +overload the stubs for ``str`` in Pyre's copy of typeshed. + + +Appendix A: Other Uses +====================== + +To simplify the discussion and require minimal security knowledge, we +focused on SQL injections throughout the PEP. ``LiteralString``, +however, can also be used to prevent many other kinds of `injection +vulnerabilities `_. + +Command Injection +----------------- + +APIs such as ``subprocess.run`` accept a string which can be run as a +shell command: + +:: + + subprocess.run(f"echo 'Hello {name}'", shell=True) + +If user-controlled data is included in the command string, the code is +vulnerable to "command injection"; i.e., an attacker can run malicious +commands. For example, a value of ``' && rm -rf / #`` would result in +the following destructive command being run: + +:: + + echo 'Hello ' && rm -rf / #' + +This vulnerability could be prevented by updating ``run`` to only +accept ``LiteralString`` when used in ``shell=True`` mode. Here is one +simplified stub: + +:: + + def run(command: LiteralString, *args: str, shell: bool=...): ... + +Cross Site Scripting (XSS) +-------------------------- + +Most popular Python web frameworks, such as Django, use a templating +engine to produce HTML from user data. These templating languages +auto-escape user data before inserting it into the HTML template and +thus prevent cross site scripting (XSS) vulnerabilities. + +But a common way to `bypass auto-escaping +`_ +and render HTML as-is is to use functions like ``mark_safe`` in +`Django +`_ +or ``do_mark_safe`` in `Jinja2 +`_, +which cause XSS vulnerabilities: + +:: + + dangerous_string = django.utils.safestring.mark_safe(f"") + return(dangerous_string) + +This vulnerability could be prevented by updating ``mark_safe`` to +only accept ``LiteralString``: + +:: + + def mark_safe(s: LiteralString) -> str: ... + +Server Side Template Injection (SSTI) +------------------------------------- + +Templating frameworks, such as Jinja, allow Python expressions which +will be evaluated and substituted into the rendered result: + +:: + + template_str = "There are {{ len(values) }} values: {{ values }}" + template = jinja2.Template(template_str) + template.render(values=[1, 2]) + # Result: "There are 2 values: [1, 2]" + +If an attacker controls all or part of the template string, they can +insert expressions which execute arbitrary code and `compromise +`_ +the application: + +:: + + malicious_str = "{{''.__class__.__base__.__subclasses__()[408]('rm - rf /',shell=True)}}" + template = jinja2.Template(malicious_str) + template.render() + # Result: The shell command 'rm - rf /' is run + +Template injection exploits like this could be prevented by updating +the ``Template`` API to only accept ``LiteralString``: + +:: + + class Template: + def __init__(self, source: LiteralString): ... + + +Logging Format String Injection +------------------------------- + +Logging frameworks often allow their input strings to contain +formatting directives. At its worst, allowing users to control the +logged string has led to `CVE-2021-44228 +`_ (colloquially +known as ``log4shell``), which has been described as the `"most +critical vulnerability of the last decade" +`_. +While no Python frameworks are currently known to be vulnerable to a +similar attack, the built-in logging framework does provide formatting +options which are vulnerable to Denial of Service attacks from +externally controlled logging strings. The following example +illustrates a simple denial of service scenario: + +:: + + external_string = "%(foo)999999999s" + ... + # Tries to add > 1GB of whitespace to the logged string: + logger.info(f'Received: {external_string}', some_dict) + +This kind of attack could be prevented by requiring that the format +string passed to the logger be a ``LiteralString`` and that all +externally controlled data be passed separately as arguments (as +proposed in `Issue 46200 `_): + +:: + + def info(msg: LiteralString, *args: object) -> None: + ... + + +Appendix B: Limitations +======================= + +There are a number of ways ``LiteralString`` could still fail to +prevent users from passing strings built from non-literal data to an +API: + +1. If the developer does not use a type checker or does not add type +annotations, then violations will go uncaught. + +2. ``cast(LiteralString, non_literal_string)`` could be used to lie to +the type checker and allow a dynamic string value to masquerade as a +``LiteralString``. The same goes for a variable that has type ``Any``. + +3. Comments such as ``# type: ignore`` could be used to ignore +warnings about non-literal strings. + +4. Trivial functions could be constructed to convert a ``str`` to a +``LiteralString``: + +:: + + def make_literal(s: str) -> LiteralString: + letters: Dict[str, LiteralString] = { + "A": "A", + "B": "B", + ... + } + output: List[LiteralString] = [letters[c] for c in s] + return "".join(output) + + +We could mitigate the above using linting, code review, etc., but +ultimately a clever, malicious developer attempting to circumvent the +protections offered by ``LiteralString`` will always succeed. The +important thing to remember is that ``LiteralString`` is not intended +to protect against *malicious* developers; it is meant to protect +against benign developers accidentally using sensitive APIs in a +dangerous way (without getting in their way otherwise). + +Without ``LiteralString``, the best enforcement tool API authors have +is documentation, which is easily ignored and often not seen. With +``LiteralString``, API misuse requires conscious thought and artifacts +in the code that reviewers and future developers can notice. + +.. _PEP 675 Appendix C: + +Appendix C: ``str`` methods that preserve ``LiteralString`` +=========================================================== + +The ``str`` class has several methods that would benefit from +``LiteralString``. For example, users might expect +``"hello".capitalize()`` to have the type ``LiteralString`` similar to +the other examples we have seen in the `Inferring LiteralString +`_ section. Inferring the type +``LiteralString`` is correct because the string is not an arbitrary +user-supplied string - we know that it has the type +``Literal["HELLO"]``, which is compatible with ``LiteralString``. In +other words, the ``capitalize`` method preserves the ``LiteralString`` +type. There are several other ``str`` methods that preserve +``LiteralString``. + +We propose updating the stub for ``str`` in typeshed so that the +methods are overloaded with the ``LiteralString``-preserving +versions. This means type checkers do not have to hardcode +``LiteralString`` behavior for each method. It also lets us easily +support new methods in the future by updating the typeshed stub. + +For example, to preserve literal types for the ``capitalize`` method, +we would change the stub as below: + +:: + + # before + def capitalize(self) -> str: ... + + # after + @overload + def capitalize(self: LiteralString) -> LiteralString: ... + @overload + def capitalize(self) -> str: ... + +The downside of changing the ``str`` stub is that the stub becomes +more complicated and can make error messages harder to +understand. Type checkers may need to special-case ``str`` to make +error messages understandable for users. + +Below is an exhaustive list of ``str`` methods which, when called with +arguments of type ``LiteralString``, must be treated as returning a +``LiteralString``. If this PEP is accepted, we will update these +method signatures in typeshed: + +:: + + @overload + def capitalize(self: LiteralString) -> LiteralString: ... + @overload + def capitalize(self) -> str: ... + + @overload + def casefold(self: LiteralString) -> LiteralString: ... + @overload + def casefold(self) -> str: ... + + @overload + def center(self: LiteralString, __width: SupportsIndex, __fillchar: LiteralString = ...) -> LiteralString: ... + @overload + def center(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... + + if sys.version_info >= (3, 8): + @overload + def expandtabs(self: LiteralString, tabsize: SupportsIndex = ...) -> LiteralString: ... + @overload + def expandtabs(self, tabsize: SupportsIndex = ...) -> str: ... + + else: + @overload + def expandtabs(self: LiteralString, tabsize: int = ...) -> LiteralString: ... + @overload + def expandtabs(self, tabsize: int = ...) -> str: ... + + @overload + def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... + @overload + def format(self, *args: str, **kwargs: str) -> str: ... + + @overload + def join(self: LiteralString, __iterable: Iterable[LiteralString]) -> LiteralString: ... + @overload + def join(self, __iterable: Iterable[str]) -> str: ... + + @overload + def ljust(self: LiteralString, __width: SupportsIndex, __fillchar: LiteralString = ...) -> LiteralString: ... + @overload + def ljust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... + + @overload + def lower(self: LiteralString) -> LiteralString: ... + @overload + def lower(self) -> LiteralString: ... + + @overload + def lstrip(self: LiteralString, __chars: LiteralString | None = ...) -> LiteralString: ... + @overload + def lstrip(self, __chars: str | None = ...) -> str: ... + + @overload + def partition(self: LiteralString, __sep: LiteralString) -> tuple[LiteralString, LiteralString, LiteralString]: ... + @overload + def partition(self, __sep: str) -> tuple[str, str, str]: ... + + @overload + def replace(self: LiteralString, __old: LiteralString, __new: LiteralString, __count: SupportsIndex = ...) -> LiteralString: ... + @overload + def replace(self, __old: str, __new: str, __count: SupportsIndex = ...) -> str: ... + + if sys.version_info >= (3, 9): + @overload + def removeprefix(self: LiteralString, __prefix: LiteralString) -> LiteralString: ... + @overload + def removeprefix(self, __prefix: str) -> str: ... + + @overload + def removesuffix(self: LiteralString, __suffix: LiteralString) -> LiteralString: ... + @overload + def removesuffix(self, __suffix: str) -> str: ... + + @overload + def rjust(self: LiteralString, __width: SupportsIndex, __fillchar: LiteralString = ...) -> LiteralString: ... + @overload + def rjust(self, __width: SupportsIndex, __fillchar: str = ...) -> str: ... + + @overload + def rpartition(self: LiteralString, __sep: LiteralString) -> tuple[LiteralString, LiteralString, LiteralString]: ... + @overload + def rpartition(self, __sep: str) -> tuple[str, str, str]: ... + + @overload + def rsplit(self: LiteralString, sep: LiteralString | None = ..., maxsplit: SupportsIndex = ...) -> list[LiteralString]: ... + @overload + def rsplit(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... + + @overload + def rstrip(self: LiteralString, __chars: LiteralString | None = ...) -> LiteralString: ... + @overload + def rstrip(self, __chars: str | None = ...) -> str: ... + + @overload + def split(self: LiteralString, sep: LiteralString | None = ..., maxsplit: SupportsIndex = ...) -> list[LiteralString]: ... + @overload + def split(self, sep: str | None = ..., maxsplit: SupportsIndex = ...) -> list[str]: ... + + @overload + def splitlines(self: LiteralString, keepends: bool = ...) -> list[LiteralString]: ... + @overload + def splitlines(self, keepends: bool = ...) -> list[str]: ... + + @overload + def strip(self: LiteralString, __chars: LiteralString | None = ...) -> LiteralString: ... + @overload + def strip(self, __chars: str | None = ...) -> str: ... + + @overload + def swapcase(self: LiteralString) -> LiteralString: ... + @overload + def swapcase(self) -> str: ... + + @overload + def title(self: LiteralString) -> LiteralString: ... + @overload + def title(self) -> str: ... + + @overload + def upper(self: LiteralString) -> LiteralString: ... + @overload + def upper(self) -> str: ... + + @overload + def zfill(self: LiteralString, __width: SupportsIndex) -> LiteralString: ... + @overload + def zfill(self, __width: SupportsIndex) -> str: ... + + @overload + def __add__(self: LiteralString, __s: LiteralString) -> LiteralString: ... + @overload + def __add__(self, __s: str) -> str: ... + + @overload + def __iter__(self: LiteralString) -> Iterator[str]: ... + @overload + def __iter__(self) -> Iterator[str]: ... + + @overload + def __mod__(self: LiteralString, __x: Union[LiteralString, Tuple[LiteralString, ...]]) -> str: ... + @overload + def __mod__(self, __x: Union[str, Tuple[str, ...]]) -> str: ... + + @overload + def __mul__(self: LiteralString, __n: SupportsIndex) -> LiteralString: ... + @overload + def __mul__(self, __n: SupportsIndex) -> str: ... + + @overload + def __repr__(self: LiteralString) -> LiteralString: ... + @overload + def __repr__(self) -> str: ... + + @overload + def __rmul__(self: LiteralString, n: SupportsIndex) -> LiteralString: ... + @overload + def __rmul__(self, n: SupportsIndex) -> str: ... + + @overload + def __str__(self: LiteralString) -> LiteralString: ... + @overload + def __str__(self) -> str: ... + + +Appendix D: Guidelines for using ``LiteralString`` in Stubs +=========================================================== + +Libraries that do not contain type annotations within their source may +specify type stubs in Typeshed. Libraries written in other languages, +such as those for machine learning, may also provide Python type +stubs. This means the type checker cannot verify that the type +annotations match the source code and must trust the type stub. Thus, +authors of type stubs need to be careful when using ``LiteralString``, +since a function may falsely appear to be safe when it is not. + +We recommend the following guidelines for using ``LiteralString`` in stubs: + ++ If the stub is for a pure function, we recommend using ``LiteralString`` + in the return type of the function or of its overloads only if all + the corresponding parameters have literal types (i.e., + ``LiteralString`` or ``Literal["a", "b"]``). + + :: + + # OK + @overload + def my_transform(x: LiteralString, y: Literal["a", "b"]) -> LiteralString: ... + @overload + def my_transform(x: str, y: str) -> str: ... + + # Not OK + @overload + def my_transform(x: LiteralString, y: str) -> LiteralString: ... + @overload + def my_transform(x: str, y: str) -> str: ... + ++ If the stub is for a ``staticmethod``, we recommend the same + guideline as above. + ++ If the stub is for any other kind of method, we recommend against + using ``LiteralString`` in the return type of the method or any of + its overloads. This is because, even if all the explicit parameters + have type ``LiteralString``, the object itself may be created using + user data and thus the return type may be user-controlled. + ++ If the stub is for a class attribute or global variable, we also + recommend against using ``LiteralString`` because the untyped code + may write arbitrary values to the attribute. + +However, we leave the final call to the library author. They may use +``LiteralString`` if they feel confident that the string returned by +the method or function or the string stored in the attribute is +guaranteed to have a literal type - i.e., the string is created by +applying only literal-preserving ``str`` operations to a string +literal. + +Note that these guidelines do not apply to inline type annotations +since the type checker can verify that, say, a method returning +``LiteralString`` does in fact return an expression of that type. + + +Resources +========= + +Literal String Types in Scala +----------------------------- + +Scala `uses +`_ +``Singleton`` as the supertype for singleton types, which includes +literal string types, such as ``"foo"``. ``Singleton`` is Scala's +generalized analogue of this PEP's ``LiteralString``. + +Tamer Abdulradi showed how Scala's literal string types can be used +for "Preventing SQL injection at compile time", Scala Days talk +`Literal types: What are they good for? +`_ +(slides 52 to 68). + +Thanks +------ + +Thanks to the following people for their feedback on the PEP: + +Edward Qiu, Jia Chen, Shannon Zhu, Gregory P. Smith, НоĐșота ĐĄĐŸĐ±ĐŸĐ»Đ”ĐČ, +CAM Gerlach, Arie Bovenberg, David Foster, and Shengye Wan + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0676.rst b/peps/pep-0676.rst new file mode 100644 index 000000000..5d532948e --- /dev/null +++ b/peps/pep-0676.rst @@ -0,0 +1,268 @@ +PEP: 676 +Title: PEP Infrastructure Process +Author: Adam Turner +Sponsor: Mariatta +PEP-Delegate: Barry Warsaw +Discussions-To: https://discuss.python.org/t/10774 +Status: Active +Type: Process +Content-Type: text/x-rst +Created: 01-Nov-2021 +Post-History: 23-Sep-2021, 30-Nov-2021 +Resolution: https://discuss.python.org/t/10774/99 + + +Abstract +======== + +This PEP addresses the infrastructure around rendering PEP files from +`reStructuredText`_ files to HTML webpages. We aim to specify a self-contained +and maintainable solution for PEP readers, authors, and editors. + + +Motivation +========== + +As of November 2021, Python Enhancement Proposals (PEPs) are rendered in a +multi-system, multi-stage process. A continuous integration (CI) task runs a +`docutils`_ script to render all PEP files individually. The CI task then +uploads a tar archive to a server, where it is retrieved and rendered into the +`python.org`_ website periodically. + +This places a constraint on the `python.org`_ website to handle raw HTML +uploads and handle PEP rendering, and makes the appropriate place to raise +issues unclear in some cases [1]_. + +This PEP provides a specification for self-contained rendering of PEPs. This +would: + +* reduce the amount of distributed configuration for supporting PEPs +* enable quality-of-life improvements for those who read, write, and review + PEPs +* solve a number of outstanding issues, and lay the path for improvements +* save volunteer maintainers' time + +We propose that PEPs are accessed through `peps.python.org`_ at the top-level +(for example `peps.python.org/pep-0008`_), and that all custom tooling to +support rendering PEPs is hosted in the `python/peps`_ repository. + + +Rationale +========= + +Simplifying and Centralising Infrastructure +------------------------------------------- + +As of November 2021, to locally render a PEP file, a PEP author or editor needs +to create a full local instance of the `python.org`_ website and run a number +of disparate scripts, following `documentation`_ that lives outside of the +`python/peps`_ repository. + +By contrast, the proposed implementation provides a single `Makefile`_ and a +Python script to render all PEP files, with options to target a web-server or +the local filesystem. + +Using a single repository to host all tooling will clarify where to raise +issues, reducing volunteer time spent in triage. + +Simplified and centralised tooling may also reduce the barrier to entry to +further improvements, as the scope of the PEP rendering infrastructure is well +defined. + + +Quality-of-Life Improvements and Resolving Issues +------------------------------------------------- + +There are several requests for additional features in reading PEPs, such as: + +* syntax highlighting [2]_ +* use of ``.. code-block::`` directives [2]_ +* support for SVG images [3]_ +* typographic quotation marks [4]_ +* additional footer information [5]_ +* intersphinx functionality [6]_ +* dark mode theme [7]_ + +These are "easy wins" from this proposal, and would serve to improve the +quality-of-life for consumers of PEPs (including reviewers and writers). + +For example, the current (as of November 2021) system runs periodically on a +schedule. This means that updates to PEPs cannot be circulated immediately, +reducing productivity. The reference implementation renders and publishes all +PEPs on every commit to the repository, solving the issue by design. + +The reference implementation fixes several issues [8]_. For example: + +* list styles are currently not respected by `python.org`_'s stylesheets +* support for updating images in PEPs is challenging in `python.org`_ + +Third-party providers such as `Read the Docs`_ or `Netlify`_ can enhance this +experience with features such as automatic rendering of pull requests. + + +Specification +============= + +The proposed specification for rendering the PEP files to HTML is as per the +`reference implementation`_. + +The rendered PEPs MUST be available at `peps.python.org`_. These SHOULD be +hosted as static files, and MAY be behind a content delivery network (CDN). + +A service to render previews of pull requests SHOULD be provided. This service +MAY be integrated with the hosting and deployment solution. + +The following redirect rules MUST be created for the `python.org`_ domain: + +* ``/peps/`` -> https://peps.python.org/ +* ``/dev/peps/`` -> https://peps.python.org/ +* ``/peps/(.*)\.html`` -> https://peps.python.org/$1 +* ``/dev/peps/(.*)`` -> https://peps.python.org/$1 + +The following nginx configuration would achieve this: + +.. code-block:: nginx + + location ~ ^/dev/peps/?(.*)$ { + return 308 https://peps.python.org/$1/; + } + + location ~ ^/peps/(.*)\.html$ { + return 308 https://peps.python.org/$1/; + } + + location ^/(dev/)?peps(/.*)?$ { + return 308 https://peps.python.org/; + } + +Redirects MUST be implemented to preserve `URL fragments`_ for backward +compatibility purposes. + +Backwards Compatibility +======================= + +Due to server-side redirects to the new canonical URLs, links in previously +published materials referring to the old URL schemes will be guaranteed to work. +All PEPs will continue to render correctly, and a custom stylesheet in the +reference implementation improves presentation for some elements (most notably +code blocks and block quotes). Therefore, this PEP presents no backwards +compatibility issues. + + +Security Implications +===================== + +The main `python.org`_ website will no longer process raw HTML uploads, +closing a potential threat vector. PEP rendering and deployment processes will +use modern, well-maintained code and secure automated platforms, further +reducing the potential attack surface. Therefore, we see no negative security +impact. + + +How to Teach This +================= + +The new canonical URLs will be publicised in the documentation. However, this +is mainly a backend infrastructure change, and there should be minimal +end-user impact. PEP 1 and PEP 12 will be updated as needed. + + +Reference Implementation +======================== + +The proposed implementation has been merged into the `python/peps`_ repository +in a series of pull requests [9]_. It uses the `Sphinx`_ documentation system +with a custom theme (supporting light and dark colour schemes) and extensions. + +This already automatically renders all PEPs on every commit, and publishes them +to `python.github.io/peps`_. The high level documentation for the system covers +`how to render PEPs locally `__ and +`the implementation of the system `__. + + +Rejected Ideas +============== + +It would likely be possible to amend the current (as of November 2021) +rendering process to include a subset of the quality-of-life improvements and +issue mitigations mentioned above. However, we do not believe that this would +solve the distributed tooling issue. + +It would be possible to use the output from the proposed rendering system and +import it into `python.org`_. We would argue that this would be the worst of +both worlds, as a great deal of complexity is added whilst none is removed. + + +Acknowledgements +================ + +- Hugo van Kemenade +- Pablo Galindo Salgado +- Éric Araujo +- Mariatta +- C.A.M. Gerlach + + +Footnotes +========= + +.. _documentation: https://pythondotorg.readthedocs.io/pep_generation.html +.. _docutils: https://docutils.sourceforge.io +.. _Makefile: https://www.gnu.org/software/make/manual/make.html#Introduction +.. _Netlify: https://www.netlify.com/ +.. _peps.python.org: https://peps.python.org/ +.. _peps.python.org/pep-0008: https://peps.python.org/pep-0008/ +.. _python.github.io/peps: https://python.github.io/peps +.. _python.org: https://www.python.org +.. _python/peps: https://github.com/python/peps +.. _Read the Docs: https://readthedocs.org +.. _reStructuredText: https://docutils.sourceforge.io/rst.html +.. _Sphinx: https://www.sphinx-doc.org/en/master/ +.. _URL fragments: https://url.spec.whatwg.org/#concept-url-fragment + +.. [1] For example, + `pythondotorg#1024 `__, + `pythondotorg#1038 `__, + `pythondotorg#1387 `__, + `pythondotorg#1388 `__, + `pythondotorg#1393 `__, + `pythondotorg#1564 `__, + `pythondotorg#1913 `__, +.. [2] Requested: `pythondotorg#1063 `__, + `pythondotorg#1206 `__, + `pythondotorg#1638 `__, + `peps#159 `__, + `comment in peps#1571 `__, + `peps#1577 `__, +.. [3] Requested: `peps#701 `__ +.. [4] Requested: `peps#165 `__ +.. [5] Requested: `pythondotorg#1564 `__ +.. [6] Requested: `comment in peps#2 `__ +.. [7] Requested: `in python-dev `__ +.. [8] As of November 2021, see + `peps#1387 `__, + `pythondotorg#824 `__, + `pythondotorg#1556 `__, +.. [9] Implementation PRs: + `peps#1930 `__, + `peps#1931 `__, + `peps#1932 `__, + `peps#1933 `__, + `peps#1934 `__ + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/peps/pep-0677.rst b/peps/pep-0677.rst new file mode 100644 index 000000000..a55eb8049 --- /dev/null +++ b/peps/pep-0677.rst @@ -0,0 +1,1254 @@ +PEP: 677 +Title: Callable Type Syntax +Author: Steven Troxler , + Pradeep Kumar Srinivasan +Sponsor: Guido van Rossum +Discussions-To: python-dev@python.org +Status: Rejected +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 13-Dec-2021 +Python-Version: 3.11 +Post-History: 16-Dec-2021 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/NHCLHCU2XCWTBGF732WESMN42YYVKOXB/ + +Abstract +======== + +This PEP introduces a concise and friendly syntax for callable types, +supporting the same functionality as ``typing.Callable`` but with an +arrow syntax inspired by the syntax for typed function +signatures. This allows types like ``Callable[[int, str], bool]`` to +be written as ``(int, str) -> bool``. + +The proposed syntax supports all the functionality provided by +``typing.Callable`` and ``typing.Concatenate``, and is intended to +work as a drop-in replacement. + + +Motivation +========== + +One way to make code safer and easier to analyze is by making sure +that functions and classes are well-typed. In Python we have type +annotations, the framework for which is defined in :pep:`484`, to provide +type hints that can find bugs as well as helping with editor tooling +like tab completion, static analysis tooling, and code review. + +Consider the following untyped code:: + + def flat_map(func, l): + out = [] + for element in l: + out.extend(func(element)) + return out + + + def wrap(x: int) -> list[int]: + return [x] + + def add(x: int, y: int) -> int: + return x + y + + flat_map(wrap, [1, 2, 3]) # no runtime error, output is [1, 2, 3] + flat_map(add, [1, 2, 3]) # runtime error: `add` expects 2 arguments, got 1 + + +We can add types to this example to detect the runtime error:: + + from typing import Callable + + def flat_map( + func: Callable[[int], list[int]], + l: list[int] + ) -> list[int]: + .... + + ... + + + flat_map(wrap, [1, 2, 3]) # type checks okay, output is [1, 2, 3] + flat_map(add, [1, 2, 3]) # type check error + +There are a few usability challenges with ``Callable`` we can see here: + +- It is verbose, particularly for more complex function signatures. +- It relies on two levels of nested brackets, unlike any other generic + type. This can be especially hard to read when some of the type + parameters are themselves generic types. +- The bracket structure is not visually similar to how function signatures + are written. +- It requires an explicit import, unlike many of the other most common + types like ``list`` and ``dict``. + +Possibly as a result, `programmers often fail to write complete +Callable types +`_. +Such untyped or partially-typed callable types do not check the +parameter types or return types of the given callable and thus negate +the benefits of static typing. For example, they might write this:: + + + from typing import Callable + + def flat_map( + func: Callable[..., Any], + l: list[int] + ) -> list[int]: + .... + + ... + + + flat_map(add, [1, 2, 3]) # oops, no type check error! + +There's some partial type information here - we at least know that ``func`` +needs to be callable. But we've dropped too much type information for +type checkers to find the bug. + +With our proposal, the example looks like this:: + + def flat_map( + func: (int) -> list[int], + l: list[int] + ) -> list[int]: + out = [] + for element in l: + out.extend(f(element)) + return out + + ... + +The type ``(int) -> list[int]`` is more concise, uses an arrow similar +to the one indicating a return type in a function header, avoids +nested brackets, and does not require an import. + + +Rationale +========= + +The ``Callable`` type is widely used. For example, `as of October 2021 +it was +`_ +the fifth most common complex type in typeshed, after ``Optional``, +``Tuple``, ``Union``, and ``List``. + +The others have had their syntax improved and the need for imports +eliminated by either :pep:`604` or :pep:`585`: + +- ``typing.Optional[int]`` is written ``int | None`` +- ``typing.Union[int, str]`` is written ``int | str`` +- ``typing.List[int]`` is written ``list[int]`` +- ``typing.Tuple[int, str]`` is written ``tuple[int, str]`` + +The ``typing.Callable`` type is used almost as often as these other +types, is more complicated to read and write, and still requires an +import and bracket-based syntax. + +In this proposal, we chose to support all the existing semantics of +``typing.Callable``, without adding support for new features. We made +this decision after examining how frequently each feature might be +used in existing typed and untyped open-source code. We determined +that the vast majority of use cases are covered. + +We considered adding support for named, optional, and variadic +arguments. However, we decided against including these features, as +our analysis showed they are infrequently used. When they are really +needed, it is possible to type these using `callback protocols +`_. + +An Arrow Syntax for Callable Types +---------------------------------- + +We are proposing a succinct, easy-to-use syntax for +``typing.Callable`` that looks similar to function headers in Python. +Our proposal closely follows syntax used by several popular languages +such as `Typescript +`_, +`Kotlin `_, and `Scala +`_. + +Our goals are that: + +- Callable types using this syntax will be easier to learn and use, + particularly for developers with experience in other languages. +- Library authors will be more likely to use expressive types for + callables that enable type checkers to better understand code and + find bugs, as in the ``decorator`` example above. + +Consider this simplified real-world example from a web server, written +using the existing ``typing.Callable``:: + + from typing import Awaitable, Callable + from app_logic import Response, UserSetting + + + def customize_response( + response: Response, + customizer: Callable[[Response, list[UserSetting]], Awaitable[Response]] + ) -> Response: + ... + +With our proposal, this code can be abbreviated to:: + + from app_logic import Response, UserSetting + + def customize_response( + response: Response, + customizer: async (Response, list[UserSetting]) -> Response, + ) -> Response: + ... + +This is shorter and requires fewer imports. It also has far less +nesting of square brackets - only one level, as opposed to three in +the original code. + +Compact Syntax for ``ParamSpec`` +-------------------------------- + +A particularly common case where library authors leave off type information +for callables is when defining decorators. Consider the following:: + + + from typing import Any, Callable + + def with_retries( + f: Callable[..., Any] + ) -> Callable[..., Any]: + def wrapper(retry_once, *args, **kwargs): + if retry_once: + try: return f(*args, **kwargs) + except Exception: pass + return f(*args, **kwargs) + return wrapper + + @with_retries + def f(x: int) -> int: + return x + + + f(y=10) # oops - no type error! + +In the code above, it is clear that the decorator should produce a +function whose signature is like that of the argument ``f`` other +than an additional ``retry_once`` argument. But the use of ``...`` +prevents a type checker from seeing this and alerting a user that +``f(y=10)`` is invalid. + + +With :pep:`612` it is possible to type decorators like this correctly +as follows:: + + from typing import Any, Callable, Concatenate, ParamSpec, TypeVar + + R = TypeVar("R") + P = ParamSpec("P") + + def with_retries( + f: Callable[P, R] + ) -> Callable[Concatenate[bool, P] R]: + def wrapper(retry_once: bool, *args: P.args, **kwargs: P.kwargs) -> R: + ... + return wrapper + + ... + + +With our proposed syntax, the properly-typed decorator example becomes +concise and the type representations are visually descriptive:: + + from typing import Any, ParamSpec, TypeVar + + R = TypeVar("R") + P = ParamSpec("P") + + def with_retries( + f: (**P) -> R + ) -> (bool, **P) -> R: + ... + +Comparing to Other Languages +---------------------------- + +Many popular programming languages use an arrow syntax similar +to the one we are proposing here. + +TypeScript +~~~~~~~~~~ + +In `TypeScript +`_, +function types are expressed in a syntax almost the same as the one we +are proposing, but the arrow token is ``=>`` and arguments have names:: + + (x: int, y: str) => bool + +The names of the arguments are not actually relevant to the type. So, +for example, this is the same callable type:: + + (a: int, b: str) => bool + +Kotlin +~~~~~~ + +Function types in `Kotlin `_ permit +an identical syntax to the one we are proposing, for example:: + + (Int, String) -> Bool + +It also optionally allows adding names to the arguments, for example:: + + (x: Int, y: String) -> Bool + +As in TypeScript, the argument names (if provided) are just there for +documentation and are not part of the type itself. + +Scala +~~~~~ + +`Scala `_ +uses the ``=>`` arrow for function types. Other than that, their syntax is +the same as the one we are proposing, for example:: + + (Int, String) => Bool + +Scala, like Python, has the ability to provide function arguments by name. +Function types can optionally include names, for example:: + + (x: Int, y: String) => Bool + +Unlike in TypeScript and Kotlin, these names are part of the type if +provided - any function implementing the type must use the same names. +This is similar to the extended syntax proposal we describe in our +`Rejected Alternatives`_ section. + +Function Definitions vs Callable Type Annotations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In all of the languages listed above, type annotations for function +definitions use a ``:`` rather than a ``->``. For example, in TypeScript +a simple add function looks like this:: + + function higher_order(fn: (a: string) => string): string { + return fn("Hello, World"); + } + +Scala and Kotlin use essentially the same ``:`` syntax for return +annotations. The ``:`` makes sense in these languages because they +all use ``:`` for type annotations of +parameters and variables, and the use for function return types is +similar. + +In Python we use ``:`` to denote the start of a function body and +``->`` for return annotations. As a result, even though our proposal +is superficially the same as these other languages the context is +different. There is potential for more confusion in Python when +reading function definitions that include callable types. + +This is a key concern for which we are seeking feedback with our draft +PEP; one idea we have floated is to use ``=>`` instead to make it easier +to differentiate. + + +The ML Language Family +~~~~~~~~~~~~~~~~~~~~~~ + +Languages in the ML family, including `F# +`_, +`OCaml +`_, +and `Haskell `_, all use +``->`` to represent function types. All of them use a parentheses-free +syntax with multiple arrows, for example in Haskell:: + + Integer -> String -> Bool + +The use of multiple arrows, which differs from our proposal, makes +sense for languages in this family because they use automatic +`currying `_ of function arguments, +which means that a multi-argument function behaves like a single-argument +function returning a function. + +Specification +============= + +Typing Behavior +--------------- + +Type checkers should treat the new syntax with exactly the same +semantics as ``typing.Callable``. + +As such, a type checker should treat the following pairs exactly the +same:: + + from typing import Awaitable, Callable, Concatenate, ParamSpec, TypeVarTuple + + P = ParamSpec("P") + Ts = TypeVarTuple('Ts') + + f0: () -> bool + f0: Callable[[], bool] + + f1: (int, str) -> bool + f1: Callable[[int, str], bool] + + f2: (...) -> bool + f2: Callable[..., bool] + + f3: async (str) -> str + f3: Callable[[str], Awaitable[str]] + + f4: (**P) -> bool + f4: Callable[P, bool] + + f5: (int, **P) -> bool + f5: Callable[Concatenate[int, P], bool] + + f6: (*Ts) -> bool + f6: Callable[[*Ts], bool] + + f7: (int, *Ts, str) -> bool + f7: Callable[[int, *Ts, str], bool] + + +Grammar and AST +--------------- + +The proposed new syntax can be described by these AST changes to `Parser/Python.asdl +`_:: + + expr = + | AsyncCallableType(callable_type_arguments args, expr returns) + | CallableType(callable_type_arguments args, expr returns) + + callable_type_arguments = AnyArguments + | ArgumentsList(expr* posonlyargs) + | Concatenation(expr* posonlyargs, expr param_spec) + + +Here are our proposed changes to the `Python Grammar +`:: + + expression: + | disjunction disjunction 'else' expression + | callable_type_expression + | disjunction + | lambdef + + callable_type_expression: + | callable_type_arguments '->' expression + | ASYNC callable_type_arguments '->' expression + + callable_type_arguments: + | '(' '...' [','] ')' + | '(' callable_type_positional_argument* ')' + | '(' callable_type_positional_argument* callable_type_param_spec ')' + + callable_type_positional_argument: + | !'...' expression ',' + | !'...' expression &')' + + callable_type_param_spec: + | '**' expression ',' + | '**' expression &')' + + + +If :pep:`646` is accepted, we intend to include support for unpacked +types in two ways. To support the "star-for-unpack" syntax proposed in +:pep:`646`, we will modify the grammar for +``callable_type_positional_argument`` as follows:: + + callable_type_positional_argument: + | !'...' expression ',' + | !'...' expression &')' + | '*' expression ',' + | '*' expression &')' + +With this change, a type of the form ``(int, *Ts) -> bool`` should +evaluate the AST form:: + + CallableType( + ArgumentsList(Name("int"), Starred(Name("Ts")), + Name("bool") + ) + +and be treated by type checkers as equivalent to or ``Callable[[int, +*Ts], bool]`` or ``Callable[[int, Unpack[Ts]], bool]``. + + +Implications of the Grammar +--------------------------- + +Precedence of -> +~~~~~~~~~~~~~~~~ + + +``->`` binds less tightly than other operators, both inside types and +in function signatures, so the following two callable types are +equivalent:: + + (int) -> str | bool + (int) -> (str | bool) + + +``->`` associates to the right, both inside types and in function +signatures. So the following pairs are equivalent:: + + (int) -> (str) -> bool + (int) -> ((str) -> bool) + + def f() -> (int, str) -> bool: pass + def f() -> ((int, str) -> bool): pass + + def f() -> (int) -> (str) -> bool: pass + def f() -> ((int) -> ((str) -> bool)): pass + + +Because operators bind more tightly than ``->``, parentheses are +required whenever an arrow type is intended to be inside an argument +to an operator like ``|``:: + + (int) -> () -> int | () -> bool # syntax error! + (int) -> (() -> int) | (() -> bool) # okay + + +We discussed each of these behaviors and believe they are desirable: + +- Union types (represented by ``A | B`` according to :pep:`604`) are + valid in function signature returns, so we need to allow operators + in the return position for consistency. +- Given that operators bind more tightly than ``->`` it is correct + that a type like ``bool | () -> bool`` must be a syntax error. We + should be sure the error message is clear because this may be a + common mistake. +- Associating ``->`` to the right, rather than requiring explicit + parentheses, is consistent with other languages like TypeScript and + respects the principle that valid expressions should normally be + substitutable when possible. + +``async`` Keyword +~~~~~~~~~~~~~~~~~ + +All of the binding rules still work for async callable types:: + + (int) -> async (float) -> str | bool + (int) -> (async (float) -> (str | bool)) + + def f() -> async (int, str) -> bool: pass + def f() -> (async (int, str) -> bool): pass + + def f() -> async (int) -> async (str) -> bool: pass + def f() -> (async (int) -> (async (str) -> bool)): pass + + +Trailing Commas +~~~~~~~~~~~~~~~ + +- Following the precedent of function signatures, putting a comma in + an empty arguments list is illegal: ``(,) -> bool`` is a syntax + error. +- Again following precedent, trailing commas are otherwise always + permitted:: + + + ((int,) -> bool == (int) -> bool + ((int, **P,) -> bool == (int, **P) -> bool + ((...,) -> bool) == ((...) -> bool) + +Allowing trailing commas also gives autoformatters more flexibility +when splitting callable types across lines, which is always legal +following standard python whitespace rules. + + +Disallowing ``...`` as an Argument Type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Under normal circumstances, any valid expression is permitted where we +want a type annotation and ``...`` is a valid expression. This is +never semantically valid and all type checkers would reject it, but +the grammar would allow it if we did not explicitly prevent this. + +Since ``...`` is meaningless as a type and there are usability +concerns, our grammar rules it out and the following is a syntax +error:: + + (int, ...) -> bool + +We decided that there were compelling reasons to do this: + +- The semantics of ``(...) -> bool`` are different from ``(T) -> bool`` + for any valid type T: ``(...)`` is a special form indicating + ``AnyArguments`` whereas ``T`` is a type parameter in the arguments + list. +- ``...`` is used as a placeholder default value to indicate an + optional argument in stubs and callback protocols. Allowing it in + the position of a type could easily lead to confusion and possibly + bugs due to typos. +- In the ``tuple`` generic type, we special-case ``...`` to mean + "more of the same", e.g. a ``tuple[int, ...]`` means a tuple with + one or more integers. We do not use ``...`` in a a similar way + in callable types, so to prevent misunderstandings it makes sense + to prevent this. + + + +Incompatibility with other possible uses of ``*`` and ``**`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The use of ``**P`` for supporting :pep:`612` ``ParamSpec`` rules out any +future proposal using a bare ``**`` to type +``kwargs``. This seems acceptable because: + +- If we ever do want such a syntax, it would be clearer to require an + argument name anyway. This would also make the type look more + similar to a function signature. In other words, if we ever support + typing ``kwargs`` in callable types, we would prefer ``(int, + **kwargs: str)`` rather than ``(int, **str)``. +- :pep:`646` unpacking syntax would rule out using ``*`` for + ``args``. The ``kwargs`` case is similar enough that this rules out + a bare ``**`` anyway. + + + +Compatibility with Arrow-Based Lambda Syntax +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To the best of our knowledge there is no active discussion of +arrow-style lambda syntax that we are aware of, but it is nonetheless +worth considering what possibilities would be ruled out by adopting +this proposal. + +It would be incompatible with this proposal to adopt the same a +parenthesized ``->``-based arrow syntax for lambdas, e.g. ``(x, y) -> +x + y`` for ``lambda x, y: x + y``. + + +Our view is that if we want arrow syntax for lambdas in the future, it +would be a better choice to use ``=>``, e.g. ``(x, y) => x + y``. +Many languages use the same arrow token for both lambdas and callable +types, but Python is unique in that types are expressions and have to +evaluate to runtime values. Our view is that this merits using +separate tokens, and given the existing use of ``->`` for return types +in function signatures it would be more coherent to use ``->`` for +callable types and ``=>`` for lambdas. + +Runtime Behavior +---------------- + +The new AST nodes need to evaluate to runtime types, and we have two goals for the +behavior of these runtime types: + +- They should expose a structured API that is descriptive and powerful + enough to be compatible with extending the type to include new features + like named and variadic arguments. +- They should also expose an API that is backward-compatible with + ``typing.Callable``. + +Evaluation and Structured API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We intend to create new builtin types to which the new AST nodes will +evaluate, exposing them in the ``types`` module. + +Our plan is to expose a structured API as if they were defined as follows:: + + class CallableType: + is_async: bool + arguments: Ellipsis | tuple[CallableTypeArgument] + return_type: object + + class CallableTypeArgument: + kind: CallableTypeArgumentKind + annotation: object + + @enum.global_enum + class CallableTypeArgumentKind(enum.IntEnum): + POSITIONAL_ONLY: int = ... + PARAM_SPEC: int = ... + + +The evaluation rules are expressed in terms of the following +pseudocode:: + + def evaluate_callable_type( + callable_type: ast.CallableType | ast.AsyncCallableType: + ) -> CallableType: + return CallableType( + is_async=isinstance(callable_type, ast.AsyncCallableType), + arguments=_evaluate_arguments(callable_type.arguments), + return_type=evaluate_expression(callable_type.returns), + ) + + def _evaluate_arguments(arguments): + match arguments: + case ast.AnyArguments(): + return Ellipsis + case ast.ArgumentsList(posonlyargs): + return tuple( + _evaluate_arg(arg) for arg in args + ) + case ast.ArgumentsListConcatenation(posonlyargs, param_spec): + return tuple( + *(evaluate_arg(arg) for arg in args), + _evaluate_arg(arg=param_spec, kind=PARAM_SPEC) + ) + if isinstance(arguments, Any + return Ellipsis + + def _evaluate_arg(arg, kind=POSITIONAL_ONLY): + return CallableTypeArgument( + kind=POSITIONAL_ONLY, + annotation=evaluate_expression(value) + ) + + +Backward-Compatible API +~~~~~~~~~~~~~~~~~~~~~~~ + +To get backward compatibility with the existing ``types.Callable`` API, +which relies on fields ``__args__`` and ``__parameters__``, we can define +them as if they were written in terms of the following:: + + import itertools + import typing + + def get_args(t: CallableType) -> tuple[object]: + return_type_arg = ( + typing.Awaitable[t.return_type] + if t.is_async + else t.return_type + ) + arguments = t.arguments + if isinstance(arguments, Ellipsis): + argument_args = (Ellipsis,) + else: + argument_args = (arg.annotation for arg in arguments) + return ( + *arguments_args, + return_type_arg + ) + + def get_parameters(t: CallableType) -> tuple[object]: + out = [] + for arg in get_args(t): + if isinstance(arg, typing.ParamSpec): + out.append(t) + else: + out.extend(arg.__parameters__) + return tuple(out) + + +Additional Behaviors of ``types.CallableType`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As with the ``A | B`` syntax for unions introduced in :pep:`604`: + +- The ``__eq__`` method should treat equivalent ``typing.Callable`` + values as equal to values constructed using the builtin syntax, and + otherwise should behave like the ``__eq__`` of ``typing.Callable``. +- The ``__repr__`` method should produce an arrow syntax representation that, + when evaluated, gives us back an equal ``types.CallableType`` instance. + + +Rejected Alternatives +===================== + +Many of the alternatives we considered would have been more expressive +than ``typing.Callable``, for example adding support for describing +signatures that include named, optional, and variadic arguments. + +To determine which features we most needed to support with a callable +type syntax, we did an extensive analysis of existing projects: + +- `stats on the use of the Callable type `_; +- `stats on how untyped and partially-typed callbacks are actually used `_. + +We decided on a simple proposal with improved syntax for the existing +``Callable`` type because the vast majority of callbacks can be correctly +described by the existing ``typing.Callable`` semantics: + +- Positional parameters: By far the most important case to handle well + is simple callable types with positional parameters, such as + ``(int, str) -> bool`` +- ParamSpec and Concatenate: The next most important feature is good + support for :pep:`612` ``ParamSpec`` and ``Concatenate`` types like + ``(**P) -> bool`` and ``(int, **P) -> bool``. These are common + primarily because of the heavy use of decorator patterns in python + code. +- TypeVarTuples: The next most important feature, assuming :pep:`646` is + accepted, is for unpacked types which are common because of cases + where a wrapper passes along ``*args`` to some other function. + +Features that other, more complicated proposals would support account +for fewer than 2% of the use cases we found. These are already +expressible using callback protocols, and since they are uncommon we +decided that it made more sense to move forward with a simpler syntax. + +Extended Syntax Supporting Named and Optional Arguments +------------------------------------------------------- + +Another alternative was for a compatible but more complex syntax that +could express everything in this PEP but also named, optional, and +variadic arguments. In this “extended” syntax proposal the following +types would have been equivalent:: + + class Function(typing.Protocol): + def f(self, x: int, /, y: float, *, z: bool = ..., **kwargs: str) -> bool: + ... + + Function = (int, y: float, *, z: bool = ..., **kwargs: str) -> bool + +Advantages of this syntax include: - Most of the advantages of the +proposal in this PEP (conciseness, :pep:`612` support, etc) - +Furthermore, the ability to handle named, optional, and variadic +arguments + +We decided against proposing it for the following reasons: + +- The implementation would have been more difficult, and usage stats + demonstrate that fewer than 3% of use cases would benefit from any + of the added features. +- The group that debated these proposals was split down the middle + about whether these changes are desirable: + + - On the one hand, they make callable types more expressive. On the + other hand, they could easily confuse users who have not read the + full specification of callable type syntax. + - We believe the simpler syntax proposed in this PEP, which + introduces no new semantics and closely mimics syntax in other + popular languages like Kotlin, Scala, and TypesScript, is much + less likely to confuse users. + +- We intend to implement the current proposal in a way that is + forward-compatible with the more complicated extended syntax. If the + community decides after more experience and discussion that we want + the additional features, it should be straightforward to propose + them in the future. +- Even a full extended syntax cannot replace the use of callback + protocols for overloads. For example, no closed form of callable type + could express a function that maps bools to bools and ints to floats, + like this callback protocol.:: + + from typing import overload, Protocol + + class OverloadedCallback(Protocol) + + @overload + def __call__(self, x: int) -> float: ... + + @overload + def __call__(self, x: bool) -> bool: ... + + def __call__(self, x: int | bool) -> float | bool: ... + + + f: OverloadedCallback = ... + f(True) # bool + f(3) # float + + + +We confirmed that the current proposal is forward-compatible with +extended syntax by +`implementing `_ +a grammar and AST for this extended syntax on top of our reference +implementation of this PEP's grammar. + + +Syntax Closer to Function Signatures +------------------------------------ + +One alternative we had floated was a syntax much more similar to +function signatures. + +In this proposal, the following types would have been equivalent:: + + class Function(typing.Protocol): + def f(self, x: int, /, y: float, *, z: bool = ..., **kwargs: str) -> bool: + ... + + Function = (x: int, /, y: float, *, z: bool = ..., **kwargs: str) -> bool + + +The benefits of this proposal would have included: + +- Perfect syntactic consistency between signatures and callable types. +- Support for more features of function signatures (named, optional, + variadic args) that this PEP does not support. + +Key downsides that led us to reject the idea include the following: + +- A large majority of use cases only use positional-only arguments. This + syntax would be more verbose for that use case, both because of requiring + argument names and an explicit ``/``, for example ``(int, /) -> bool`` where + our proposal allows ``(int) -> bool`` +- The requirement for explicit ``/`` for positional-only arguments has + a high risk of causing frequent bugs - which often would not be + detected by unit tests - where library authors would accidentally + use types with named arguments. +- Our analysis suggests that support for ``ParamSpec`` is key, but the + scoping rules laid out in :pep:`612` would have made this difficult. + + +Other Proposals Considered +-------------------------- + +Functions-as-Types +~~~~~~~~~~~~~~~~~~ + +An idea we looked at very early on was to `allow using functions as types +`_. +The idea is allowing a function to stand in for its own call +signature, with roughly the same semantics as the ``__call__`` method +of callback protocols:: + + def CallableType( + positional_only: int, + /, + named: str, + *args: float, + keyword_only: int = ..., + **kwargs: str + ) -> bool: ... + + f: CallableType = ... + f(5, 6.6, 6.7, named=6, x="hello", y="world") # typechecks as bool + +This may be a good idea, but we do not consider it a viable +replacement for callable types: + +- It would be difficult to handle ``ParamSpec``, which we consider a + critical feature to support. +- When using functions as types, the callable types are not first-class + values. Instead, they require a separate, out-of-line function + definition to define a type alias +- It would not support more features than callback protocols, and seems + more like a shorter way to write them than a replacement for + ``Callable``. + +Hybrid keyword-arrow Syntax +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the Rust language, a keyword ``fn`` is used to indicate functions +in much the same way as Python's ``def``, and callable types are +indicated using a hybrid arrow syntax ``Fn(i64, String) -> bool``. + +We could use the ``def`` keyword in callable types for Python, for +example our two-parameter boolean function could be written as +``def(int, str) -> bool``. But we think this might confuse readers +into thinking ``def(A, B) -> C`` is a lambda, particularly because +Javascript's ``function`` keyword is used in both named and anonymous +functions. + +Parenthesis-Free Syntax +~~~~~~~~~~~~~~~~~~~~~~~ + +We considered a parentheses-free syntax that would have been even more +concise:: + + int, str -> bool + +We decided against it because this is not visually as similar to +existing function header syntax. Moreover, it is visually similar to +lambdas, which bind names with no parentheses: ``lambda x, y: x == +y``. + +Requiring Outer Parentheses +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A concern with the current proposal is readability, particularly +when callable types are used in return type position which leads to +multiple top-level ``->`` tokens, for example:: + + def make_adder() -> (int) -> int: + return lambda x: x + 1 + +We considered a few ideas to prevent this by changing rules about +parentheses. One was to move the parentheses to the outside, so +that a two-argument boolean function is written ``(int, str -> bool)``. +With this change, the example above becomes:: + + def make_adder() -> (int -> int): + return lambda x: x + 1 + +This makes the nesting of many examples that are difficult to +follow clear, but we rejected it because + +- Currently in Python commas bind very loosely, which means it might be common + to misread ``(int, str -> bool)`` as a tuple whose first element is an int, + rather than a two-parameter callable type. +- It is not very similar to function header syntax, and one of our goals was + familiar syntax inspired by function headers. +- This syntax may be more readable for deaply nested callables like the one + above, but deep nesting is not very common. Encouraging extra parentheses + around callable types in return position via a style guide would have most of + the readability benefit without the downsides. + +We also considered requiring parentheses on both the parameter list and the +outside, e.g. ``((int, str) -> bool)``. With this change, the example above +becomes:: + + def make_adder() -> ((int) -> int): + return lambda x: x + 1 + +We rejected this change because: + +- The outer parentheses only help readability in some cases, mostly when a + callable type is used in return position. In many other cases they hurt + readability rather than helping. +- We agree that it might make sense to encourage outer parentheses in several + cases, particularly callable types in function return annotations. But + + - We believe it is more appropriate to encourage this in style guides, + linters, and autoformatters than to bake it into the parser and throw + syntax errors. + - Moreover, if a type is complicated enough that readability is a concern + we can always use type aliases, for example:: + + IntToIntFunction: (int) -> int + + def make_adder() -> IntToIntFunction: + return lambda x: x + 1 + + +Making ``->`` bind tighter than ``|`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to allow both ``->`` and ``|`` tokens in type expressions we +had to choose precedence. In the current proposal, this is a function +returning an optional boolean:: + + (int, str) -> bool | None # equivalent to (int, str) -> (bool | None) + +We considered having ``->`` bind tighter so that instead the expression +would parse as ``((int, str) -> bool) | None``. There are two advantages +to this: + +- It means we no would longer have to treat ``None | (int, str) -> + bool`` as a syntax error. +- Looking at typeshed today, optional callable arguments are very common + because using ``None`` as a default value is a standard Python idiom. + Having ``->`` bind tighter would make these easier to write. + +We decided against this for a few reasons: + +- The function header ``def f() -> int | None: ...`` is legal + and indicates a function returning an optional int. To be consistent + with function headers, callable types should do the same. +- TypeScript is the other popular language we know of that uses both + ``->`` and ``|`` tokens in type expressions, and they have ``|`` bind + tighter. While we do not have to follow their lead, we prefer to do + so. +- We do acknowledge that optional callable types are common and + having ``|`` bind tighter forces extra parentheses, which makes these + types harder to write. But code is read more often than written, and + we believe that requiring the outer parentheses for an optional callable + type like ``((int, str) -> bool) | None`` is preferable for readability. + + +Introducing type-strings +~~~~~~~~~~~~~~~~~~~~~~~~ + +Another idea was adding a new “special string” syntax and putting the type +inside of it, for example ``t”(int, str) -> bool”``. We rejected this +because it is not as readable, and seems out of step with `guidance +`_ +from the Steering Council on ensuring that type expressions do not +diverge from the rest of Python's syntax. + + +Improving Usability of the Indexed Callable Type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we do not want to add new syntax for callable types, we could +look at how to make the existing type easier to read. One proposal +would be to make the builtin ``callable`` function indexable so +that it could be used as a type:: + + callable[[int, str], bool] + +This change would be analogous to :pep:`585` that made built in collections +like ``list`` and ``dict`` usable as types, and would make imports +more convenient, but it wouldn't help readability of the types themselves +much. + +In order to reduce the number of brackets needed in complex callable +types, it would be possible to allow tuples for the argument list:: + + callable[(int, str), bool] + +This actually is a significant readability improvement for +multi-argument functions, but the problem is that it makes callables +with one arguments, which are the most common arity, hard to +write: because ``(x)`` evaluates to ``x``, they would have to be +written like ``callable[(int,), bool]``. We find this awkward. + +Moreover, none of these ideas help as much with reducing verbosity +as the current proposal, nor do they introduce as strong a visual cue +as the ``->`` between the parameter types and the return type. + +Alternative Runtime Behaviors +----------------------------- + +The hard requirements on our runtime API are that: + +- It must preserve backward compatibility with ``typing.Callable`` via + ``__args__`` and ``__params__``. +- It must provide a structured API, which should be extensible if + in the future we try to support named and variadic arguments. + +Alternative APIs +~~~~~~~~~~~~~~~~ + +We considered having the runtime data ``types.CallableType`` use a +more structured API where there would be separate fields for +``posonlyargs`` and ``param_spec``. The current proposal was +was inspired by the ``inspect.Signature`` type. + +We use "argument" in our field and type names, unlike "parameter" +as in ``inspect.Signature``, in order to avoid confusion with +the ``callable_type.__parameters__`` field from the legacy API +that refers to type parameters rather than callable parameters. + +Using the plain return type in ``__args__`` for async types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is debatable whether we are required to preserve backward compatibility +of ``__args__`` for async callable types like ``async (int) -> str``. The +reason is that one could argue they are not expressible directly +using ``typing.Callable``, and therefore it would be fine to set +``__args__`` as ``(int, int)`` rather than ``(int, typing.Awaitable[int])``. + +But we believe this would be problematic. By preserving the appearance +of a backward-compatible API while actually breaking its semantics on +async types, we would cause runtime type libraries that attempt to +interpret ``Callable`` using ``__args__`` to fail silently. + +It is for this reason that we automatically wrap the return type in +``Awaitable``. + +Backward Compatibility +====================== + +This PEP proposes a major syntax improvement over ``typing.Callable``, +but the static semantics are the same. + +As such, the only thing we need for backward compatibility is to +ensure that types specified via the new syntax behave the same as +equivalent ``typing.Callable`` and ``typing.Concatenate`` values they +intend to replace. + +There is no particular interaction between this proposal and ``from +__future__ import annotations`` - just like any other type annotation +it will be unparsed to a string at module import, and +``typing.get_type_hints`` should correctly evaluate the resulting +strings in cases where that is possible. + +This is discussed in more detail in the Runtime Behavior section. + + +Reference Implementation +======================== + +We have a working `implementation +`_ +of the AST and Grammar with tests verifying that the grammar proposed +here has the desired behaviors. + +The runtime behavior is not yet implemented. As discussed in the +`Runtime Behavior`_ portion of the spec we have a detailed plan for +both a backward-compatible API and a more structured API in +`a separate doc +`_ +where we are also open to discussion and alternative ideas. + + +Open Issues +=========== + +Details of the Runtime API +-------------------------- + +We have attempted to provide a complete behavior specification in +the `Runtime Behavior`_ section of this PEP. + +But there are probably more details that we will not realize we +need to define until we build a full reference implementation. + +Optimizing ``SyntaxError`` messages +----------------------------------- + +The current reference implementation has a fully-functional parser and +all edge cases presented here have been tested. + +But there are some known cases where the errors are not as informative +as we would like. For example, because ``(int, ...) -> bool`` is +illegal but ``(int, ...)`` is a valid tuple, we currently produce a +syntax error flagging the ``->`` as the problem even though the real +cause of the error is using ``...`` as an argument type. + +This is not part of the specification *per se* but is an important +detail to address in our implementation. The solution will likely +involve adding ``invalid_.*`` rules to ``python.gram`` and customizing +error messages. + +Resources +========= + +Background and History +---------------------- + +:pep:`PEP 484 specifies +<484#suggested-syntax-for-python-2-7-and-straddling-code>` +a very similar syntax for function type hint *comments* for use in +code that needs to work on Python 2.7. For example:: + + def f(x, y): + # type: (int, str) -> bool + ... + +At that time we used indexing operations to specify generic types like +``typing.Callable`` because we decided not to add syntax for +types. However, we have since begun to do so, e.g. with :pep:`604`. + +**Maggie** proposed better callable type syntax as part of a larger +`presentation on typing simplifications +`_ +at the PyCon Typing Summit 2021. + +**Steven** `brought up this proposal on typing-sig +`_. +We had several meetings to discuss alternatives, and `this presentation +`_ +led us to the current proposal. + +**Pradeep** `brought this proposal to python-dev +`_ +for feedback. + +Acknowledgments +--------------- + +Thanks to the following people for their feedback on the PEP and help +planning the reference implementation: + +Alex Waygood, Eric Traut, Guido van Rossum, James Hilton-Balfe, +Jelle Zijlstra, Maggie Moss, Tuomas Suutari, Shannon Zhu. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/peps/pep-0678.rst b/peps/pep-0678.rst new file mode 100644 index 000000000..140cef283 --- /dev/null +++ b/peps/pep-0678.rst @@ -0,0 +1,385 @@ +PEP: 678 +Title: Enriching Exceptions with Notes +Author: Zac Hatfield-Dodds +Sponsor: Irit Katriel +Discussions-To: https://discuss.python.org/t/pep-678-enriching-exceptions-with-notes/13374 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Requires: 654 +Created: 20-Dec-2021 +Python-Version: 3.11 +Post-History: 27-Jan-2022 +Resolution: https://discuss.python.org/t/pep-678-enriching-exceptions-with-notes/13374/100 + + +Abstract +======== +Exception objects are typically initialized with a message that describes the +error which has occurred. Because further information may be available when +the exception is caught and re-raised, or included in an ``ExceptionGroup``, +this PEP proposes to add ``BaseException.add_note(note)``, a +``.__notes__`` attribute holding a list of notes so added, and to +update the builtin traceback formatting code to include notes in the formatted +traceback following the exception string. + +This is particularly useful in relation to :pep:`654` ``ExceptionGroup``\ s, +which make previous workarounds ineffective or confusing. Use cases have been +identified in the standard library, Hypothesis and ``cattrs`` packages, and +common code patterns with retries. + + +Motivation +========== +When an exception is created in order to be raised, it is usually initialized +with information that describes the error that has occurred. There are cases +where it is useful to add information after the exception was caught. For +example, + +- testing libraries may wish to show the values involved in a failing + assertion, or the steps to reproduce a failure (e.g. ``pytest`` and + ``hypothesis``; example below). +- code which retries an operation on error may wish to associate an iteration, + timestamp, or other explanation with each of several errors - especially if + re-raising them in an ``ExceptionGroup``. +- programming environments for novices can provide more detailed descriptions + of various errors, and tips for resolving them. + +Existing approaches must pass this additional information around while keeping +it in sync with the state of raised, and potentially caught or chained, +exceptions. This is already error-prone, and made more difficult by :pep:`654` +``ExceptionGroup``\ s, so the time is right for a built-in solution. We +therefore propose to add: + +- a new method ``BaseException.add_note(note: str)``, +- ``BaseException.__notes__``, a list of note strings added using + ``.add_note()``, and +- support in the builtin traceback formatting code such that notes are + displayed in the formatted traceback following the exception string. + + +Example usage +------------- +:: + + >>> try: + ... raise TypeError('bad type') + ... except Exception as e: + ... e.add_note('Add some information') + ... raise + ... + Traceback (most recent call last): + File "", line 2, in + TypeError: bad type + Add some information + >>> + +When collecting exceptions into an exception group, we may want to add context +information for the individual errors. In the following example with +`Hypothesis' proposed support for ExceptionGroup +`__, each exception +includes a note of the minimal failing example:: + + from hypothesis import given, strategies as st, target + + @given(st.integers()) + def test(x): + assert x < 0 + assert x > 0 + + + + Exception Group Traceback (most recent call last): + | File "test.py", line 4, in test + | def test(x): + | + | File "hypothesis/core.py", line 1202, in wrapped_test + | raise the_error_hypothesis_found + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | ExceptionGroup: Hypothesis found 2 distinct failures. + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "test.py", line 6, in test + | assert x > 0 + | ^^^^^^^^^^^^ + | AssertionError: assert -1 > 0 + | + | Falsifying example: test( + | x=-1, + | ) + +---------------- 2 ---------------- + | Traceback (most recent call last): + | File "test.py", line 5, in test + | assert x < 0 + | ^^^^^^^^^^^^ + | AssertionError: assert 0 < 0 + | + | Falsifying example: test( + | x=0, + | ) + +------------------------------------ + + +Non-goals +--------- +Tracking multiple notes as a list, rather than by concatenating strings when +notes are added, is intended to maintain the distinction between the +individual notes. This might be required in specialized use cases, such +as translation of the notes by packages like ``friendly-traceback``. + +However, ``__notes__`` is *not* intended to carry structured data. If your +note is for use by a program rather than display to a human, `we recommend +`__ +instead (or additionally) choosing a convention for an attribute, e.g. +``err._parse_errors = ...`` on the error or ``ExceptionGroup``. + +As a rule of thumb, we suggest that you should prefer `exception chaining +`__ when the +error is going to be re-raised or handled as an individual error, and prefer +``.add_note()`` when you want to avoid changing the exception type or +are collecting multiple exception objects to handle together. [1]_ + + +Specification +============= + +``BaseException`` gains a new method ``.add_note(note: str)``. If ``note`` is +a string, ``.add_note(note)`` appends it to the ``__notes__`` list, creating +the attribute if it does not already exist. If ``note`` is not a string, +``.add_note()`` raises ``TypeError``. + +Libraries may clear existing notes by modifying or deleting the ``__notes__`` +list, if it has been created, including clearing all notes with +``del err.__notes__``. This allows full control over the attached notes, +without overly complicating the API or adding multiple names to +``BaseException.__dict__``. + +When an exception is displayed by the interpreter's builtin traceback-rendering code, +its notes (if there are any) appear immediately after the exception message, in the order +in which they were added, with each note starting on a new line. + +If ``__notes__`` has been created, ``BaseExceptionGroup.subgroup`` and +``BaseExceptionGroup.split`` create a new list for each new instance, containing +the same contents as the original exception group's ``__notes__``. + +We *do not* specify the expected behaviour when users have assigned a non-list +value to ``__notes__``, or a list which contains non-string elements. +Implementations might choose to emit warnings, discard or ignore bad values, +convert them to strings, raise an exception, or do something else entirely. + + +Backwards Compatibility +======================= + +System-defined or "dunder" names (following the pattern ``__*__``) are part of +the language specification, with `unassigned names reserved for future use and +subject to breakage without warning +`__. +We are also unaware of any code which *would* be broken by adding ``__notes__``. + +We were also unable to find any code which would be broken by the addition of +``BaseException.add_note()``: while searching Google and `GitHub finds several +definitions `__ +of an ``.add_note()`` method, none of them are on a subclass of +``BaseException``. + + +How to Teach This +================= + +The ``add_note()`` method and ``__notes__`` attribute will be documented as +part of the language standard, and explained as part of `the "Errors and +Exceptions" tutorial `__. + + +Reference Implementation +======================== + +Following discussions related to :pep:`654` [2]_, an early version of this +proposal was `implemented in `__ +and released in CPython 3.11.0a3, with a mutable string-or-none ``__note__`` +attribute. + +`CPython PR #31317 `__ +implements ``.add_note()`` and ``__notes__``. + + +Rejected Ideas +============== + +.. _print_idea: + +Use ``print()`` (or ``logging``, etc.) +-------------------------------------- +Reporting explanatory or contextual information about an error by printing or +logging has historically been an acceptable workaround. However, we dislike +the way this separates the content from the exception object it refers to - +which can lead to "orphan" reports if the error was caught and handled later, +or merely significant difficulties working out which explanation corresponds to +which error. The new ``ExceptionGroup`` type intensifies these existing +challenges. + +Keeping the ``__notes__`` attached to the exception object, in the same way as +the ``__traceback__`` attribute, eliminates these problems. + + +``raise Wrapper(explanation) from err`` +--------------------------------------- +An alternative pattern is to use exception chaining: by raising a 'wrapper' +exception containing the context or explanation ``from`` the current exception, +we avoid the separation challenges from ``print()``. However, this has two key +problems. + +First, it changes the type of the exception, which is often a breaking change +for downstream code. We consider *always* raising a ``Wrapper`` exception +unacceptably inelegant; but because custom exception types might have any +number of required arguments we can't always create an instance of the *same* +type with our explanation. In cases where the exact exception type is known +this can work, such as the standard library ``http.client`` `code +`__, +but not for libraries which call user code. + +Second, exception chaining reports several lines of additional detail, which +are distracting for experienced users and can be very confusing for beginners. +For example, six of the eleven lines reported for this simple example relate to +exception chaining, and are unnecessary with ``BaseException.add_note()``: + +.. code-block:: python + + class Explanation(Exception): + def __str__(self): + return "\n" + str(self.args[0]) + + try: + raise AssertionError("Failed!") + except Exception as e: + raise Explanation("You can reproduce this error by ...") from e + +.. code-block:: + + $ python example.py + Traceback (most recent call last): + File "example.py", line 6, in + raise AssertionError(why) + AssertionError: Failed! + # These lines are + The above exception was the direct cause of ... # confusing for new + # users, and they + Traceback (most recent call last): # only exist due + File "example.py", line 8, in # to implementation + raise Explanation(msg) from e # constraints :-( + Explanation: # Hence this PEP! + You can reproduce this error by ... + +**In cases where these two problems do not apply, we encourage use of exception +chaining rather than** ``__notes__``. + + +An assignable ``__note__`` attribute +------------------------------------ +The first draft and implementation of this PEP defined a single attribute +``__note__``, which defaulted to ``None`` but could have a string assigned. +This is substantially simpler if, and only if, there is at most one note. + +To promote interoperability and support translation of error messages by +libraries such as ``friendly-traceback``, without resorting to dubious parsing +heuristics, we therefore settled on the ``.add_note()``-and-``__notes__`` API. + + +Subclass Exception and add note support downstream +-------------------------------------------------- +Traceback printing is built into the C code, and reimplemented in pure Python +in ``traceback.py``. To get ``err.__notes__`` printed from a downstream +implementation would *also* require writing custom traceback-printing code; +while this could be shared between projects and reuse some pieces of +traceback.py [3]_ we prefer to implement this once, upstream. + +Custom exception types could implement their ``__str__`` method to include our +proposed ``__notes__`` semantics, but this would be rarely and inconsistently +applicable. + + +Don't attach notes to ``Exception``\ s, just store them in ``ExceptionGroup``\ s +-------------------------------------------------------------------------------- +The initial motivation for this PEP was to associate a note with each error +in an ``ExceptionGroup``. At the cost of a remarkably awkward API and the +cross-referencing problem discussed `above `__, this +use-case could be supported by storing notes on the ``ExceptionGroup`` +instance instead of on each exception it contains. + +We believe that the cleaner interface, and other use-cases described above, +are sufficient to justify the more general feature proposed by this PEP. + + +Add a helper function ``contextlib.add_exc_note()`` +--------------------------------------------------- +It `was suggested +`__ +that we add a utility such as the one below to the standard library. We do not +see this idea as core to the proposal of this PEP, and thus leave it for later +or downstream implementation - perhaps based on this example code: + +.. code-block:: python + + @contextlib.contextmanager + def add_exc_note(note: str): + try: + yield + except Exception as err: + err.add_note(note) + raise + + with add_exc_note(f"While attempting to frobnicate {item=}"): + frobnicate_or_raise(item) + + +Augment the ``raise`` statement +------------------------------- +One discussion proposed ``raise Exception() with "note contents"``, but this +does not address the original motivation of compatibility with +``ExceptionGroup``. + +Furthermore, we do not believe that the problem we are solving requires or +justifies new language syntax. + + +Acknowledgements +================ +We wish to thank the many people who have assisted us through conversation, +code review, design advice, and implementation: Adam Turner, Alex Grönholm, +AndrĂ© Roberge, Barry Warsaw, Brett Cannon, CAM Gerlach, Carol Willing, Damian, +Erlend Aasland, Etienne Pot, Gregory Smith, Guido van Rossum, Irit Katriel, +Jelle Zijlstra, Ken Jin, Kumar Aditya, Mark Shannon, Matti Picus, Petr +Viktorin, Will McGugan, and pseudonymous commenters on Discord and Reddit. + + +References +========== + +.. [1] this principle was established in the 2003 mail thread which led to :pep:`3134`, + and included a proposal for a group-of-exceptions type! + https://mail.python.org/pipermail/python-dev/2003-January/032492.html +.. [2] particularly those at https://bugs.python.org/issue45607, + https://discuss.python.org/t/accepting-pep-654-exception-groups-and-except/10813/9, + https://github.com/python/cpython/pull/28569#discussion_r721768348, and +.. [3] We note that the ``exceptiongroup`` backport package maintains an exception + hook and monkeypatch for ``TracebackException`` for Pythons older than 3.11, + and encourage library authors to avoid creating additional and incompatible + backports. We also reiterate our preference for builtin support which + makes such measures unnecessary. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/peps/pep-0679.rst b/peps/pep-0679.rst new file mode 100644 index 000000000..ed941b823 --- /dev/null +++ b/peps/pep-0679.rst @@ -0,0 +1,175 @@ +PEP: 679 +Title: Allow parentheses in assert statements +Author: Pablo Galindo Salgado +Discussions-To: https://discuss.python.org/t/pep-679-allow-parentheses-in-assert-statements/13003 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 07-Jan-2022 +Python-Version: 3.12 + + +Abstract +======== + +This PEP proposes to allow parentheses surrounding the two-argument form of +assert statements. This will cause the interpreter to reinterpret what before +would have been an assert with a two-element tuple that will always be True +(``assert (expression, message)``) to an assert statement with a subject and a +failure message, equivalent to the statement with the parentheses removed +(``assert expression, message``). + + +Motivation +========== + +It is a common user mistake when using the form of the assert statement that includes +the error message to surround it with parentheses. Unfortunately, this mistake +passes undetected as the assert will always pass, because it is +interpreted as an assert statement where the expression is a two-tuple, which +always has truth-y value. + +The mistake most often happens when extending the test or description beyond a +single line, as parentheses are the natural way to do that. + +This is so common that a ``SyntaxWarning`` is `now emitted by the compiler +`_. + +Additionally, some other statements in the language allow parenthesized forms +in one way or another like ``import`` statements (``from x import (a,b,c)``) and +``del`` statements (``del (a,b,c)``). + +Allowing parentheses not only will remove the common mistake but also will allow +users and auto-formatters to format long assert statements over multiple lines +in what the authors of this document believe will be a more natural way. +Although is possible to currently format long ``assert`` statements over +multiple lines as:: + + assert ( + very very long + expression + ), ( + "very very long " + "message" + ) + +the authors of this document believe the parenthesized form is more clear and more consistent with +the formatting of other grammar constructs:: + + assert ( + very very long + expression, + + "very very long " + "message", + ) + +This change has been originally discussed and proposed in [bpo-46167]_. + +Rationale +========= + +This change can be implemented in the parser or in the compiler. We have +selected implementing this change in the parser because doing it in the compiler +will require re-interpreting the AST of an assert statement with a two-tuple:: + + Module( + body=[ + Assert( + test=Tuple( + elts=[ + Name(id='x', ctx=Load()), + Name(id='y', ctx=Load())], + ctx=Load()))], + type_ignores=[]) + +as the AST of an assert statement with an expression and a message:: + + Module( + body=[ + Assert( + test=Name(id='x', ctx=Load()), + msg=Name(id='y', ctx=Load()))], + type_ignores=[]) + +The problem with this approach is that the AST of the first form will +technically be "incorrect" as we already have a specialized form for the AST of +an assert statement with a test and a message (the second one). This +means that many tools that deal with ASTs will need to be aware of this change +in semantics, which will be confusing as there is already a correct form that +better expresses the new meaning. + +Specification +============= + +This PEP proposes changing the grammar of the ``assert`` statement to: :: + + | 'assert' '(' expression ',' expression [','] ')' &(NEWLINE | ';') + | 'assert' a=expression [',' expression ] + +Where the first line is the new form of the assert statement that allows +parentheses. The lookahead is needed so statements like ``assert (a, b) <= c, +"something"`` are still parsed correctly and to prevent the parser to eagerly +capture the tuple as the full statement. + +Optionally, new "invalid" rule can be added to produce custom syntax errors to +cover tuples with 0, 1, 3 or more elements. + + +Backwards Compatibility +======================= + +The change is not technically backwards compatible, as parsing ``assert (x,y)`` +is currently interpreted as an assert statement with a 2-tuple as the subject, +while after this change it will be interpreted as ``assert x,y``. + +On the other hand, assert statements of this kind always pass, so they are +effectively not doing anything in user code. The authors of this document think +that this backwards incompatibility nature is beneficial, as it will highlight +these cases in user code while before they will have passed unnoticed (assuming that +these cases still exist because users are ignoring syntax warnings). + +Security Implications +===================== + +There are no security implications for this change. + + +How to Teach This +================= + +The new form of the ``assert`` statement will be documented as part of the language +standard. + +When teaching the form with error message of the ``assert`` statement to users, +now it can be noted that adding parentheses also work as expected, which allows to break +the statement over multiple lines. + + +Reference Implementation +======================== + +A proposed draft PR with the change exist in [GH-30247]_. + + +References +========== + +.. [bpo-46167] https://bugs.python.org/issue46167 +.. [GH-30247] https://github.com/python/cpython/pull/30247 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/peps/pep-0680.rst b/peps/pep-0680.rst new file mode 100644 index 000000000..97e250d15 --- /dev/null +++ b/peps/pep-0680.rst @@ -0,0 +1,531 @@ +PEP: 680 +Title: tomllib: Support for Parsing TOML in the Standard Library +Author: Taneli Hukkinen, Shantanu Jain +Sponsor: Petr Viktorin +Discussions-To: https://discuss.python.org/t/13040 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 01-Jan-2022 +Python-Version: 3.11 +Post-History: 11-Jan-2022 +Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/3AHGWYY562HHO55L4Z2OVYUFZP5W73IS/ + + +Abstract +======== + +This PEP proposes adding the ``tomllib`` module to the standard library for +parsing TOML (Tom's Obvious Minimal Language, +`https://toml.io `_). + + +Motivation +========== + +TOML is the format of choice for Python packaging, as evidenced by +:pep:`517`, :pep:`518` and :pep:`621`. This creates a bootstrapping +problem for Python build tools, forcing them to vendor a TOML parsing +package or employ other undesirable workarounds, and causes serious issues +for repackagers and other downstream consumers. Including TOML support in +the standard library would neatly solve all of these issues. + +Further, many Python tools are now configurable via TOML, such as +``black``, ``mypy``, ``pytest``, ``tox``, ``pylint`` and ``isort``. +Many that are not, such as ``flake8``, cite the lack of standard library +support as a `main reason why +`__. +Given the special place TOML already has in the Python ecosystem, it makes sense +for it to be an included battery. + +Finally, TOML as a format is increasingly popular (for the reasons +outlined in :pep:`518`), with various Python TOML libraries having about +2000 reverse dependencies on PyPI (for comparison, ``requests`` has about +28000 reverse dependencies). Hence, this is likely to be a generally useful +addition, even looking beyond the needs of Python packaging and related tools. + + +Rationale +========= + +This PEP proposes basing the standard library support for reading TOML on the +third-party library ``tomli`` +(`github.com/hukkin/tomli `_). + +Many projects have recently switched to using ``tomli``, such as ``pip``, +``build``, ``pytest``, ``mypy``, ``black``, ``flit``, ``coverage``, +``setuptools-scm`` and ``cibuildwheel``. + +``tomli`` is actively maintained and well-tested. It is about 800 lines +of code with 100% test coverage, and passes all tests in the +`proposed official TOML compliance test suite +`_, as well as +`the more established BurntSushi/toml-test suite +`_. + + +Specification +============= + +A new module ``tomllib`` will be added to the Python standard library, +exposing the following public functions: + +.. code-block:: + + def load( + fp: SupportsRead[bytes], + /, + *, + parse_float: Callable[[str], Any] = ..., + ) -> dict[str, Any]: ... + + def loads( + s: str, + /, + *, + parse_float: Callable[[str], Any] = ..., + ) -> dict[str, Any]: ... + +``tomllib.load`` deserializes a binary file-like object containing a +TOML document to a Python ``dict``. +The ``fp`` argument must have a ``read()`` method with the same API as +``io.RawIOBase.read()``. + +``tomllib.loads`` deserializes a ``str`` instance containing a TOML document +to a Python ``dict``. + +The ``parse_float`` argument is a callable object that takes as input the +original string representation of a TOML float, and returns a corresponding +Python object (similar to ``parse_float`` in ``json.load``). +For example, the user may pass a function returning a ``decimal.Decimal``, +for use cases where exact precision is important. By default, TOML floats +are parsed as instances of the Python ``float`` type. + +The returned object contains only basic Python objects (``str``, ``int``, +``bool``, ``float``, ``datetime.{datetime,date,time}``, ``list``, ``dict`` with +string keys), and the results of ``parse_float``. + +``tomllib.TOMLDecodeError`` is raised in the case of invalid TOML. + +Note that this PEP does not propose ``tomllib.dump`` or ``tomllib.dumps`` +functions; see `Including an API for writing TOML`_ for details. + + +Maintenance Implications +======================== + +Stability of TOML +----------------- + +The release of TOML 1.0.0 in January 2021 indicates the TOML format should +now be officially considered stable. Empirically, TOML has proven to be a +stable format even prior to the release of TOML 1.0.0. From the +`changelog `__, we +can see that TOML has had no major changes since April 2020, and has had +two releases in the past five years (2017-2021). + +In the event of changes to the TOML specification, we can treat minor +revisions as bug fixes and update the implementation in place. In the event of +major breaking changes, we should preserve support for TOML 1.x. + + +Maintainability of proposed implementation +------------------------------------------ + +The proposed implementation (``tomli``) is pure Python, well tested and +weighs in at under 1000 lines of code. It is minimalist, offering a smaller API +surface area than other TOML implementations. + +The author of ``tomli`` is willing to help integrate ``tomli`` into the standard +library and help maintain it, `as per this post +`__. +Furthermore, Python core developer Petr Viktorin has indicated a willingness +to maintain a read API, `as per this post +`__. + +Rewriting the parser in C is not deemed necessary at this time. It is rare for +TOML parsing to be a bottleneck in applications, and users with higher performance +needs can use a third-party library (as is already often the case with JSON, +despite Python offering a standard library C-extension module). + + +TOML support a slippery slope for other things +---------------------------------------------- + +As discussed in the `Motivation`_ section, TOML holds a special place in the +Python ecosystem, for reading :pep:`518` ``pyproject.toml`` packaging +and tool configuration files. +This chief reason to include TOML in the standard library does not apply to +other formats, such as YAML or MessagePack. + +In addition, the simplicity of TOML distinguishes it from other formats like +YAML, which are highly complicated to construct and parse. + +An API for writing TOML may, however, be added in a future PEP. + + +Backwards Compatibility +======================= + +This proposal has no backwards compatibility issues within the standard +library, as it describes a new module. +Any existing third-party module named ``tomllib`` will break, as +``import tomllib`` will import the standard library module. +However, ``tomllib`` is not registered on PyPI, so it is unlikely that any +module with this name is widely used. + +Note that we avoid using the more straightforward name ``toml`` to avoid +backwards compatibility implications for users who have pinned versions of the +current ``toml`` PyPI package. +For more details, see the `Alternative names for the module`_ section. + + +Security Implications +===================== + +Errors in the implementation could cause potential security issues. +However, the parser's output is limited to simple data types; inability to load +arbitrary classes avoids security issues common in more "powerful" formats like +pickle and YAML. Also, the implementation will be in pure Python, which reduces +security issues endemic to C, such as buffer overflows. + + +How to Teach This +================= + +The API of ``tomllib`` mimics that of other well-established file format +libraries, such as ``json`` and ``pickle``. The lack of a ``dump`` function will +be explained in the documentation, with a link to relevant third-party libraries +(e.g. ``tomlkit``, ``tomli-w``, ``pytomlpp``). + + +Reference Implementation +======================== + +The proposed implementation can be found at https://github.com/hukkin/tomli + + +Rejected Ideas +============== + +Basing on another TOML implementation +------------------------------------- + +Several potential alternative implementations exist: + +* ``tomlkit`` is well established, actively maintained and supports TOML 1.0.0. + An important difference is that ``tomlkit`` supports style roundtripping. As a + result, it has a more complex API and implementation (about 5x as much code as + ``tomli``). Its author does not believe that ``tomlkit`` is a good choice for + the standard library. + +* ``toml`` is a very widely used library. However, it is not actively + maintained, does not support TOML 1.0.0 and has a number of known bugs. Its + API is more complex than that of ``tomli``. It allows customising output style + through a complicated encoder API, and some very limited and mostly unused + functionality to preserve input style through an undocumented decoder API. + For more details on its API differences from this PEP, refer to `Appendix A + `_. + +* ``pytomlpp`` is a Python wrapper for the C++ project ``toml++``. Pure Python + libraries are easier to maintain than extension modules. + +* ``rtoml`` is a Python wrapper for the Rust project ``toml-rs`` and hence has + similar shortcomings to ``pytomlpp``. + In addition, it does not support TOML 1.0.0. + +* Writing an implementation from scratch. It's unclear what we would get from + this; ``tomli`` meets our needs and the author is willing to help with its + inclusion in the standard library. + + +Including an API for writing TOML +--------------------------------- + +There are several reasons to not include an API for writing TOML. + +The ability to write TOML is not needed for the use cases that motivate this +PEP: core Python packaging tools, and projects that need to read TOML +configuration files. + +Use cases that involve editing an existing TOML file (as opposed to writing a +brand new one) are better served by a style preserving library. TOML is +intended as a human-readable and -editable configuration format, so it's +important to preserve comments, formatting and other markup. This requires +a parser whose output includes style-related metadata, making it impractical +to output plain Python types like ``str`` and ``dict``. Furthermore, it +substantially complicates the design of the API. + +Even without considering style preservation, there are too many degrees of +freedom in how to design a write API. For example, what default style +(indentation, vertical and horizontal spacing, quotes, etc) should the library +use for the output, and how much control should users be given over it? +How should the library handle input and output validation? Should it support +serialization of custom types, and if so, how? While there are reasonable +options for resolving these issues, the nature of the standard library is such +that we only get "one chance to get it right". + +Currently, no CPython core developers have expressed willingness to maintain a +write API, or sponsor a PEP that includes one. Since it is hard to change +or remove something in the standard library, it is safer to err on the side of +exclusion for now, and potentially revisit this later. + +Therefore, writing TOML is left to third-party libraries. If a good API and +relevant use cases for it are found later, write support can be added in a +future PEP. + + +Assorted API details +-------------------- + +Types accepted as the first argument of ``tomllib.load`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +The ``toml`` library on PyPI allows passing paths (and lists of path-like +objects, ignoring missing files and merging the documents into a single object) +to its ``load`` function. However, allowing this here would be inconsistent +with the behavior of ``json.load``, ``pickle.load`` and other standard library +functions. If we agree that consistency here is desirable, +allowing paths is out of scope for this PEP. This can easily and explicitly +be worked around in user code, or by using a third-party library. + +The proposed API takes a binary file, while ``toml.load`` takes a text file and +``json.load`` takes either. Using a binary file allows us to ensure UTF-8 is +the encoding used (ensuring correct parsing on platforms with other default +encodings, such as Windows), and avoid incorrectly parsing files containing +single carriage returns as valid TOML due to universal newlines in text mode. + + +Type accepted as the first argument of ``tomllib.loads`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +While ``tomllib.load`` takes a binary file, ``tomllib.loads`` takes +a text string. This may seem inconsistent at first. + +Quoting the `TOML v1.0.0 specification `_: + + A TOML file must be a valid UTF-8 encoded Unicode document. + +``tomllib.loads`` does not intend to load a TOML file, but rather the +document that the file stores. The most natural representation of +a Unicode document in Python is ``str``, not ``bytes``. + +It is possible to add ``bytes`` support in the future if needed, but +we are not aware of any use cases for it. + + +Controlling the type of mappings returned by ``tomllib.load[s]`` +---------------------------------------------------------------- + +The ``toml`` library on PyPI accepts a ``_dict`` argument in its ``load[s]`` +functions, which works similarly to the ``object_hook`` argument in +``json.load[s]``. There are several uses of ``_dict`` found on +https://grep.app; however, almost all of them are passing +``_dict=OrderedDict``, which should be unnecessary as of Python 3.7. +We found two instances of relevant use: in one case, a custom class was passed +for friendlier KeyErrors; in the other, the custom class had several +additional lookup and mutation methods (e.g. to help resolve dotted keys). + +Such a parameter is not necessary for the core use cases outlined in the +`Motivation`_ section. The absence of this can be pretty easily worked around +using a wrapper class, transformer function, or a third-party library. Finally, +support could be added later in a backward-compatible way. + + +Removing support for ``parse_float`` in ``tomllib.load[s]`` +----------------------------------------------------------- + +This option is not strictly necessary, since TOML floats should be implemented +as "IEEE 754 binary64 values", which is equivalent to a Python ``float`` on most +architectures. + +The TOML specification uses the word "SHOULD", however, implying a +recommendation that can be ignored for valid reasons. Parsing floats +differently, such as to ``decimal.Decimal``, allows users extra precision beyond +that promised by the TOML format. In the author of ``tomli``'s experience, this +is particularly useful in scientific and financial applications. This is also +useful for other cases that need greater precision, or where end-users include +non-developers who may not be aware of the limits of binary64 floats. + +There are also niche architectures where the Python ``float`` is not a IEEE 754 +binary64 value. The ``parse_float`` argument allows users to achieve correct +TOML semantics even on such architectures. + + +Alternative names for the module +-------------------------------- + +Ideally, we would be able to use the ``toml`` module name. + +However, the ``toml`` package on PyPI is widely used, so there are backward +compatibility concerns. Since the standard library takes precedence over third +party packages, libraries and applications who current depend on the ``toml`` +package would likely break when upgrading Python versions due to the many +API incompatibilities listed in `Appendix A `_, +even if they pin their dependency versions. + +To further clarify, applications with pinned dependencies are of greatest +concern here. Even if we were able to obtain control of the ``toml`` PyPI +package name and repurpose it for a backport of the proposed new module, +we would still break users on new Python versions that included it in the +standard library, regardless of whether they have pinned an older version of +the existing ``toml`` package. This is unfortunate, since pinning +would likely be a common response to breaking changes introduced by repurposing +the ``toml`` package as a backport (that is incompatible with today's ``toml``). + +Finally, the ``toml`` package on PyPI is not actively maintained, but as of +yet, efforts to request that the author add other maintainers +`have been unsuccessful `__, +so action here would likely have to be taken without the author's consent. + +Instead, this PEP proposes the name ``tomllib``. This mirrors ``plistlib`` +and ``xdrlib``, two other file format modules in the standard library, as well +as other modules, such as ``pathlib``, ``contextlib`` and ``graphlib``. + +Other names considered but rejected include: + +* ``tomlparser``. This mirrors ``configparser``, but is perhaps somewhat less + appropriate if we include a write API in the future. +* ``tomli``. This assumes we use ``tomli`` as the basis for implementation. +* ``toml`` under some namespace, such as ``parser.toml``. However, this is + awkward, especially so since existing parsing libraries like ``json``, + ``pickle``, ``xml``, ``html`` etc. would not be included in the namespace. + + +Previous Discussion +=================== + +* `bpo-40059: Provide a toml module in the standard library + `_ +* `[Python-Dev] Adding a toml module to the standard lib? + `_ +* `[Python-Ideas] Python standard library TOML module + `_ +* `[Packaging] Adopting/recommending a toml parser? + `_ +* `hukkin/tomli#141: Please consider pushing tomli into the stdlib + `_ + + +.. _PEP 680 Appendix A: + +Appendix A: Differences between proposed API and ``toml`` +========================================================= + +This appendix covers the differences between the API proposed in this PEP and +that of the third-party package ``toml``. These differences are relevant to +understanding the amount of breakage we could expect if we used the ``toml`` +name for the standard library module, as well as to better understand the design +space. Note that this list might not be exhaustive. + +#. No proposed inclusion of a write API (no ``toml.dump[s]``) + + This PEP currently proposes not including a write API; that is, there will + be no equivalent of ``toml.dump`` or ``toml.dumps``, as discussed at + `Including an API for writing TOML`_. + + If we included a write API, it would be relatively straightforward to + convert most code that uses ``toml`` to the new standard library module + (acknowledging that this is very different from a compatible API, as it + would still require code changes). + + A significant fraction of ``toml`` users rely on this, based on comparing + `occurrences of "toml.load" `__ + to `occurrences of "toml.dump" `__. + +#. Different first argument of ``toml.load`` + + ``toml.load`` has the following signature: + + .. code-block:: + + def load( + f: Union[SupportsRead[str], str, bytes, list[PathLike | str | bytes]], + _dict: Type[MutableMapping[str, Any]] = ..., + decoder: TomlDecoder = ..., + ) -> MutableMapping[str, Any]: ... + + This is quite different from the first argument proposed in this PEP: + ``SupportsRead[bytes]``. + + Recapping the reasons for this, previously mentioned at + `Types accepted as the first argument of tomllib.load`_: + + * Allowing paths (and even lists of paths) as arguments is inconsistent with + other similar functions in the standard library. + * Using ``SupportsRead[bytes]`` allows us to ensure UTF-8 is the encoding used, + and avoid incorrectly parsing single carriage returns as valid TOML. + + A significant fraction of ``toml`` users rely on this, based on manual + inspection of `occurrences of "toml.load" + `__. + +#. Errors + + ``toml`` raises ``TomlDecodeError``, vs. the proposed :pep:`8`-compliant + ``TOMLDecodeError``. + + A significant fraction of ``toml`` users rely on this, based on + `occurrences of "TomlDecodeError" + `__. + +#. ``toml.load[s]`` accepts a ``_dict`` argument + + Discussed at `Controlling the type of mappings returned by tomllib.load[s]`_. + + As mentioned there, almost all usage consists of ``_dict=OrderedDict``, + which is not necessary in Python 3.7 and later. + +#. ``toml.load[s]`` support an undocumented ``decoder`` argument + + It seems the intended use case is for an implementation of comment + preservation. The information recorded is not sufficient to roundtrip the + TOML document preserving style, the implementation has known bugs, the + feature is undocumented and we could only find one instance of its use on + https://grep.app. + + The `toml.TomlDecoder interface + `__ + exposed is far from simple, containing nine methods. + + Users are likely better served by a more complete implementation of + style-preserving parsing and writing. + +#. ``toml.dump[s]`` support an ``encoder`` argument + + Note that we currently propose to not include a write API; however, if that + were to change, these differences would likely become relevant. + + The ``encoder`` argument enables two use cases: + + * control over how custom types should be serialized, and + * control over how output should be formatted. + + The first is reasonable; however, we could only find two instances of + this on https://grep.app. One of these two used this ability to add + support for dumping ``decimal.Decimal``, which a potential standard library + implementation would support out of the box. + If needed for other types, this use case could be well served by the + equivalent of the ``default`` argument in ``json.dump``. + + The second use case is enabled by allowing users to specify subclasses of + `toml.TomlEncoder + `__ + and overriding methods to specify parts of the TOML writing process. The API + consists of five methods and exposes substantial implementation detail. + + There is some usage of the ``encoder`` API on https://grep.app; however, it + appears to account for a tiny fraction of the overall usage of ``toml``. + +#. Timezones + + ``toml`` uses and exposes custom ``toml.tz.TomlTz`` timezone objects. The + proposed implementation uses ``datetime.timezone`` objects from the standard + library. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0681.rst b/peps/pep-0681.rst new file mode 100644 index 000000000..e1b64b97c --- /dev/null +++ b/peps/pep-0681.rst @@ -0,0 +1,763 @@ +PEP: 681 +Title: Data Class Transforms +Author: Erik De Bonte , + Eric Traut +Sponsor: Jelle Zijlstra +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/EAALIHA3XEDFDNG2NRXTI3ERFPAD65Z4/ +Status: Accepted +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 02-Dec-2021 +Python-Version: 3.11 +Post-History: `24-Apr-2021 `__, + `13-Dec-2021 `__, + `22-Feb-2022 `__ +Resolution: https://mail.python.org/archives/list/python-dev@python.org/message/R4A2IYLGFHKFDYJPSDA5NFJ6N7KRPJ6D/ + + +Abstract +======== + +:pep:`557` introduced the dataclass to the Python stdlib. Several popular +libraries have behaviors that are similar to dataclasses, but these +behaviors cannot be described using standard type annotations. Such +projects include attrs, pydantic, and object relational mapper (ORM) +packages such as SQLAlchemy and Django. + +Most type checkers, linters and language servers have full support for +dataclasses. This proposal aims to generalize this functionality and +provide a way for third-party libraries to indicate that certain +decorator functions, classes, and metaclasses provide behaviors +similar to dataclasses. + +These behaviors include: + +* Synthesizing an ``__init__`` method based on declared + data fields. +* Optionally synthesizing ``__eq__``, ``__ne__``, ``__lt__``, + ``__le__``, ``__gt__`` and ``__ge__`` methods. +* Supporting "frozen" classes, a way to enforce immutability during + static type checking. +* Supporting "field specifiers", which describe attributes of + individual fields that a static type checker must be aware of, + such as whether a default value is provided for the field. + +The full behavior of the stdlib dataclass is described in the `Python +documentation <#dataclass-docs_>`_. + +This proposal does not affect CPython directly except for the addition +of a ``dataclass_transform`` decorator in ``typing.py``. + + +Motivation +========== + +There is no existing, standard way for libraries with dataclass-like +semantics to declare their behavior to type checkers. To work around +this limitation, Mypy custom plugins have been developed for many of +these libraries, but these plugins don't work with other type +checkers, linters or language servers. They are also costly to +maintain for library authors, and they require that Python developers +know about the existence of these plugins and download and configure +them within their environment. + + +Rationale +========= + +The intent of this proposal is not to support every feature of every +library with dataclass-like semantics, but rather to make it possible +to use the most common features of these libraries in a way that is +compatible with static type checking. If a user values these libraries +and also values static type checking, they may need to avoid using +certain features or make small adjustments to the way they use them. +That's already true for the Mypy custom plugins, which +don't support every feature of every dataclass-like library. + +As new features are added to dataclasses in the future, we intend, when +appropriate, to add support for those features on +``dataclass_transform`` as well. Keeping these two feature sets in +sync will make it easier for dataclass users to understand and use +``dataclass_transform`` and will simplify the maintenance of dataclass +support in type checkers. + +Additionally, we will consider adding ``dataclass_transform`` support +in the future for features that have been adopted by multiple +third-party libraries but are not supported by dataclasses. + + +Specification +============= + +The ``dataclass_transform`` decorator +------------------------------------- + +This specification introduces a new decorator function in +the ``typing`` module named ``dataclass_transform``. This decorator +can be applied to either a function that is itself a decorator, +a class, or a metaclass. The presence of +``dataclass_transform`` tells a static type checker that the decorated +function, class, or metaclass performs runtime "magic" that transforms +a class, endowing it with dataclass-like behaviors. + +If ``dataclass_transform`` is applied to a function, using the decorated +function as a decorator is assumed to apply dataclass-like semantics. +If the function has overloads, the ``dataclass_transform`` decorator can +be applied to the implementation of the function or any one, but not more +than one, of the overloads. When applied to an overload, the +``dataclass_transform`` decorator still impacts all usage of the +function. + +If ``dataclass_transform`` is applied to a class, dataclass-like +semantics will be assumed for any class that directly or indirectly +derives from the decorated class or uses the decorated class as a +metaclass. Attributes on the decorated class and its base classes +are not considered to be fields. + +Examples of each approach are shown in the following sections. Each +example creates a ``CustomerModel`` class with dataclass-like semantics. +The implementation of the decorated objects is omitted for brevity, +but we assume that they modify classes in the following ways: + +* They synthesize an ``__init__`` method using data fields declared + within the class and its parent classes. +* They synthesize ``__eq__`` and ``__ne__`` methods. + +Type checkers supporting this PEP will recognize that the +``CustomerModel`` class can be instantiated using the synthesized +``__init__`` method: + +.. code-block:: python + + # Using positional arguments + c1 = CustomerModel(327, "John Smith") + + # Using keyword arguments + c2 = CustomerModel(id=327, name="John Smith") + + # These calls will generate runtime errors and should be flagged as + # errors by a static type checker. + c3 = CustomerModel() + c4 = CustomerModel(327, first_name="John") + c5 = CustomerModel(327, "John Smith", 0) + +Decorator function example +'''''''''''''''''''''''''' + +.. code-block:: python + + _T = TypeVar("_T") + + # The ``create_model`` decorator is defined by a library. + # This could be in a type stub or inline. + @typing.dataclass_transform() + def create_model(cls: Type[_T]) -> Type[_T]: + cls.__init__ = ... + cls.__eq__ = ... + cls.__ne__ = ... + return cls + + # The ``create_model`` decorator can now be used to create new model + # classes, like this: + @create_model + class CustomerModel: + id: int + name: str + +Class example +''''''''''''' + +.. code-block:: python + + # The ``ModelBase`` class is defined by a library. This could be in + # a type stub or inline. + @typing.dataclass_transform() + class ModelBase: ... + + # The ``ModelBase`` class can now be used to create new model + # subclasses, like this: + class CustomerModel(ModelBase): + id: int + name: str + +Metaclass example +''''''''''''''''' + +.. code-block:: python + + # The ``ModelMeta`` metaclass and ``ModelBase`` class are defined by + # a library. This could be in a type stub or inline. + @typing.dataclass_transform() + class ModelMeta(type): ... + + class ModelBase(metaclass=ModelMeta): ... + + # The ``ModelBase`` class can now be used to create new model + # subclasses, like this: + class CustomerModel(ModelBase): + id: int + name: str + +Decorator function and class/metaclass parameters +------------------------------------------------- + +A decorator function, class, or metaclass that provides dataclass-like +functionality may accept parameters that modify certain behaviors. +This specification defines the following parameters that static type +checkers must honor if they are used by a dataclass transform. Each of +these parameters accepts a bool argument, and it must be possible for +the bool value (``True`` or ``False``) to be statically evaluated. + +* ``eq``, ``order``, ``frozen``, ``init`` and ``unsafe_hash`` are parameters + supported in the stdlib dataclass, with meanings defined in + :pep:`PEP 557 <557#id7>`. +* ``kw_only``, ``match_args`` and ``slots`` are parameters supported + in the stdlib dataclass, first introduced in Python 3.10. + +``dataclass_transform`` parameters +---------------------------------- + +Parameters to ``dataclass_transform`` allow for some basic +customization of default behaviors: + +.. code-block:: python + + _T = TypeVar("_T") + + def dataclass_transform( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + field_specifiers: tuple[type | Callable[..., Any], ...] = (), + **kwargs: Any, + ) -> Callable[[_T], _T]: ... + +* ``eq_default`` indicates whether the ``eq`` parameter is assumed to + be True or False if it is omitted by the caller. If not specified, + ``eq_default`` will default to True (the default assumption for + dataclass). +* ``order_default`` indicates whether the ``order`` parameter is + assumed to be True or False if it is omitted by the caller. If not + specified, ``order_default`` will default to False (the default + assumption for dataclass). +* ``kw_only_default`` indicates whether the ``kw_only`` parameter is + assumed to be True or False if it is omitted by the caller. If not + specified, ``kw_only_default`` will default to False (the default + assumption for dataclass). +* ``field_specifiers`` specifies a static list of supported classes + that describe fields. Some libraries also supply functions to + allocate instances of field specifiers, and those functions may + also be specified in this tuple. If not specified, + ``field_specifiers`` will default to an empty tuple (no field + specifiers supported). The standard dataclass behavior supports + only one type of field specifier called ``Field`` plus a helper + function (``field``) that instantiates this class, so if we were + describing the stdlib dataclass behavior, we would provide the + tuple argument ``(dataclasses.Field, dataclasses.field)``. +* ``kwargs`` allows arbitrary additional keyword args to be passed to + ``dataclass_transform``. This gives type checkers the freedom to + support experimental parameters without needing to wait for changes + in ``typing.py``. Type checkers should report errors for any + unrecognized parameters. + +In the future, we may add additional parameters to +``dataclass_transform`` as needed to support common behaviors in user +code. These additions will be made after reaching consensus on +typing-sig rather than via additional PEPs. + +The following sections provide additional examples showing how these +parameters are used. + +Decorator function example +'''''''''''''''''''''''''' + +.. code-block:: python + + # Indicate that the ``create_model`` function assumes keyword-only + # parameters for the synthesized ``__init__`` method unless it is + # invoked with ``kw_only=False``. It always synthesizes order-related + # methods and provides no way to override this behavior. + @typing.dataclass_transform(kw_only_default=True, order_default=True) + def create_model( + *, + frozen: bool = False, + kw_only: bool = True, + ) -> Callable[[Type[_T]], Type[_T]]: ... + + # Example of how this decorator would be used by code that imports + # from this library: + @create_model(frozen=True, kw_only=False) + class CustomerModel: + id: int + name: str + +Class example +''''''''''''' + +.. code-block:: python + + # Indicate that classes that derive from this class default to + # synthesizing comparison methods. + @typing.dataclass_transform(eq_default=True, order_default=True) + class ModelBase: + def __init_subclass__( + cls, + *, + init: bool = True, + frozen: bool = False, + eq: bool = True, + order: bool = True, + ): + ... + + # Example of how this class would be used by code that imports + # from this library: + class CustomerModel( + ModelBase, + init=False, + frozen=True, + eq=False, + order=False, + ): + id: int + name: str + +Metaclass example +''''''''''''''''' + +.. code-block:: python + + # Indicate that classes that use this metaclass default to + # synthesizing comparison methods. + @typing.dataclass_transform(eq_default=True, order_default=True) + class ModelMeta(type): + def __new__( + cls, + name, + bases, + namespace, + *, + init: bool = True, + frozen: bool = False, + eq: bool = True, + order: bool = True, + ): + ... + + class ModelBase(metaclass=ModelMeta): + ... + + # Example of how this class would be used by code that imports + # from this library: + class CustomerModel( + ModelBase, + init=False, + frozen=True, + eq=False, + order=False, + ): + id: int + name: str + + +Field specifiers +----------------- + +Most libraries that support dataclass-like semantics provide one or +more "field specifier" types that allow a class definition to provide +additional metadata about each field in the class. This metadata can +describe, for example, default values, or indicate whether the field +should be included in the synthesized ``__init__`` method. + +Field specifiers can be omitted in cases where additional metadata is +not required: + +.. code-block:: python + + @dataclass + class Employee: + # Field with no specifier + name: str + + # Field that uses field specifier class instance + age: Optional[int] = field(default=None, init=False) + + # Field with type annotation and simple initializer to + # describe default value + is_paid_hourly: bool = True + + # Not a field (but rather a class variable) because type + # annotation is not provided. + office_number = "unassigned" + + +Field specifier parameters +''''''''''''''''''''''''''' + +Libraries that support dataclass-like semantics and support field +specifier classes typically use common parameter names to construct +these field specifiers. This specification formalizes the names and +meanings of the parameters that must be understood for static type +checkers. These standardized parameters must be keyword-only. + +These parameters are a superset of those supported by +``dataclasses.field``, excluding those that do not have an impact on +type checking such as ``compare`` and ``hash``. + +Field specifier classes are allowed to use other +parameters in their constructors, and those parameters can be +positional and may use other names. + +* ``init`` is an optional bool parameter that indicates whether the + field should be included in the synthesized ``__init__`` method. If + unspecified, ``init`` defaults to True. Field specifier functions + can use overloads that implicitly specify the value of ``init`` + using a literal bool value type + (``Literal[False]`` or ``Literal[True]``). +* ``default`` is an optional parameter that provides the default value + for the field. +* ``default_factory`` is an optional parameter that provides a runtime + callback that returns the default value for the field. If neither + ``default`` nor ``default_factory`` are specified, the field is + assumed to have no default value and must be provided a value when + the class is instantiated. +* ``factory`` is an alias for ``default_factory``. Stdlib dataclasses + use the name ``default_factory``, but attrs uses the name ``factory`` + in many scenarios, so this alias is necessary for supporting attrs. +* ``kw_only`` is an optional bool parameter that indicates whether the + field should be marked as keyword-only. If true, the field will be + keyword-only. If false, it will not be keyword-only. If unspecified, + the value of the ``kw_only`` parameter on the object decorated with + ``dataclass_transform`` will be used, or if that is unspecified, the + value of ``kw_only_default`` on ``dataclass_transform`` will be used. +* ``alias`` is an optional str parameter that provides an alternative + name for the field. This alternative name is used in the synthesized + ``__init__`` method. + +It is an error to specify more than one of ``default``, +``default_factory`` and ``factory``. + +This example demonstrates the above: + +.. code-block:: python + + # Library code (within type stub or inline) + # In this library, passing a resolver means that init must be False, + # and the overload with Literal[False] enforces that. + @overload + def model_field( + *, + default: Optional[Any] = ..., + resolver: Callable[[], Any], + init: Literal[False] = False, + ) -> Any: ... + + @overload + def model_field( + *, + default: Optional[Any] = ..., + resolver: None = None, + init: bool = True, + ) -> Any: ... + + @typing.dataclass_transform( + kw_only_default=True, + field_specifiers=(model_field, )) + def create_model( + *, + init: bool = True, + ) -> Callable[[Type[_T]], Type[_T]]: ... + + # Code that imports this library: + @create_model(init=False) + class CustomerModel: + id: int = model_field(resolver=lambda : 0) + name: str + + +Runtime behavior +---------------- + +At runtime, the ``dataclass_transform`` decorator's only effect is to +set an attribute named ``__dataclass_transform__`` on the decorated +function or class to support introspection. The value of the attribute +should be a dict mapping the names of the ``dataclass_transform`` +parameters to their values. + +For example: + +.. code-block:: python + + { + "eq_default": True, + "order_default": False, + "kw_only_default": False, + "field_specifiers": (), + "kwargs": {} + } + + +Dataclass semantics +------------------- + +Except where stated otherwise in this PEP, classes impacted by +``dataclass_transform``, either by inheriting from a class that is +decorated with ``dataclass_transform`` or by being decorated with +a function decorated with ``dataclass_transform``, are assumed to +behave like stdlib ``dataclass``. + +This includes, but is not limited to, the following semantics: + +* Frozen dataclasses cannot inherit from non-frozen dataclasses. A + class that has been decorated with ``dataclass_transform`` is + considered neither frozen nor non-frozen, thus allowing frozen + classes to inherit from it. Similarly, a class that directly + specifies a metaclass that is decorated with ``dataclass_transform`` + is considered neither frozen nor non-frozen. + + Consider these class examples: + + .. code-block:: python + + # ModelBase is not considered either "frozen" or "non-frozen" + # because it is decorated with ``dataclass_transform`` + @typing.dataclass_transform() + class ModelBase(): ... + + # Vehicle is considered non-frozen because it does not specify + # "frozen=True". + class Vehicle(ModelBase): + name: str + + # Car is a frozen class that derives from Vehicle, which is a + # non-frozen class. This is an error. + class Car(Vehicle, frozen=True): + wheel_count: int + + And these similar metaclass examples: + + .. code-block:: python + + @typing.dataclass_transform() + class ModelMeta(type): ... + + # ModelBase is not considered either "frozen" or "non-frozen" + # because it directly specifies ModelMeta as its metaclass. + class ModelBase(metaclass=ModelMeta): ... + + # Vehicle is considered non-frozen because it does not specify + # "frozen=True". + class Vehicle(ModelBase): + name: str + + # Car is a frozen class that derives from Vehicle, which is a + # non-frozen class. This is an error. + class Car(Vehicle, frozen=True): + wheel_count: int + +* Field ordering and inheritance is assumed to follow the rules + specified in :pep:`557 <557#inheritance>`. This includes the effects of + overrides (redefining a field in a child class that has already been + defined in a parent class). + +* :pep:`PEP 557 indicates <557#post-init-parameters>` that + all fields without default values must appear before + fields with default values. Although not explicitly + stated in PEP 557, this rule is ignored when ``init=False``, and + this specification likewise ignores this requirement in that + situation. Likewise, there is no need to enforce this ordering when + keyword-only parameters are used for ``__init__``, so the rule is + not enforced if ``kw_only`` semantics are in effect. + +* As with ``dataclass``, method synthesis is skipped if it would + overwrite a method that is explicitly declared within the class. + Method declarations on base classes do not cause method synthesis to + be skipped. + + For example, if a class declares an ``__init__`` method explicitly, + an ``__init__`` method will not be synthesized for that class. + +* KW_ONLY sentinel values are supported as described in `the Python + docs <#kw-only-docs_>`_ and `bpo-43532 <#kw-only-issue_>`_. + +* ClassVar attributes are not considered dataclass fields and are + `ignored by dataclass mechanisms <#class-var_>`_. + + +Undefined behavior +------------------ + +If multiple ``dataclass_transform`` decorators are found, either on a +single function (including its overloads), a single class, or within a +class hierarchy, the resulting behavior is undefined. Library authors +should avoid these scenarios. + + +Reference Implementation +======================== + +`Pyright <#pyright_>`_ contains the reference implementation of type +checker support for ``dataclass_transform``. Pyright's +``dataClasses.ts`` `source file <#pyright-impl_>`_ would be a good +starting point for understanding the implementation. + +The `attrs <#attrs-usage_>`_ and `pydantic <#pydantic-usage_>`_ +libraries are using ``dataclass_transform`` and serve as real-world +examples of its usage. + + +Rejected Ideas +============== + +``auto_attribs`` parameter +-------------------------- + +The attrs library supports an ``auto_attribs`` parameter that +indicates whether class members decorated with :pep:`526` variable +annotations but with no assignment should be treated as data fields. + +We considered supporting ``auto_attribs`` and a corresponding +``auto_attribs_default`` parameter, but decided against this because it +is specific to attrs. + +Django does not support declaring fields using type annotations only, +so Django users who leverage ``dataclass_transform`` should be aware +that they should always supply assigned values. + +``cmp`` parameter +----------------- + +The attrs library supports a bool parameter ``cmp`` that is equivalent +to setting both ``eq`` and ``order`` to True. We chose not to support +a ``cmp`` parameter, since it only applies to attrs. Users can emulate +the ``cmp`` behaviour by using the ``eq`` and ``order`` parameter names +instead. + +Automatic field name aliasing +----------------------------- + +The attrs library performs `automatic aliasing <#attrs-aliasing_>`_ of +field names that start with a single underscore, stripping the +underscore from the name of the corresponding ``__init__`` parameter. + +This proposal omits that behavior since it is specific to attrs. Users +can manually alias these fields using the ``alias`` parameter. + +Alternate field ordering algorithms +----------------------------------- + +The attrs library currently supports two approaches to ordering the +fields within a class: + +* Dataclass order: The same ordering used by dataclasses. This is the + default behavior of the older APIs (e.g. ``attr.s``). +* Method Resolution Order (MRO): This is the default behavior of the + newer APIs (e.g. define, mutable, frozen). Older APIs (e.g. ``attr.s``) + can opt into this behavior by specifying ``collect_by_mro=True``. + +The resulting field orderings can differ in certain diamond-shaped +multiple inheritance scenarios. + +For simplicity, this proposal does not support any field ordering +other than that used by dataclasses. + +Fields redeclared in subclasses +------------------------------- + +The attrs library differs from stdlib dataclasses in how it +handles inherited fields that are redeclared in subclasses. The +dataclass specification preserves the original order, but attrs +defines a new order based on subclasses. + +For simplicity, we chose to only support the dataclass behavior. +Users of attrs who rely on the attrs-specific ordering will not see +the expected order of parameters in the synthesized ``__init__`` +method. + +Django primary and foreign keys +------------------------------- + +Django applies `additional logic for primary and foreign keys +<#django-ids_>`_. For example, it automatically adds an ``id`` field +(and ``__init__`` parameter) if there is no field designated as a +primary key. + +As this is not broadly applicable to dataclass libraries, this +additional logic is not accommodated with this proposal, so +users of Django would need to explicitly declare the ``id`` field. + +Class-wide default values +------------------------- + +SQLAlchemy requested that we expose a way to specify that the default +value of all fields in the transformed class is ``None``. It is typical +that all SQLAlchemy fields are optional, and ``None`` indicates that +the field is not set. + +We chose not to support this feature, since it is specific to +SQLAlchemy. Users can manually set ``default=None`` on these fields +instead. + +Descriptor-typed field support +------------------------------ + +We considered adding a boolean parameter on ``dataclass_transform`` +to enable better support for fields with descriptor types, which is +common in SQLAlchemy. When enabled, the type of each parameter on the +synthesized ``__init__`` method corresponding to a descriptor-typed +field would be the type of the value parameter to the descriptor's +``__set__`` method rather than the descriptor type itself. Similarly, +when setting the field, the ``__set__`` value type would be expected. +And when getting the value of the field, its type would be expected to +match the return type of ``__get__``. + +This idea was based on the belief that ``dataclass`` did not properly +support descriptor-typed fields. In fact it does, but type checkers +(at least mypy and pyright) did not reflect the runtime behavior which +led to our misunderstanding. For more details, see the +`Pyright bug <#pyright-descriptor-bug_>`__. + +``converter`` field specifier parameter +---------------------------------------- + +The attrs library supports a ``converter`` field specifier parameter, +which is a ``Callable`` that is called by the generated +``__init__`` method to convert the supplied value to some other +desired value. This is tricky to support since the parameter type in +the synthesized ``__init__`` method needs to accept uncovered values, +but the resulting field is typed according to the output of the +converter. + +Some aspects of this issue are detailed in a +`Pyright discussion <#converters_>`_. + +There may be no good way to support this because there's not enough +information to derive the type of the input parameter. One possible +solution would be to add support for a ``converter`` field specifier +parameter but then use the ``Any`` type for the corresponding +parameter in the ``__init__`` method. + + +References +========== +.. _#dataclass-docs: https://docs.python.org/3.11/library/dataclasses.html +.. _#pyright: https://github.com/Microsoft/pyright +.. _#pyright-impl: https://github.com/microsoft/pyright/blob/main/packages/pyright-internal/src/analyzer/dataClasses.ts +.. _#attrs-usage: https://github.com/python-attrs/attrs/pull/796 +.. _#pydantic-usage: https://github.com/samuelcolvin/pydantic/pull/2721 +.. _#attrs-aliasing: https://www.attrs.org/en/stable/init.html#private-attributes +.. _#django-ids: https://docs.djangoproject.com/en/4.0/topics/db/models/#automatic-primary-key-fields +.. _#converters: https://github.com/microsoft/pyright/discussions/1782?sort=old#discussioncomment-653909 +.. _#kw-only-docs: https://docs.python.org/3/library/dataclasses.html#dataclasses.KW_ONLY +.. _#kw-only-issue: https://bugs.python.org/issue43532 +.. _#class-var: https://docs.python.org/3/library/dataclasses.html#class-variables +.. _#pyright-descriptor-bug: https://github.com/microsoft/pyright/issues/3245 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0682.rst b/peps/pep-0682.rst new file mode 100644 index 000000000..a6ecf22e1 --- /dev/null +++ b/peps/pep-0682.rst @@ -0,0 +1,224 @@ +PEP: 682 +Title: Format Specifier for Signed Zero +Author: John Belmonte +Sponsor: Mark Dickinson +PEP-Delegate: Mark Dickinson +Discussions-To: https://discuss.python.org/t/pep-682-format-specifier-for-signed-zero/13596 +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Created: 29-Jan-2022 +Python-Version: 3.11 +Post-History: 08-Feb-2022 +Resolution: https://discuss.python.org/t/accepting-pep-682-format-specifier-for-signed-zero/14088 + + +Abstract +======== + +Though ``float`` and ``Decimal`` types can represent `signed zero`_, in many +fields of mathematics negative zero is surprising or unwanted -- especially +in the context of displaying an (often rounded) numerical result. This PEP +proposes an extension to the `string format specification`_ allowing negative +zero to be normalized to positive zero. + +.. _`signed zero`: https://en.wikipedia.org/wiki/Signed_zero +.. _`string format specification`: https://docs.python.org/3/library/string.html#formatstrings + + +Motivation +========== + +Here is negative zero:: + + >>> x = -0. + >>> x + -0.0 + +When formatting a number, negative zero can result from rounding. Assuming +the user's intention is truly to discard precision, the distinction between +negative and positive zero of the rounded result might be considered an +unwanted artifact:: + + >>> for x in (.002, -.001, .060): + ... print(f'{x: .1f}') + 0.0 + -0.0 + 0.1 + +There are various approaches to clearing the sign of a negative zero. It +can be achieved without a conditional by adding positive zero:: + + >>> x = -0. + >>> x + 0. + 0.0 + +To normalize negative zero when formatting, it is necessary to perform +a redundant (and error-prone) pre-rounding of the input:: + + >>> for x in (.002, -.001, .060): + ... print(f'{round(x, 1) + 0.: .1f}') + 0.0 + 0.0 + 0.1 + +There is ample evidence that, regardless of the language, programmers are +often looking for a way to suppress negative zero, and landing on a +variety of workarounds (pre-round, post-regex, etc.). A sampling: + +* `How to have negative zero always formatted as positive zero in a + python string?`_ (Python, post-regex) +* `(Iron)Python formatting issue with modulo operator & "negative zero"`_ + (Python, pre-round) +* `Negative sign in case of zero in java`_ (Java, post-regex) +* `Prevent small negative numbers printing as "-0"`_ (Objective-C, custom + number formatter) + +What we would like instead is a first-class option to normalize negative +zero, on top of everything else that numerical string formatting already +offers. + +.. _`How to have negative zero always formatted as positive zero in a python string?`: https://stackoverflow.com/questions/11010683/how-to-have-negative-zero-always-formatted-as-positive-zero-in-a-python-string/36604981#36604981 +.. _`(Iron)Python formatting issue with modulo operator & "negative zero"`: https://stackoverflow.com/questions/41564311/ironpython-formatting-issue-with-modulo-operator-negative-zero/41564834#41564834 +.. _`Negative sign in case of zero in java`: https://stackoverflow.com/questions/11929096/negative-sign-in-case-of-zero-in-java +.. _`Prevent small negative numbers printing as "-0"`: https://stackoverflow.com/questions/10969399/prevent-small-negative-numbers-printing-as-0 + + +Rationale +========= + +There are use cases where negative zero is unwanted in formatted number +output -- arguably, not wanting it is more common. Expanding the format +specification is the best way to support this because number formatting +already incorporates rounding, and the normalization of negative zero must +happen after rounding. + +While it is possible to pre-round and normalize a number before formatting, +it's tedious and prone to error if the rounding doesn't precisely match +that of the format spec. Furthermore, functions that wrap formatting would +find themselves having to parse format specs to extract the precision +information. For example, consider how this utility for formatting +one-dimensional numerical arrays would be complicated by such pre-rounding: + +.. code-block:: python + + def format_vector(v, format_spec='8.2f'): + """Format a vector (any iterable) using given per-term format string.""" + return f"[{','.join(f'{term:{format_spec}}' for term in v)}]" + +To date, there doesn't appear to be any other widely-used language or library +providing a formatting option for negative zero. However, the same ``z`` +option syntax and semantics specified below have been `proposed for C++ +std::format()`_. While the proposal was withdrawn for C++20, a consensus +proposal is promised for C++23. (The original `feature request`_ prompting +this PEP was argued without knowledge of the C++ proposal.) + +When Rust developers debated whether to suppress negative zero in ``print`` +output, they took a small `survey of other languages`_. Notably, it didn't +mention any language providing an option for negative zero handling. + +.. _`proposed for C++ std::format()`: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p1496r2.pdf +.. _`feature request`: https://bugs.python.org/issue45995 +.. _`survey of other languages`: https://github.com/rust-lang/rfcs/issues/1074#issuecomment-718243936 + + +Specification +============= + +An optional, literal ``z`` is added to the +`Format Specification Mini-Language`_ following ``sign``: + +.. code-block:: text + + [[fill]align][sign][z][#][0][width][grouping_option][.precision][type] + +where ``z`` is allowed for floating-point presentation types (``f``, ``g``, +etc., as defined by the format specification documentation). Support for +``z`` is provided by the ``.__format__()`` method of each numeric type, +allowing the specifier to be used in f-strings, built-in ``format()``, and +``str.format()``. + +When ``z`` is present, negative zero (whether the original value or the +result of rounding) will be normalized to positive zero. + +Synopsis:: + + >>> x = -.00001 + >>> f'{x:z.1f}' + '0.0' + + >>> x = decimal.Decimal('-.00001') + >>> '{:+z.1f}'.format(x) + '+0.0' + +.. _`Format Specification Mini-Language`: https://docs.python.org/3/library/string.html#format-specification-mini-language + + +Design Notes +------------ +The solution must be opt-in, because we can't change the behavior of +programs that may be expecting or relying on negative zero when formatting +numbers. + +The proposed extension is intentionally ``[sign][z]`` rather than +``[sign[z]]``. The default for ``sign`` (``-``) is not widely known or +explicitly written, so this avoids everyone having to learn it just to use +the ``z`` option. + +While f-strings, built-in ``format()``, and ``str.format()`` can access +the new option, %-formatting cannot. There is already precedent for not +extending %-formatting with new options, as was the case for the +``,`` option (:pep:`378`). + +C99 ``printf`` already uses the ``z`` option character for another +purpose: qualifying the unsigned type (``u``) to match the length of +``size_t``. However, since the signed zero option specifically disallows +``z`` for integer presentation types, it's possible to disambiguate the two +uses, should C want to adopt this new option. + + +Backwards Compatibility +======================= + +The new formatting behavior is opt-in, so numerical formatting of existing +programs will not be affected. + + +How to Teach This +================= +A typical introductory Python course will not cover string formatting +in full detail. For such a course, no adjustments would need to be made. +For a course that does go into details of the string format specification, +a single example demonstrating the effect of the ``z`` option on a negative +value that's rounded to zero by the formatting should be enough. For an +independent developer encountering the feature in someone else's code, +reference to the `Format Specification Mini-Language`_ section of the +library reference manual should suffice. + +.. _`Format Specification Mini-Language`: https://docs.python.org/3/library/string.html#format-specification-mini-language + + +Reference Implementation +======================== + +A reference implementation exists at `pull request #30049`_. + +.. _`pull request #30049`: https://github.com/python/cpython/pull/30049 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/peps/pep-0683.rst b/peps/pep-0683.rst new file mode 100644 index 000000000..6abdc1b46 --- /dev/null +++ b/peps/pep-0683.rst @@ -0,0 +1,920 @@ +PEP: 683 +Title: Immortal Objects, Using a Fixed Refcount +Author: Eric Snow , Eddie Elizondo +Discussions-To: https://discuss.python.org/t/18183 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 10-Feb-2022 +Python-Version: 3.12 +Post-History: `16-Feb-2022 `__, + `19-Feb-2022 `__, + `28-Feb-2022 `__, + `12-Aug-2022 `__, +Resolution: https://discuss.python.org/t/18183/26 + + +PEP Acceptance Conditions +========================= + +The PEP was accepted with conditions: + +* we must apply the primary proposal + in `Solutions for Accidental De-Immortalization`_ + (reset the immortal refcount in ``tp_dealloc()``) +* types without this may not be immortalized (in CPython's code) +* the PEP must be updated with final benchmark results once + the implementation is finalized +* we will have one last round of discussion about those results at that point + + +Abstract +======== + +Currently the CPython runtime maintains a +`small amount of mutable state `_ in the +allocated memory of each object. Because of this, otherwise immutable +objects are actually mutable. This can have a large negative impact +on CPU and memory performance, especially for approaches to increasing +Python's scalability. + +This proposal mandates that, internally, CPython will support marking +an object as one for which that runtime state will no longer change. +Consequently, such an object's refcount will never reach 0, and thus +the object will never be cleaned up (except when the runtime knows +it's safe to do so, like during runtime finalization). +We call these objects "immortal". (Normally, only a relatively small +number of internal objects will ever be immortal.) +The fundamental improvement here is that now an object +can be truly immutable. + +Scope +----- + +Object immortality is meant to be an internal-only feature, so this +proposal does not include any changes to public API or behavior +(with one exception). As usual, we may still add some private +(yet publicly accessible) API to do things like immortalize an object +or tell if one is immortal. Any effort to expose this feature to users +would need to be proposed separately. + +There is one exception to "no change in behavior": refcounting semantics +for immortal objects will differ in some cases from user expectations. +This exception, and the solution, are discussed below. + +Most of this PEP focuses on an internal implementation that satisfies +the above mandate. However, those implementation details are not meant +to be strictly proscriptive. Instead, at the least they are included +to help illustrate the technical considerations required by the mandate. +The actual implementation may deviate somewhat as long as it satisfies +the constraints outlined below. Furthermore, the acceptability of any +specific implementation detail described below does not depend on +the status of this PEP, unless explicitly specified. + +For example, the particular details of: + +* how to mark something as immortal +* how to recognize something as immortal +* which subset of functionally immortal objects are marked as immortal +* which memory-management activities are skipped or modified for immortal objects + +are not only CPython-specific but are also private implementation +details that are expected to change in subsequent versions. + +Implementation Summary +---------------------- + +Here's a high-level look at the implementation: + +If an object's refcount matches a very specific value (defined below) +then that object is treated as immortal. The CPython C-API and runtime +will not modify the refcount (or other runtime state) of an immortal +object. The runtime will now be explicitly responsible for deallocating +all immortal objects during finalization, unless statically allocated. +(See `Object Cleanup`_ below.) + +Aside from the change to refcounting semantics, there is one other +possible negative impact to consider. The threshold for an "acceptable" +performance penalty for immortal objects is 2% (the consensus at the +2022 Language Summit). A naive implementation of the approach described +below makes CPython roughly 4% slower. However, the implementation +is ~performance-neutral~ once known mitigations are applied. + +TODO: Update the performance impact for the latest branch +(both for GCC and for clang). + + +Motivation +========== + +As noted above, currently all objects are effectively mutable. That +includes "immutable" objects like ``str`` instances. This is because +every object's refcount is frequently modified as the object is used +during execution. This is especially significant for a number of +commonly used global (builtin) objects, e.g. ``None``. Such objects +are used a lot, both in Python code and internally. That adds up to +a consistent high volume of refcount changes. + +The effective mutability of all Python objects has a concrete impact +on parts of the Python community, e.g. projects that aim for +scalability like Instagram or the effort to make the GIL +per-interpreter. Below we describe several ways in which refcount +modification has a real negative effect on such projects. +None of that would happen for objects that are truly immutable. + +Reducing CPU Cache Invalidation +------------------------------- + +Every modification of a refcount causes the corresponding CPU cache +line to be invalidated. This has a number of effects. + +For one, the write must be propagated to other cache levels +and to main memory. This has small effect on all Python programs. +Immortal objects would provide a slight relief in that regard. + +On top of that, multi-core applications pay a price. If two threads +(running simultaneously on distinct cores) are interacting with the +same object (e.g. ``None``) then they will end up invalidating each +other's caches with each incref and decref. This is true even for +otherwise immutable objects like ``True``, ``0``, and ``str`` instances. +CPython's GIL helps reduce this effect, since only one thread runs at a +time, but it doesn't completely eliminate the penalty. + +Avoiding Data Races +------------------- + +Speaking of multi-core, we are considering making the GIL +a per-interpreter lock, which would enable true multi-core parallelism. +Among other things, the GIL currently protects against races between +multiple concurrent threads that may incref or decref the same object. +Without a shared GIL, two running interpreters could not safely share +any objects, even otherwise immutable ones like ``None``. + +This means that, to have a per-interpreter GIL, each interpreter must +have its own copy of *every* object. That includes the singletons and +static types. We have a viable strategy for that but it will require +a meaningful amount of extra effort and extra complexity. + +The alternative is to ensure that all shared objects are truly immutable. +There would be no races because there would be no modification. This +is something that the immortality proposed here would enable for +otherwise immutable objects. With immortal objects, +support for a per-interpreter GIL +becomes much simpler. + +Avoiding Copy-on-Write +---------------------- + +For some applications it makes sense to get the application into +a desired initial state and then fork the process for each worker. +This can result in a large performance improvement, especially +memory usage. Several enterprise Python users (e.g. Instagram, +YouTube) have taken advantage of this. However, the above +refcount semantics drastically reduce the benefits and +have led to some sub-optimal workarounds. + +Also note that "fork" isn't the only operating system mechanism +that uses copy-on-write semantics. Another example is ``mmap``. +Any such utility will potentially benefit from fewer copy-on-writes +when immortal objects are involved, when compared to using only +"mortal" objects. + + +Rationale +========= + +The proposed solution is obvious enough that both of this proposal's +authors came to the same conclusion (and implementation, more or less) +independently. The Pyston project `uses a similar approach `_. +Other designs were also considered. Several possibilities have also +been discussed on python-dev in past years. + +Alternatives include: + +* use a high bit to mark "immortal" but do not change ``Py_INCREF()`` +* add an explicit flag to objects +* implement via the type (``tp_dealloc()`` is a no-op) +* track via the object's type object +* track with a separate table + +Each of the above makes objects immortal, but none of them address +the performance penalties from refcount modification described above. + +In the case of per-interpreter GIL, the only realistic alternative +is to move all global objects into ``PyInterpreterState`` and add +one or more lookup functions to access them. Then we'd have to +add some hacks to the C-API to preserve compatibility for the +may objects exposed there. The story is much, much simpler +with immortal objects. + + +Impact +====== + +Benefits +-------- + +Most notably, the cases described in the above examples stand +to benefit greatly from immortal objects. Projects using pre-fork +can drop their workarounds. For the per-interpreter GIL project, +immortal objects greatly simplifies the solution for existing static +types, as well as objects exposed by the public C-API. + +In general, a strong immutability guarantee for objects enables Python +applications to scale better, particularly in +`multi-process deployments `_. This is because they can then +leverage multi-core parallelism without such a significant tradeoff in +memory usage as they now have. The cases we just described, as well as +those described above in `Motivation`_, reflect this improvement. + +Performance +----------- + +A naive implementation shows `a 4% slowdown`_. We have demonstrated +a return to ~performance-neutral~ with a handful of basic mitigations +applied. See the `mitigations`_ section below. + +On the positive side, immortal objects save a significant amount of +memory when used `with a pre-fork model `_. Also, immortal +objects provide opportunities for specialization in the eval loop that +would improve performance. + +.. _a 4% slowdown: https://github.com/python/cpython/pull/19474#issuecomment-1032944709 + +TODO: Update the performance impact for the latest branch +(both for GCC and for clang). + +Backward Compatibility +---------------------- + +Ideally this internal-only feature would be completely compatible. +However, it does involve a change to refcount semantics in some cases. +Only immortal objects are affected, but this includes high-use objects +like ``None``, ``True``, and ``False``. + +Specifically, when an immortal object is involved: + +* code that inspects the refcount will see a really, really large value +* the new noop behavior may break code that: + + * depends specifically on the refcount to always increment or decrement + (or have a specific value from ``Py_SET_REFCNT()``) + * relies on any specific refcount value, other than 0 or 1 + * directly manipulates the refcount to store extra information there + +* in 32-bit pre-3.12 `Stable ABI`_ extensions, + objects may leak due to `Accidental Immortality`_ +* such extensions may crash due to `Accidental De-Immortalizing`_ + +Again, those changes in behavior only apply to immortal objects, +not the vast majority of objects a user will use. Furthermore, +users cannot mark an object as immortal so no user-created objects +will ever have that changed behavior. Users that rely on any of +the changing behavior for global (builtin) objects are already +in trouble. So the overall impact should be small. + +Also note that code which checks for refleaks should keep working fine, +unless it checks for hard-coded small values relative to some immortal +object. The problems noticed by `Pyston`_ shouldn't apply here since +we do not modify the refcount. + +See `Public Refcount Details`_ below for further discussion. + +Accidental Immortality +'''''''''''''''''''''' + +Hypothetically, a non-immortal object could be incref'ed so much +that it reaches the magic value needed to be considered immortal. +That means it would never be decref'ed all the way back to 0, so it +would accidentally leak (never be cleaned up). + +With 64-bit refcounts, this accidental scenario is so unlikely that +we need not worry. Even if done deliberately by using ``Py_INCREF()`` +in a tight loop and each iteration only took 1 CPU cycle, it would take +2^60 cycles (if the immortal bit were 2^60). At a fast 5 GHz that would +still take nearly 250,000,000 seconds (over 2,500 days)! + +Also note that it is doubly unlikely to be a problem because it wouldn't +matter until the refcount would have gotten back to 0 and the object +cleaned up. So any object that hit that magic "immortal" refcount value +would have to be decref'ed that many times again before the change +in behavior would be noticed. + +Again, the only realistic way that the magic refcount would be reached +(and then reversed) is if it were done deliberately. (Of course, the +same thing could be done efficiently using ``Py_SET_REFCNT()`` though +that would be even less of an accident.) At that point we don't +consider it a concern of this proposal. + +On builds with much smaller maximum refcounts, like 32-bit platforms, +the consequences aren't so obvious. Let's say the magic refcount +were 2^30. Using the same specs as above, it would take roughly +4 seconds to accidentally immortalize an object. Under reasonable +conditions, it is still highly unlikely that an object be accidentally +immortalized. It would have to meet these criteria: + +* targeting a non-immortal object (so not one of the high-use builtins) +* the extension increfs without a corresponding decref + (e.g. returns from a function or method) +* no other code decrefs the object in the meantime + +Even at a much less frequent rate it would not take long to reach +accidental immortality (on 32-bit). However, then it would have to run +through the same number of (now noop-ing) decrefs before that one object +would be effectively leaking. This is highly unlikely, especially because +the calculations assume no decrefs. + +Furthermore, this isn't all that different from how such 32-bit extensions +can already incref an object past 2^31 and turn the refcount negative. +If that were an actual problem then we would have heard about it. + +Between all of the above cases, the proposal doesn't consider +accidental immortality a problem. + +Stable ABI +'''''''''' + +The implementation approach described in this PEP is compatible +with extensions compiled to the stable ABI (with the exception +of `Accidental Immortality`_ and `Accidental De-Immortalizing`_). +Due to the nature of the stable ABI, unfortunately, such extensions +use versions of ``Py_INCREF()``, etc. that directly modify the object's +``ob_refcnt`` field. This will invalidate all the performance benefits +of immortal objects. + +However, we do ensure that immortal objects (mostly) stay immortal +in that situation. We set the initial refcount of immortal objects to +a value for which we can identify the object as immortal and which +continues to do so even if the refcount is modified by an extension. +(For example, suppose we used one of the high refcount bits to indicate +that an object was immortal. We would set the initial refcount to a +higher value that still matches the bit, like halfway to the next bit. +See `_Py_IMMORTAL_REFCNT`_.) +At worst, objects in that situation would feel the effects +described in the `Motivation`_ section. Even then +the overall impact is unlikely to be significant. + +Accidental De-Immortalizing +''''''''''''''''''''''''''' + +32-bit builds of older stable ABI extensions can take +`Accidental Immortality`_ to the next level. + +Hypothetically, such an extension could incref an object to a value on +the next highest bit above the magic refcount value. For example, if +the magic value were 2^30 and the initial immortal refcount were thus +2^30 + 2^29 then it would take 2^29 increfs by the extension to reach +a value of 2^31, making the object non-immortal. +(Of course, a refcount that high would probably already cause a crash, +regardless of immortal objects.) + +The more problematic case is where such a 32-bit stable ABI extension +goes crazy decref'ing an already immortal object. Continuing with the +above example, it would take 2^29 asymmetric decrefs to drop below the +magic immortal refcount value. So an object like ``None`` could be +made mortal and subject to decref. That still wouldn't be a problem +until somehow the decrefs continue on that object until it reaches 0. +For statically allocated immortal objects, like ``None``, the extension +would crash the process if it tried to dealloc the object. For any +other immortal objects, the dealloc might be okay. However, there +might be runtime code expecting the formerly-immortal object to be +around forever. That code would probably crash. + +Again, the likelihood of this happening is extremely small, even on +32-bit builds. It would require roughly a billion decrefs on that +one object without a corresponding incref. The most likely scenario is +the following: + +A "new" reference to ``None`` is returned by many functions and methods. +Unlike with non-immortal objects, the 3.12 runtime will basically never +incref ``None`` before giving it to the extension. However, the +extension *will* decref it when done with it (unless it returns it). +Each time that exchange happens with the one object, we get one step +closer to a crash. + +How realistic is it that some form of that exchange (with a single +object) will happen a billion times in the lifetime of a Python process +on 32-bit? If it is a problem, how could it be addressed? + +As to how realistic, the answer isn't clear currently. However, the +mitigation is simple enough that we can safely proceed under the +assumption that it would not be a problem. + +We look at possible solutions +`later on `_. + +Alternate Python Implementations +-------------------------------- + +This proposal is CPython-specific. However, it does relate to the +behavior of the C-API, which may affect other Python implementations. +Consequently, the effect of changed behavior described in +`Backward Compatibility`_ above also applies here (e.g. if another +implementation is tightly coupled to specific refcount values, other +than 0, or on exactly how refcounts change, then they may impacted). + +Security Implications +--------------------- + +This feature has no known impact on security. + +Maintainability +--------------- + +This is not a complex feature so it should not cause much mental +overhead for maintainers. The basic implementation doesn't touch +much code so it should have much impact on maintainability. There +may be some extra complexity due to performance penalty mitigation. +However, that should be limited to where we immortalize all objects +post-init and later explicitly deallocate them during runtime +finalization. The code for this should be relatively concentrated. + + +Specification +============= + +The approach involves these fundamental changes: + +* add `_Py_IMMORTAL_REFCNT`_ (the magic value) to the internal C-API +* update ``Py_INCREF()`` and ``Py_DECREF()`` to no-op for objects + that match the magic refcount +* do the same for any other API that modifies the refcount +* stop modifying ``PyGC_Head`` for immortal GC objects ("containers") +* ensure that all immortal objects are cleaned up during + runtime finalization + +Then setting any object's refcount to ``_Py_IMMORTAL_REFCNT`` +makes it immortal. + +(There are other minor, internal changes which are not described here.) + +In the following sub-sections we dive into the most significant details. +First we will cover some conceptual topics, followed by more concrete +aspects like specific affected APIs. + +Public Refcount Details +----------------------- + +In `Backward Compatibility`_ we introduced possible ways that user code +might be broken by the change in this proposal. Any contributing +misunderstanding by users is likely due in large part to the names of +the refcount-related API and to how the documentation explains those +API (and refcounting in general). + +Between the names and the docs, we can clearly see answers +to the following questions: + +* what behavior do users expect? +* what guarantees do we make? +* do we indicate how to interpret the refcount value they receive? +* what are the use cases under which a user would set an object's + refcount to a specific value? +* are users setting the refcount of objects they did not create? + +As part of this proposal, we must make sure that users can clearly +understand on which parts of the refcount behavior they can rely and +which are considered implementation details. Specifically, they should +use the existing public refcount-related API and the only refcount +values with any meaning are 0 and 1. (Some code relies on 1 as an +indicator that the object can be safely modified.) All other values +are considered "not 0 or 1". + +This information will be clarified +in the `documentation `_. + +Arguably, the existing refcount-related API should be modified to reflect +what we want users to expect. Something like the following: + +* ``Py_INCREF()`` -> ``Py_ACQUIRE_REF()`` (or only support ``Py_NewRef()``) +* ``Py_DECREF()`` -> ``Py_RELEASE_REF()`` +* ``Py_REFCNT()`` -> ``Py_HAS_REFS()`` +* ``Py_SET_REFCNT()`` -> ``Py_RESET_REFS()`` and ``Py_SET_NO_REFS()`` + +However, such a change is not a part of this proposal. It is included +here to demonstrate the tighter focus for user expectations that would +benefit this change. + +Constraints +----------- + +* ensure that otherwise immutable objects can be truly immutable +* minimize performance penalty for normal Python use cases +* be careful when immortalizing objects that we don't actually expect + to persist until runtime finalization. +* be careful when immortalizing objects that are not otherwise immutable +* ``__del__`` and weakrefs must continue working properly + +Regarding "truly" immutable objects, this PEP doesn't impact the +effective immutability of any objects, other than the per-object +runtime state (e.g. refcount). So whether or not some immortal object +is truly (or even effectively) immutable can only be settled separately +from this proposal. For example, str objects are generally considered +immutable, but ``PyUnicodeObject`` holds some lazily cached data. This +PEP has no influence on how that state affects str immutability. + +Immortal Mutable Objects +------------------------ + +Any object can be marked as immortal. We do not propose any +restrictions or checks. However, in practice the value of making an +object immortal relates to its mutability and depends on the likelihood +it would be used for a sufficient portion of the application's lifetime. +Marking a mutable object as immortal can make sense in some situations. + +Many of the use cases for immortal objects center on immutability, so +that threads can safely and efficiently share such objects without +locking. For this reason a mutable object, like a dict or list, would +never be shared (and thus no immortality). However, immortality may +be appropriate if there is sufficient guarantee that the normally +mutable object won't actually be modified. + +On the other hand, some mutable objects will never be shared between +threads (at least not without a lock like the GIL). In some cases it +may be practical to make some of those immortal too. For example, +``sys.modules`` is a per-interpreter dict that we do not expect to +ever get freed until the corresponding interpreter is finalized +(assuming it isn't replaced). By making it immortal, we would +no longer incur the extra overhead during incref/decref. + +We explore this idea further in the `mitigations`_ section below. + +Implicitly Immortal Objects +--------------------------- + +If an immortal object holds a reference to a normal (mortal) object +then that held object is effectively immortal. This is because that +object's refcount can never reach 0 until the immortal object releases +it. + +Examples: + +* containers like ``dict`` and ``list`` +* objects that hold references internally like ``PyTypeObject`` with + its ``tp_subclasses`` and ``tp_weaklist`` +* an object's type (held in ``ob_type``) + +Such held objects are thus implicitly immortal for as long as they are +held. In practice, this should have no real consequences since it +really isn't a change in behavior. The only difference is that the +immortal object (holding the reference) doesn't ever get cleaned up. + +We do not propose that such implicitly immortal objects be changed +in any way. They should not be explicitly marked as immortal just +because they are held by an immortal object. That would provide +no advantage over doing nothing. + +Un-Immortalizing Objects +------------------------ + +This proposal does not include any mechanism for taking an immortal +object and returning it to a "normal" condition. Currently there +is no need for such an ability. + +On top of that, the obvious approach is to simply set the refcount +to a small value. However, at that point there is no way in knowing +which value would be safe. Ideally we'd set it to the value that it +would have been if it hadn't been made immortal. However, that value +will have long been lost. Hence the complexities involved make it less +likely that an object could safely be un-immortalized, even if we +had a good reason to do so. + +_Py_IMMORTAL_REFCNT +------------------- + +We will add two internal constants:: + + _Py_IMMORTAL_BIT - has the top-most available bit set (e.g. 2^62) + _Py_IMMORTAL_REFCNT - has the two top-most available bits set + +The actual top-most bit depends on existing uses for refcount bits, +e.g. the sign bit or some GC uses. We will use the highest bit possible +after consideration of existing uses. + +The refcount for immortal objects will be set to ``_Py_IMMORTAL_REFCNT`` +(meaning the value will be halfway between ``_Py_IMMORTAL_BIT`` and the +value at the next highest bit). However, to check if an object is +immortal we will compare (bitwise-and) its refcount against just +``_Py_IMMORTAL_BIT``. + +The difference means that an immortal object will still be considered +immortal, even if somehow its refcount were modified (e.g. by an older +stable ABI extension). + +Note that top two bits of the refcount are already reserved for other +uses. That's why we are using the third top-most bit. + +The implementation is also open to using other values for the immortal +bit, such as the sign bit or 2^31 (for saturated refcounts on 64-bit). + +Affected API +------------ + +API that will now ignore immortal objects: + +* (public) ``Py_INCREF()`` +* (public) ``Py_DECREF()`` +* (public) ``Py_SET_REFCNT()`` +* (private) ``_Py_NewReference()`` + +API that exposes refcounts (unchanged but may now return large values): + +* (public) ``Py_REFCNT()`` +* (public) ``sys.getrefcount()`` + +(Note that ``_Py_RefTotal``, and consequently ``sys.gettotalrefcount()``, +will not be affected.) + +TODO: clarify the status of ``_Py_RefTotal``. + +Also, immortal objects will not participate in GC. + +Immortal Global Objects +----------------------- + +All runtime-global (builtin) objects will be made immortal. +That includes the following: + +* singletons (``None``, ``True``, ``False``, ``Ellipsis``, ``NotImplemented``) +* all static types (e.g. ``PyLong_Type``, ``PyExc_Exception``) +* all static objects in ``_PyRuntimeState.global_objects`` (e.g. identifiers, + small ints) + +The question of making the full objects actually immutable (e.g. +for per-interpreter GIL) is not in the scope of this PEP. + +Object Cleanup +-------------- + +In order to clean up all immortal objects during runtime finalization, +we must keep track of them. + +For GC objects ("containers") we'll leverage the GC's permanent +generation by pushing all immortalized containers there. During +runtime shutdown, the strategy will be to first let the runtime try +to do its best effort of deallocating these instances normally. Most +of the module deallocation will now be handled by +``pylifecycle.c:finalize_modules()`` where we clean up the remaining +modules as best as we can. It will change which modules are available +during ``__del__``, but that's already explicitly undefined behavior +in the docs. Optionally, we could do some topological ordering +to guarantee that user modules will be deallocated first before +the stdlib modules. Finally, anything left over (if any) can be found +through the permanent generation GC list which we can clear +after ``finalize_modules()`` is done. + +For non-container objects, the tracking approach will vary on a +case-by-case basis. In nearly every case, each such object is directly +accessible on the runtime state, e.g. in a ``_PyRuntimeState`` or +``PyInterpreterState`` field. We may need to add a tracking mechanism +to the runtime state for a small number of objects. + +None of the cleanup will have a significant effect on performance. + +.. _mitigations: + +Performance Regression Mitigations +---------------------------------- + +In the interest of clarity, here are some of the ways we are going +to try to recover some of the `4% performance `_ +we lose with the naive implementation of immortal objects. + +Note that none of this section is actually part of the proposal. + +at the end of runtime init, mark all objects as immortal +'''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +We can apply the concept from +`Immortal Mutable Objects`_ in the pursuit of getting back some of +that 4% performance we lose with the naive implementation of immortal +objects. At the end of runtime init we can mark *all* objects as +immortal and avoid the extra cost in incref/decref. We only need +to worry about immutability with objects that we plan on sharing +between threads without a GIL. + +drop unnecessary hard-coded refcount operations +''''''''''''''''''''''''''''''''''''''''''''''' + +Parts of the C-API interact specifically with objects that we know +to be immortal, like ``Py_RETURN_NONE``. Such functions and macros +can be updated to drop any refcount operations. + +specialize for immortal objects in the eval loop +'''''''''''''''''''''''''''''''''''''''''''''''' + +There are opportunities to optimize operations in the eval loop +involving specific known immortal objects (e.g. ``None``). The +general mechanism is described in :pep:`659`. Also see `Pyston`_. + +other possibilities +''''''''''''''''''' + +* mark every interned string as immortal +* mark the "interned" dict as immortal if shared else share all interned strings +* (Larry,MAL) mark all constants unmarshalled for a module as immortal +* (Larry,MAL) allocate (immutable) immortal objects in their own memory page(s) +* saturated refcounts using the 32 least-significant bits + +Solutions for Accidental De-Immortalization +------------------------------------------- + +In the `Accidental De-Immortalizing`_ section we outlined a possible +negative consequence of immortal objects. Here we look at some +of the options to deal with that. + +Note that we enumerate solutions here to illustrate that satisfactory +options are available, rather than to dictate how the problem will +be solved. + +Also note the following: + +* this only matters in the 32-bit stable-ABI case +* it only affects immortal objects +* there are no user-defined immortal objects, only built-in types +* most immortal objects will be statically allocated + (and thus already must fail if ``tp_dealloc()`` is called) +* only a handful of immortal objects will be used often enough + to possibly face this problem in practice (e.g. ``None``) +* the main problem to solve is crashes coming from ``tp_dealloc()`` + +One fundamental observation for a solution is that we can reset +an immortal object's refcount to ``_Py_IMMORTAL_REFCNT`` +when some condition is met. + +With all that in mind, a simple, yet effective, solution would be +to reset an immortal object's refcount in ``tp_dealloc()``. +``NoneType`` and ``bool`` already have a ``tp_dealloc()`` that calls +``Py_FatalError()`` if triggered. The same goes for other types based +on certain conditions, like ``PyUnicodeObject`` (depending on +``unicode_is_singleton()``), ``PyTupleObject``, and ``PyTypeObject``. +In fact, the same check is important for all statically declared object. +For those types, we would instead reset the refcount. For the +remaining cases we would introduce the check. In all cases, +the overhead of the check in ``tp_dealloc()`` should be too small +to matter. + +Other (less practical) solutions: + +* periodically reset the refcount for immortal objects +* only do that for high-use objects +* only do it if a stable-ABI extension has been imported +* provide a runtime flag for disabling immortality + +(`The discussion thread `__ +has further detail.) + +Regardless of the solution we end up with, we can do something else +later if necessary. + +TODO: Add a note indicating that the implemented solution does not +affect the overall ~performance-neutral~ outcome. + +Documentation +------------- + +The immortal objects behavior and API are internal, implementation +details and will not be added to the documentation. + +However, we will update the documentation to make public guarantees +about refcount behavior more clear. That includes, specifically: + +* ``Py_INCREF()`` - change "Increment the reference count for object o." + to "Indicate taking a new reference to object o." +* ``Py_DECREF()`` - change "Decrement the reference count for object o." + to "Indicate no longer using a previously taken reference to object o." +* similar for ``Py_XINCREF()``, ``Py_XDECREF()``, ``Py_NewRef()``, + ``Py_XNewRef()``, ``Py_Clear()`` +* ``Py_REFCNT()`` - add "The refcounts 0 and 1 have specific meanings + and all others only mean code somewhere is using the object, + regardless of the value. + 0 means the object is not used and will be cleaned up. + 1 means code holds exactly a single reference." +* ``Py_SET_REFCNT()`` - refer to ``Py_REFCNT()`` about how values over 1 + may be substituted with some over value + +We *may* also add a note about immortal objects to the following, +to help reduce any surprise users may have with the change: + +* ``Py_SET_REFCNT()`` (a no-op for immortal objects) +* ``Py_REFCNT()`` (value may be surprisingly large) +* ``sys.getrefcount()`` (value may be surprisingly large) + +Other API that might benefit from such notes are currently undocumented. +We wouldn't add such a note anywhere else (including for ``Py_INCREF()`` +and ``Py_DECREF()``) since the feature is otherwise transparent to users. + + +Reference Implementation +======================== + +The implementation is proposed on GitHub: + +https://github.com/python/cpython/pull/19474 + + +Open Issues +=========== + +* how realistic is the `Accidental De-Immortalizing`_ concern? + + +References +========== + +.. _Pyston: https://mail.python.org/archives/list/python-dev@python.org/message/JLHRTBJGKAENPNZURV4CIJSO6HI62BV3/ + +.. _Facebook: https://www.facebook.com/watch/?v=437636037237097&t=560 + +Prior Art +--------- + +* `Pyston`_ + +Discussions +----------- + +This was discussed in December 2021 on python-dev: + +* https://mail.python.org/archives/list/python-dev@python.org/thread/7O3FUA52QGTVDC6MDAV5WXKNFEDRK5D6/#TBTHSOI2XRWRO6WQOLUW3X7S5DUXFAOV +* https://mail.python.org/archives/list/python-dev@python.org/thread/PNLBJBNIQDMG2YYGPBCTGOKOAVXRBJWY + +Runtime Object State +-------------------- + +Here is the internal state that the CPython runtime keeps +for each Python object: + +* `PyObject.ob_refcnt`_: the object's `refcount `_ +* `_PyGC_Head `_: (optional) the object's node in a list of `"GC" objects `_ +* `_PyObject_HEAD_EXTRA `_: (optional) the object's node in the list of heap objects + +``ob_refcnt`` is part of the memory allocated for every object. +However, ``_PyObject_HEAD_EXTRA`` is allocated only if CPython was built +with ``Py_TRACE_REFS`` defined. ``PyGC_Head`` is allocated only if the +object's type has ``Py_TPFLAGS_HAVE_GC`` set. Typically this is only +container types (e.g. ``list``). Also note that ``PyObject.ob_refcnt`` +and ``_PyObject_HEAD_EXTRA`` are part of ``PyObject_HEAD``. + +.. _PyObject.ob_refcnt: https://github.com/python/cpython/blob/80a9ba537f1f1666a9e6c5eceef4683f86967a1f/Include/object.h#L107 +.. _PyGC_Head: https://github.com/python/cpython/blob/80a9ba537f1f1666a9e6c5eceef4683f86967a1f/Include/internal/pycore_gc.h#L11-L20 +.. _PyObject_HEAD_EXTRA: https://github.com/python/cpython/blob/80a9ba537f1f1666a9e6c5eceef4683f86967a1f/Include/object.h#L68-L72 + +.. _refcounting: + +Reference Counting, with Cyclic Garbage Collection +-------------------------------------------------- + +Garbage collection is a memory management feature of some programming +languages. It means objects are cleaned up (e.g. memory freed) +once they are no longer used. + +Refcounting is one approach to garbage collection. The language runtime +tracks how many references are held to an object. When code takes +ownership of a reference to an object or releases it, the runtime +is notified and it increments or decrements the refcount accordingly. +When the refcount reaches 0, the runtime cleans up the object. + +With CPython, code must explicitly take or release references using +the C-API's ``Py_INCREF()`` and ``Py_DECREF()``. These macros happen +to directly modify the object's refcount (unfortunately, since that +causes ABI compatibility issues if we want to change our garbage +collection scheme). Also, when an object is cleaned up in CPython, +it also releases any references (and resources) it owns +(before it's memory is freed). + +Sometimes objects may be involved in reference cycles, e.g. where +object A holds a reference to object B and object B holds a reference +to object A. Consequently, neither object would ever be cleaned up +even if no other references were held (i.e. a memory leak). The +most common objects involved in cycles are containers. + +CPython has dedicated machinery to deal with reference cycles, which +we call the "cyclic garbage collector", or often just +"garbage collector" or "GC". Don't let the name confuse you. +It only deals with breaking reference cycles. + +See the docs for a more detailed explanation of refcounting +and cyclic garbage collection: + +* https://docs.python.org/3.11/c-api/intro.html#reference-counts +* https://docs.python.org/3.11/c-api/refcounting.html +* https://docs.python.org/3.11/c-api/typeobj.html#c.PyObject.ob_refcnt +* https://docs.python.org/3.11/c-api/gcsupport.html + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/peps/pep-0684.rst b/peps/pep-0684.rst new file mode 100644 index 000000000..65c9e5fa1 --- /dev/null +++ b/peps/pep-0684.rst @@ -0,0 +1,886 @@ +PEP: 684 +Title: A Per-Interpreter GIL +Author: Eric Snow +Discussions-To: https://discuss.python.org/t/pep-684-a-per-interpreter-gil/19583 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Requires: 683 +Created: 08-Mar-2022 +Python-Version: 3.12 +Post-History: `08-Mar-2022 `__, + `29-Sep-2022 `__, + `28-Oct-2022 `__, +Resolution: https://discuss.python.org/t/19583/42 + + +Abstract +======== + +Since Python 1.5 (1997), CPython users can run multiple interpreters +in the same process. However, interpreters in the same process +have always shared a significant +amount of global state. This is a source of bugs, with a growing +impact as more and more people use the feature. Furthermore, +sufficient isolation would facilitate true multi-core parallelism, +where interpreters no longer share the GIL. The changes outlined in +this proposal will result in that level of interpreter isolation. + + +High-Level Summary +================== + +At a high level, this proposal changes CPython in the following ways: + +* stops sharing the GIL between interpreters, given sufficient isolation +* adds several new interpreter config options for isolation settings +* keeps incompatible extensions from causing problems + +The GIL +------- + +The GIL protects concurrent access to most of CPython's runtime state. +So all that GIL-protected global state must move to each interpreter +before the GIL can. + +(In a handful of cases, other mechanisms can be used to ensure +thread-safe sharing instead, such as locks or "immortal" objects.) + +CPython Runtime State +--------------------- + +Properly isolating interpreters requires that most of CPython's +runtime state be stored in the ``PyInterpreterState`` struct. Currently, +only a portion of it is; the rest is found either in C global variables +or in ``_PyRuntimeState``. Most of that will have to be moved. + +This directly coincides with an ongoing effort (of many years) to greatly +reduce internal use of global variables and consolidate the runtime +state into ``_PyRuntimeState`` and ``PyInterpreterState``. +(See `Consolidating Runtime Global State`_ below.) That project has +`significant merit on its own `_ +and has faced little controversy. So, while a per-interpreter GIL +relies on the completion of that effort, that project should not be +considered a part of this proposal--only a dependency. + +Other Isolation Considerations +------------------------------ + +CPython's interpreters must be strictly isolated from each other, with +few exceptions. To a large extent they already are. Each interpreter +has its own copy of all modules, classes, functions, and variables. +The CPython C-API docs `explain further `_. + +.. _caveats: https://docs.python.org/3/c-api/init.html#bugs-and-caveats + +However, aside from what has already been mentioned (e.g. the GIL), +there are a couple of ways in which interpreters still share some state. + +First of all, some process-global resources (e.g. memory, +file descriptors, environment variables) are shared. There are no +plans to change this. + +Second, some isolation is faulty due to bugs or implementations that +did not take multiple interpreters into account. This includes +CPython's runtime and the stdlib, as well as extension modules that +rely on global variables. Bugs should be opened in these cases, +as some already have been. + +Depending on Immortal Objects +----------------------------- + +:pep:`683` introduces immortal objects as a CPython-internal feature. +With immortal objects, we can share any otherwise immutable global +objects between all interpreters. Consequently, this PEP does not +need to address how to deal with the various objects +`exposed in the public C-API `_. +It also simplifies the question of what to do about the builtin +static types. (See `Global Objects`_ below.) + +Both issues have alternate solutions, but everything is simpler with +immortal objects. If PEP 683 is not accepted then this one will be +updated with the alternatives. This lets us reduce noise in this +proposal. + + +Motivation +========== + +The fundamental problem we're solving here is a lack of true multi-core +parallelism (for Python code) in the CPython runtime. The GIL is the +cause. While it usually isn't a problem in practice, at the very least +it makes Python's multi-core story murky, which makes the GIL +a consistent distraction. + +Isolated interpreters are also an effective mechanism to support +certain concurrency models. :pep:`554` discusses this in more detail. + +Indirect Benefits +----------------- + +Most of the effort needed for a per-interpreter GIL has benefits that +make those tasks worth doing anyway: + +* makes multiple-interpreter behavior more reliable +* has led to fixes for long-standing runtime bugs that otherwise + hadn't been prioritized +* has been exposing (and inspiring fixes for) previously unknown runtime bugs +* has driven cleaner runtime initialization (:pep:`432`, :pep:`587`) +* has driven cleaner and more complete runtime finalization +* led to structural layering of the C-API (e.g. ``Include/internal``) +* also see `Benefits to Consolidation`_ below + +.. XXX Add links to example GitHub issues? + +Furthermore, much of that work benefits other CPython-related projects: + +* performance improvements ("`faster-cpython`_") +* pre-fork application deployment (e.g. `Instagram server`_) +* extension module isolation (see :pep:`630`, etc.) +* embedding CPython + +.. _faster-cpython: https://github.com/faster-cpython/ideas + +.. _Instagram server: https://instagram-engineering.com/copy-on-write-friendly-python-garbage-collection-ad6ed5233ddf + +Existing Use of Multiple Interpreters +------------------------------------- + +The C-API for multiple interpreters has been used for many years. +However, until relatively recently the feature wasn't widely known, +nor extensively used (with the exception of mod_wsgi). + +In the last few years use of multiple interpreters has been increasing. +Here are some of the public projects using the feature currently: + +* `mod_wsgi `_ +* `OpenStack Ceph `_ +* `JEP `_ +* `Kodi `_ + +Note that, with :pep:`554`, multiple interpreter usage would likely +grow significantly (via Python code rather than the C-API). + +PEP 554 (Multiple Interpreters in the Stdlib) +--------------------------------------------- + +:pep:`554` is strictly about providing a minimal stdlib module +to give users access to multiple interpreters from Python code. +In fact, it specifically avoids proposing any changes related to +the GIL. Consider, however, that users of that module would benefit +from a per-interpreter GIL, which makes PEP 554 more appealing. + + +Rationale +========= + +During initial investigations in 2014, a variety of possible solutions +for multi-core Python were explored, but each had its drawbacks +without simple solutions: + +* the existing practice of releasing the GIL in extension modules + + * doesn't help with Python code + +* other Python implementations (e.g. Jython, IronPython) + + * CPython dominates the community + +* remove the GIL (e.g. gilectomy, "no-gil") + + * too much technical risk (at the time) + +* Trent Nelson's "PyParallel" project + + * incomplete; Windows-only at the time + +* ``multiprocessing`` + + * too much work to make it effective enough; + high penalties in some situations (at large scale, Windows) + +* other parallelism tools (e.g. dask, ray, MPI) + + * not a fit for the runtime/stdlib + +* give up on multi-core (e.g. async, do nothing) + + * this can only end in tears + +Even in 2014, it was fairly clear that a solution using isolated +interpreters did not have a high level of technical risk and that +most of the work was worth doing anyway. +(The downside was the volume of work to be done.) + + +Specification +============= + +As `summarized above `__, this proposal involves the +following changes, in the order they must happen: + +1. `consolidate global runtime state `_ + (including objects) into ``_PyRuntimeState`` +2. move nearly all of the state down into ``PyInterpreterState`` +3. finally, move the GIL down into ``PyInterpreterState`` +4. everything else + + * update the C-API + * implement extension module restrictions + * work with popular extension maintainers to help + with multi-interpreter support + +Per-Interpreter State +--------------------- + +The following runtime state will be moved to ``PyInterpreterState``: + +* all global objects that are not safely shareable (fully immutable) +* the GIL +* most mutable data that's currently protected by the GIL +* mutable data that's currently protected by some other per-interpreter lock +* mutable data that may be used independently in different interpreters + (also applies to extension modules, including those with multi-phase init) +* all other mutable data not otherwise excluded below + +Furthermore, a portion of the full global state has already been +moved to the interpreter, including GC, warnings, and atexit hooks. + +The following runtime state will not be moved: + +* global objects that are safely shareable, if any +* immutable data, often ``const`` +* effectively immutable data (treated as immutable), for example: + + * some state is initialized early and never modified again + * hashes for strings (``PyUnicodeObject``) are idempotently calculated + when first needed and then cached + +* all data that is guaranteed to be modified exclusively in the main thread, + including: + + * state used only in CPython's ``main()`` + * the REPL's state + * data modified only during runtime init (effectively immutable afterward) + +* mutable data that's protected by some global lock (other than the GIL) +* global state in atomic variables +* mutable global state that can be changed (sensibly) to atomic variables + +Memory Allocators +''''''''''''''''' + +This is one of the most sensitive parts of the work to isolate interpreters. +The simplest solution is to move the global state of the internal +"small block" allocator to ``PyInterpreterState``, as we are doing with +nearly all other runtime state. The following elaborates on the details +and rationale. + +CPython provides a memory management C-API, with `three allocator domains`_: +"raw", "mem", and "object". Each provides the equivalent of ``malloc()``, +``calloc()``, ``realloc()``, and ``free()``. A custom allocator for each +domain can be set during runtime initialization and the current allocator +can be wrapped with a hook using the same API (for example, the stdlib +tracemalloc module). The allocators are currently runtime-global, +shared by all interpreters. + +.. _three allocator domains: https://docs.python.org/3/c-api/memory.html#allocator-domains + +The "raw" allocator is expected to be thread-safe and defaults to glibc's +allocator (``malloc()``, etc.). However, the "mem" and "object" allocators +are not expected to be thread-safe and currently may rely on the GIL for +thread-safety. This is partly because the default allocator for both, +AKA "pyobject", `is not thread-safe`_. This is due to how all state for +that allocator is stored in C global variables. +(See ``Objects/obmalloc.c``.) + +.. _is not thread-safe: https://peps.python.org/pep-0445/#gil-free-pymem-malloc + +Thus we come back to the question of isolating runtime state. In order +for interpreters to stop sharing the GIL, allocator thread-safety +must be addressed. If interpreters continue sharing the allocators +then we need some other way to get thread-safety. Otherwise interpreters +must stop sharing the allocators. In both cases there are a number of +possible solutions, each with potential downsides. + +To keep sharing the allocators, the simplest solution is to use +a granular runtime-global lock around the calls to the "mem" and "object" +allocators in ``PyMem_Malloc()``, ``PyObject_Malloc()``, etc. This would +impact performance, but there are some ways to mitigate that (e.g. only +start locking once the first subinterpreter is created). + +Another way to keep sharing the allocators is to require that the "mem" +and "object" allocators be thread-safe. This would mean we'd have to +make the pyobject allocator implementation thread-safe. That could +even involve re-implementing it using an extensible allocator like +mimalloc. The potential downside is in the cost to re-implement +the allocator and the risk of defects inherent to such an endeavor. + +Regardless, a switch to requiring thread-safe allocators would impact +anyone that embeds CPython and currently sets a thread-unsafe allocator. +We'd need to consider who might be affected and how we reduce any +negative impact (e.g. add a basic C-API to help make an allocator +thread-safe). + +If we did stop sharing the allocators between interpreters, we'd have +to do so only for the "mem" and "object" allocators. We might also need +to keep a full set of global allocators for certain runtime-level usage. +There would be some performance penalty due to looking up the current +interpreter and then pointer indirection to get the allocators. +Embedders would also likely have to provide a new allocator context +for each interpreter. On the plus side, allocator hooks (e.g. tracemalloc) +would not be affected. + +Ultimately, we will go with the simplest option: + +* keep the allocators in the global runtime state +* require that they be thread-safe +* move the state of the default object allocator (AKA "small block" + allocator) to ``PyInterpreterState`` + +We experimented with `a rough implementation`_ and found it was fairly +straightforward, and the performance penalty was essentially zero. + +.. _a rough implementation: https://github.com/ericsnowcurrently/cpython/tree/try-per-interpreter-alloc + +.. _proposed capi: + +C-API +----- + +Internally, the interpreter state will now track how the import system +should handle extension modules which do not support use with multiple +interpreters. See `Restricting Extension Modules`_ below. We'll refer +to that setting here as "PyInterpreterState.strict_extension_compat". + +The following API will be made public, if they haven't been already: + +* ``PyInterpreterConfig`` (struct) +* ``PyInterpreterConfig_INIT`` (macro) +* ``PyInterpreterConfig_LEGACY_INIT`` (macro) +* ``PyThreadState * Py_NewInterpreterFromConfig(PyInterpreterConfig *)`` + +We will add two new fields to ``PyInterpreterConfig``: + +* ``int own_gil`` +* ``int strict_extensions_compat`` + +We may add other fields over time, as needed (e.g. "own_initial_thread"). + +Regarding the initializer macros, ``PyInterpreterConfig_INIT`` would +be used to get an isolated interpreter that also avoids +subinterpreter-unfriendly features. It would be the default for +interpreters created through :pep:`554`. The unrestricted (status quo) +will continue to be available through ``PyInterpreterConfig_LEGACY_INIT``, +which is already used for the main interpreter and ``Py_NewInterpreter()``. +This will not change. + +A note about the "main" interpreter: + +Below, we mention the "main" interpreter several times. This refers +to the interpreter created during runtime initialization, for which +the initial ``PyThreadState`` corresponds to the process's main thread. +It is has a number of unique responsibilities (e.g. handling signals), +as well as a special role during runtime initialization/finalization. +It is also usually (for now) the only interpreter. +(Also see https://docs.python.org/3/c-api/init.html#sub-interpreter-support.) + +PyInterpreterConfig.own_gil +''''''''''''''''''''''''''' + +If ``true`` (``1``) then the new interpreter will have its own "global" +interpreter lock. This means the new interpreter can run without +getting interrupted by other interpreters. This effectively unblocks +full use of multiple cores. That is the fundamental goal of this PEP. + +If ``false`` (``0``) then the new interpreter will use the main +interpreter's lock. This is the legacy (pre-3.12) behavior in CPython, +where all interpreters share a single GIL. Sharing the GIL like this +may be desirable when using extension modules that still depend +on the GIL for thread safety. + +In ``PyInterpreterConfig_INIT``, this will be ``true``. +In ``PyInterpreterConfig_LEGACY_INIT``, this will be ``false``. + +Also, to play it safe, for now we will not allow ``own_gil`` to be true +if a custom allocator was set during runtime init. Wrapping the allocator, +a la tracemalloc, will still be fine. + +PyInterpreterConfig.strict_extensions_compat +'''''''''''''''''''''''''''''''''''''''''''' + +``PyInterpreterConfig.strict_extension_compat`` is basically the initial +value used for "PyInterpreterState.strict_extension_compat". + +Restricting Extension Modules +----------------------------- + +Extension modules have many of the same problems as the runtime when +state is stored in global variables. :pep:`630` covers all the details +of what extensions must do to support isolation, and thus safely run in +multiple interpreters at once. This includes dealing with their globals. + +If an extension implements multi-phase init (see :pep:`489`) it is +considered compatible with multiple interpreters. All other extensions +are considered incompatible. (See `Extension Module Thread Safety`_ +for more details about how a per-interpreter GIL may affect that +classification.) + +If an incompatible extension is imported and the current +"PyInterpreterState.strict_extension_compat" value is ``true`` then the import +system will raise ``ImportError``. (For ``false`` it simply doesn't check.) +This will be done through +``importlib._bootstrap_external.ExtensionFileLoader`` (really, through +``_imp.create_dynamic()``, ``_PyImport_LoadDynamicModuleWithSpec()``, and +``PyModule_FromDefAndSpec2()``). + +Such imports will never fail in the main interpreter (or in interpreters +created through ``Py_NewInterpreter()``) since +"PyInterpreterState.strict_extension_compat" initializes to ``false`` in both +cases. Thus the legacy (pre-3.12) behavior is preserved. + +We will work with popular extensions to help them support use in +multiple interpreters. This may involve adding to CPython's public C-API, +which we will address on a case-by-case basis. + +Extension Module Compatibility +'''''''''''''''''''''''''''''' + +As noted in `Extension Modules`_, many extensions work fine in multiple +interpreters (and under a per-interpreter GIL) without needing any +changes. The import system will still fail if such a module doesn't +explicitly indicate support. At first, not many extension modules +will, so this is a potential source of frustration. + +We will address this by adding a context manager to temporarily disable +the check on multiple interpreter support: +``importlib.util.allow_all_extensions()``. More or less, it will modify +the current "PyInterpreterState.strict_extension_compat" value (e.g. through +a private ``sys`` function). + +Extension Module Thread Safety +'''''''''''''''''''''''''''''' + +If a module supports use with multiple interpreters, that mostly implies +it will work even if those interpreters do not share the GIL. The one +caveat is where a module links against a library with internal global +state that isn't thread-safe. (Even something as innocuous as a static +local variable as a temporary buffer can be a problem.) With a shared +GIL, that state is protected. Without one, such modules must wrap any +use of that state (e.g. through calls) with a lock. + +Currently, it isn't clear whether or not supports-multiple-interpreters +is sufficiently equivalent to supports-per-interpreter-gil, such that +we can avoid any special accommodations. This is still a point of +meaningful discussion and investigation. The practical distinction +between the two (in the Python community, e.g. PyPI) is not yet +understood well enough to settle the matter. Likewise, it isn't clear +what we might be able to do to help extension maintainers mitigate +the problem (assuming it is one). + +In the meantime, we must proceed as though the difference would be +large enough to cause problems for enough extension modules out there. +The solution we would apply is: + +* add a ``PyModuleDef`` slot that indicates an extension can be imported + under a per-interpreter GIL (i.e. opt in) +* add that slot as part of the definition of a "compatible" extension, + as discussed earlier + +The downside is that not a single extension module will be able to take +advantage of the per-interpreter GIL without extra effort by the module +maintainer, regardless of how minor that effort. This compounds the +problem described in `Extension Module Compatibility`_ and the same +workaround applies. Ideally, we would determine that there isn't enough +difference to matter. + +If we do end up requiring an opt-in for imports under a per-interpreter +GIL, and later determine it isn't necessary, then we can switch the +default at that point, make the old opt-in slot a noop, and add a new +``PyModuleDef`` slot for explicitly opting *out*. In fact, it makes +sense to add that opt-out slot from the beginning. + + +Documentation +------------- + +* C-API: the "Sub-interpreter support" section of ``Doc/c-api/init.rst`` + will detail the updated API +* C-API: that section will explain about the consequences of + a per-interpreter GIL +* importlib: the ``ExtensionFileLoader`` entry will note import + may fail in subinterpreters +* importlib: there will be a new entry about + ``importlib.util.allow_all_extensions()`` + + +Impact +====== + +Backwards Compatibility +----------------------- + +No behavior or APIs are intended to change due to this proposal, +with two exceptions: + +* some extensions will fail to import in some subinterpreters + (see `the next section `_) +* "mem" and "object" allocators that are currently not thread-safe + may now be susceptible to data races when used in combination + with multiple interpreters + +The existing C-API for managing interpreters will preserve its current +behavior, with new behavior exposed through new API. No other API +or runtime behavior is meant to change, including compatibility with +the stable ABI. + +See `Objects Exposed in the C-API`_ below for related discussion. + +Extension Modules +''''''''''''''''' + +Currently the most common usage of Python, by far, is with the main +interpreter running by itself. This proposal has zero impact on +extension modules in that scenario. Likewise, for better or worse, +there is no change in behavior under multiple interpreters created +using the existing ``Py_NewInterpreter()``. + +Keep in mind that some extensions already break when used in multiple +interpreters, due to keeping module state in global variables (or +due to the `internal state of linked libraries`_). They +may crash or, worse, experience inconsistent behavior. That was part +of the motivation for :pep:`630` and friends, so this is not a new +situation nor a consequence of this proposal. + +.. _internal state of linked libraries: https://github.com/pyca/cryptography/issues/2299 + +In contrast, when the `proposed API `_ is used to +create multiple interpreters, with the appropriate settings, +the behavior will change for incompatible extensions. In that case, +importing such an extension will fail (outside the main interpreter), +as explained in `Restricting Extension Modules`_. For extensions that +already break in multiple interpreters, this will be an improvement. + +Additionally, some extension modules link against libraries with +thread-unsafe internal global state. +(See `Extension Module Thread Safety`_.) +Such modules will have to start wrapping any direct or indirect use +of that state in a lock. This is the key difference from other modules +that also implement multi-phase init and thus indicate support for +multiple interpreters (i.e. isolation). + +Now we get to the break in compatibility mentioned above. Some +extensions are safe under multiple interpreters (and a per-interpreter +GIL), even though they haven't indicated that. Unfortunately, there is +no reliable way for the import system to infer that such an extension +is safe, so importing them will still fail. This case is addressed +in `Extension Module Compatibility`_ above. + +Extension Module Maintainers +---------------------------- + +One related consideration is that a per-interpreter GIL will likely +drive increased use of multiple interpreters, particularly if :pep:`554` +is accepted. Some maintainers of large extension modules have expressed +concern about the increased burden they anticipate due to increased +use of multiple interpreters. + +Specifically, enabling support for multiple interpreters will require +substantial work for some extension modules (albeit likely not many). +To add that support, the maintainer(s) of such a module (often +volunteers) would have to set aside their normal priorities and +interests to focus on compatibility (see :pep:`630`). + +Of course, extension maintainers are free to not add support for use +in multiple interpreters. However, users will increasingly demand +such support, especially if the feature grows in popularity. + +Either way, the situation can be stressful for maintainers of such +extensions, particularly when they are doing the work in their spare +time. The concerns they have expressed are understandable, and we address +the partial solution in the `Restricting Extension Modules`_ and +`Extension Module Compatibility`_ sections. + +Alternate Python Implementations +-------------------------------- + +Other Python implementation are not required to provide support for +multiple interpreters in the same process (though some do already). + +Security Implications +--------------------- + +There is no known impact to security with this proposal. + +Maintainability +--------------- + +On the one hand, this proposal has already motivated a number of +improvements that make CPython *more* maintainable. That is expected +to continue. On the other hand, the underlying work has already +exposed various pre-existing defects in the runtime that have had +to be fixed. That is also expected to continue as multiple interpreters +receive more use. Otherwise, there shouldn't be a significant impact +on maintainability, so the net effect should be positive. + +Performance +----------- + +The work to consolidate globals has already provided a number of +improvements to CPython's performance, both speeding it up and using +less memory, and this should continue. The performance benefits of a +per-interpreter GIL specifically have not been explored. At the very +least, it is not expected to make CPython slower +(as long as interpreters are sufficiently isolated). And, obviously, +it enable a variety of multi-core parallelism in Python code. + + +How to Teach This +================= + +Unlike :pep:`554`, this is an advanced feature meant for a narrow set +of users of the C-API. There is no expectation that the specifics of +the API nor its direct application will be taught. + +That said, if it were taught then it would boil down to the following: + + In addition to Py_NewInterpreter(), you can use + Py_NewInterpreterFromConfig() to create an interpreter. + The config you pass it indicates how you want that + interpreter to behave. + +Furthermore, the maintainers of any extension modules that create +isolated interpreters will likely need to explain the consequences +of a per-interpreter GIL to their users. The first thing to explain +is what :pep:`554` teaches about the concurrency model that isolated +interpreters enables. That leads into the point that Python software +written using that concurrency model can then take advantage +of multi-core parallelism, which is currently +prevented by the GIL. + +.. XXX We should add docs (a la PEP 630) that spell out how to make + an extension compatible with per-interpreter GIL. + + +Reference Implementation +======================== + + + + +Open Issues +=========== + +* Are we okay to require "mem" and "object" allocators to be thread-safe? +* How would a per-interpreter tracemalloc module relate to global allocators? +* Would the faulthandler module be limited to the main interpreter + (like the signal module) or would we leak that global state between + interpreters (protected by a granular lock)? +* Split out an informational PEP with all the relevant info, + based on the "Consolidating Runtime Global State" section? +* How likely is it that a module works under multiple interpreters + (isolation) but doesn't work under a per-interpreter GIL? + (See `Extension Module Thread Safety`_.) +* If it is likely enough, what can we do to help extension maintainers + mitigate the problem and enjoy use under a per-interpreter GIL? +* What would be a better (scarier-sounding) name + for ``allow_all_extensions``? + + +Deferred Functionality +====================== + +* ``PyInterpreterConfig`` option to always run the interpreter in a new thread +* ``PyInterpreterConfig`` option to assign a "main" thread to the interpreter + and only run in that thread + + +Rejected Ideas +============== + + + + +Extra Context +============= + +Sharing Global Objects +---------------------- + +We are sharing some global objects between interpreters. +This is an implementation detail and relates more to +`globals consolidation `_ +than to this proposal, but it is a significant enough detail +to explain here. + +The alternative is to share no objects between interpreters, ever. +To accomplish that, we'd have to sort out the fate of all our static +types, as well as deal with compatibility issues for the many objects +`exposed in the public C-API `_. + +That approach introduces a meaningful amount of extra complexity +and higher risk, though prototyping has demonstrated valid solutions. +Also, it would likely result in a performance penalty. + +`Immortal objects `_ allow us to +share the otherwise immutable global objects. That way we avoid +the extra costs. + +.. _capi objects: + +Objects Exposed in the C-API +'''''''''''''''''''''''''''' + +The C-API (including the limited API) exposes all the builtin types, +including the builtin exceptions, as well as the builtin singletons. +The exceptions are exposed as ``PyObject *`` but the rest are exposed +as the static values rather than pointers. This was one of the few +non-trivial problems we had to solve for per-interpreter GIL. + +With immortal objects this is a non-issue. + + +Consolidating Runtime Global State +---------------------------------- + +As noted in `CPython Runtime State`_ above, there is an active effort +(separate from this PEP) to consolidate CPython's global state into the +``_PyRuntimeState`` struct. Nearly all the work involves moving that +state from global variables. The project is particularly relevant to +this proposal, so below is some extra detail. + +Benefits to Consolidation +''''''''''''''''''''''''' + +Consolidating the globals has a variety of benefits: + +* greatly reduces the number of C globals (best practice for C code) +* the move draws attention to runtime state that is unstable or broken +* encourages more consistency in how runtime state is used +* makes it easier to discover/identify CPython's runtime state +* makes it easier to statically allocate runtime state in a consistent way +* better memory locality for runtime state + +Furthermore all the benefits listed in `Indirect Benefits`_ above also +apply here, and the same projects listed there benefit. + +Scale of Work +''''''''''''' + +The number of global variables to be moved is large enough to matter, +but most are Python objects that can be dealt with in large groups +(like ``Py_IDENTIFIER``). In nearly all cases, moving these globals +to the interpreter is highly mechanical. That doesn't require +cleverness but instead requires someone to put in the time. + +State To Be Moved +''''''''''''''''' + +The remaining global variables can be categorized as follows: + +* global objects + + * static types (incl. exception types) + * non-static types (incl. heap types, structseq types) + * singletons (static) + * singletons (initialized once) + * cached objects + +* non-objects + + * will not (or unlikely to) change after init + * only used in the main thread + * initialized lazily + * pre-allocated buffers + * state + +Those globals are spread between the core runtime, the builtin modules, +and the stdlib extension modules. + +For a breakdown of the remaining globals, run: + +.. code-block:: bash + + ./python Tools/c-analyzer/table-file.py Tools/c-analyzer/cpython/globals-to-fix.tsv + +Already Completed Work +'''''''''''''''''''''' + +As mentioned, this work has been going on for many years. Here are some +of the things that have already been done: + +* cleanup of runtime initialization (see :pep:`432` / :pep:`587`) +* extension module isolation machinery (see :pep:`384` / :pep:`3121` / :pep:`489`) +* isolation for many builtin modules +* isolation for many stdlib extension modules +* addition of ``_PyRuntimeState`` +* no more ``_Py_IDENTIFIER()`` +* statically allocated: + + * empty string + * string literals + * identifiers + * latin-1 strings + * length-1 bytes + * empty tuple + +Tooling +''''''' + +As already indicated, there are several tools to help identify the +globals and reason about them. + +* ``Tools/c-analyzer/cpython/globals-to-fix.tsv`` - the list of remaining globals +* ``Tools/c-analyzer/c-analyzer.py`` + + * ``analyze`` - identify all the globals + * ``check`` - fail if there are any unsupported globals that aren't ignored + +* ``Tools/c-analyzer/table-file.py`` - summarize the known globals + +Also, the check for unsupported globals is incorporated into CI so that +no new globals are accidentally added. + +Global Objects +'''''''''''''' + +Global objects that are safe to be shared (without a GIL) between +interpreters can stay on ``_PyRuntimeState``. Not only must the object +be effectively immutable (e.g. singletons, strings), but not even the +refcount can change for it to be safe. Immortality (:pep:`683`) +provides that. (The alternative is that no objects are shared, which +adds significant complexity to the solution, particularly for the +objects `exposed in the public C-API `_.) + +Builtin static types are a special case of global objects that will be +shared. They are effectively immutable except for one part: +``__subclasses__`` (AKA ``tp_subclasses``). We expect that nothing +else on a builtin type will change, even the content +of ``__dict__`` (AKA ``tp_dict``). + +``__subclasses__`` for the builtin types will be dealt with by making +it a getter that does a lookup on the current ``PyInterpreterState`` +for that type. + + +References +========== + +Related: + +* :pep:`384` "Defining a Stable ABI" +* :pep:`432` "Restructuring the CPython startup sequence" +* :pep:`489` "Multi-phase extension module initialization" +* :pep:`554` "Multiple Interpreters in the Stdlib" +* :pep:`573` "Module State Access from C Extension Methods" +* :pep:`587` "Python Initialization Configuration" +* :pep:`630` "Isolating Extension Modules" +* :pep:`683` "Immortal Objects, Using a Fixed Refcount" +* :pep:`3121` "Extension Module Initialization and Finalization" + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0685.rst b/peps/pep-0685.rst new file mode 100644 index 000000000..04d1f910a --- /dev/null +++ b/peps/pep-0685.rst @@ -0,0 +1,213 @@ +PEP: 685 +Title: Comparison of extra names for optional distribution dependencies +Author: Brett Cannon +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/14141 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 08-Mar-2022 +Post-History: `08-Mar-2022 `__ +Resolution: https://discuss.python.org/t/pep-685-comparison-of-extra-names-for-optional-distribution-dependencies/14141/55 + + +.. canonical-pypa-spec:: + + +Abstract +======== + +This PEP specifies how to normalize `distribution extra `_ +names when performing comparisons. +This prevents tools from either failing to find an extra name, or +accidentally matching against an unexpected name. + + +Motivation +========== + +The `Provides-Extra`_ core metadata specification states that an extra's +name "must be a valid Python identifier". +:pep:`508` specifies that the value of an ``extra`` marker may contain a +letter, digit, or any one of ``.``, ``-``, or ``_`` after the initial character. +There is no other `PyPA specification +`_ +which outlines how extra names should be written or normalized for comparison. +Due to the amount of packaging-related code in existence, +it is important to evaluate current practices by the community and +standardize on one that doesn't break most existing code, while being +something tool authors can agree to following. + +The issue of there being no consistent standard was brought forward by an +`initial discussion `__ +noting that the extra ``adhoc-ssl`` was not considered equal to the name +``adhoc_ssl`` by pip 22. + + +Rationale +========= + +:pep:`503` specifies how to normalize distribution names:: + + re.sub(r"[-_.]+", "-", name).lower() + +This collapses any run of the characters ``-``, ``_`` and ``.`` +down to a single ``-``. +For example, ``---`` ``.`` and ``__`` all get converted to just ``-``. +This does **not** produce a valid Python identifier, per +the core metadata 2.2 specification for extra names. + +`Setuptools 60 performs normalization `__ +via:: + + re.sub(r'[^A-Za-z0-9-.]+', '_', name).lower() + +The use of an underscore/``_`` differs from PEP 503's use of a hyphen/``-``, +and it also normalizes characters outside of those allowed by :pep:`508`. +Runs of ``.`` and ``-``, unlike PEP 503, do **not** get normalized to one ``_``, +e.g. ``..`` stays the same. To note, this is inconsistent with this function's +docstring, which *does* specify that all non-alphanumeric characters +(which would include ``-`` and ``.``) are normalized and collapsed. + +For pip 22, its +"extra normalisation behaviour is quite convoluted and erratic" [pip-erratic]_ +and so its use is not considered. + +.. [pip-erratic] Tzu-ping Chung on Python Discourse ` +for names:: + + re.sub(r"[-_.]+", "-", name).lower() + +The `core metadata`_ specification will be updated such that the allowed +names for `Provides-Extra`_ matches what :pep:`508` specifies for names. +This will bring extra naming in line with that of the Name_ field. +Because this changes what is considered valid, it will lead to a core +metadata version increase to ``2.3``. + +For tools writing `core metadata`_, +they MUST write out extra names in their normalized form. +This applies to the `Provides-Extra`_ field and the +:pep:`extra marker <0508#extras>` when used in the `Requires-Dist`_ field. + +Tools generating metadata MUST raise an error if a user specified +two or more extra names which would normalize to the same name. +Tools generating metadata MUST raise an error if an invalid extra +name is provided as appropriate for the specified core metadata version. +If a project's metadata specifies an older core metadata version and +the name would be invalid with newer core metadata versions, +tools reading that metadata SHOULD warn the user. +Tools SHOULD warn users when an invalid extra name is read and SHOULD +ignore the name to avoid ambiguity. +Tools MAY raise an error instead of a warning when reading an +invalid name, if they so desire. + + +Backwards Compatibility +======================= + +Moving to :pep:`503` normalization and :pep:`508` name acceptance +allows for all preexisting, valid names to continue to be valid. + +Based on research looking at a collection of wheels on PyPI [pypi-results]_, +the risk of extra name clashes is limited to 73 instances when considering +all extras names on PyPI, valid or not (not just those within a single package) +while *only* looking at valid names leads to only 3 clashes: + +* ``dev-test``: ``dev_test``, ``dev-test``, ``dev.test`` +* ``dev-lint``: ``dev-lint``, ``dev.lint``, ``dev_lint`` +* ``apache-beam``: ``apache-beam``, ``apache.beam`` + +By requiring tools writing core metadata to only record the normalized name, +the issue of preexisting, invalid extra names should diminish over time. + +.. [pypi-results] Paul Moore on Python Discourse https://discuss.python.org/t/14141/17 + + +Security Implications +===================== + +It is possible that for a distribution that has conflicting extra names, a +tool ends up installing dependencies that somehow weaken the security +of the system. +This is only hypothetical and if it were to occur, +it would probably be more of a security concern for the distributions +specifying such extras names rather than the distribution that pulled +them in together. + + +How to Teach This +================= + +This should be transparent to users on a day-to-day basis. +It will be up to tools to educate/stop users when they select extra +names which conflict. + + +Reference Implementation +======================== + +No reference implementation is provided aside from the code above, +but the expectation is the `packaging project`_ will provide a +function in its ``packaging.utils`` module that will implement extra name +normalization. +It will also implement extra name comparisons appropriately. +Finally, if the project ever gains the ability to write out metadata, +it will also implement this PEP. + + +Transition Plan +=============== + +There is a risk that a build tool will produce core metadata +conforming to version 2.3 and thus this PEP but which is consumed by a +tool that is unaware of this PEP (if that tool chooses to attempt to +read a core metadata version it does not directly support). +In such a case there is a chance that a user may specify an extra +using an non-normalized name which worked previously but which fails +now. + +As such, consumers of this PEP should be prioritized more than +producers so that users can be notified that they are specifying extra +names which are not normalized (and thus may break in the future). + + +Rejected Ideas +============== + +Using setuptools 60's normalization +----------------------------------- + +Initially, this PEP proposed using setuptools ``safe_extra()`` for normalization +to try to minimize backwards-compatibility issues. +However, after checking various wheels on PyPI, +it became clear that standardizing **all** naming on :pep:`508` and +:pep:`503` semantics was easier and better long-term, +while causing minimal backwards compatibility issues. + + +Open Issues +=========== + +N/A + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. _core metadata: https://packaging.python.org/en/latest/specifications/core-metadata/ +.. _Name: https://packaging.python.org/en/latest/specifications/core-metadata/#name +.. _packaging project: https://packaging.pypa.io +.. _Provides-Extra: https://packaging.python.org/en/latest/specifications/core-metadata/#provides-extra-multiple-use +.. _Requires-Dist: https://packaging.python.org/en/latest/specifications/core-metadata/#requires-dist-multiple-use diff --git a/peps/pep-0686.rst b/peps/pep-0686.rst new file mode 100644 index 000000000..301813c52 --- /dev/null +++ b/peps/pep-0686.rst @@ -0,0 +1,195 @@ +PEP: 686 +Title: Make UTF-8 mode default +Author: Inada Naoki +Discussions-To: https://discuss.python.org/t/14737 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 18-Mar-2022 +Python-Version: 3.15 +Post-History: `18-Mar-2022 `__, + `31-Mar-2022 `__ +Resolution: https://discuss.python.org/t/14737/9 + + +Abstract +======== + +This PEP proposes enabling :pep:`UTF-8 mode <540>` by default. + +With this change, Python consistently uses UTF-8 for default encoding of +files, stdio, and pipes. + + +Motivation +========== + +UTF-8 becomes de facto standard text encoding. + +* The default encoding of Python source files is UTF-8. +* JSON, TOML, YAML use UTF-8. +* Most text editors, including Visual Studio Code and Windows Notepad use + UTF-8 by default. +* Most websites and text data on the internet use UTF-8. +* And many other popular programming languages, including Node.js, Go, Rust, + and Java uses UTF-8 by default. + +Changing the default encoding to UTF-8 makes it easier for Python to +interoperate with them. + +Additionally, many Python developers using Unix forget that the default +encoding is platform dependent. +They omit to specify ``encoding="utf-8"`` when they read text files encoded +in UTF-8 (e.g. JSON, TOML, Markdown, and Python source files). +Inconsistent default encoding causes many bugs. + + +Specification +============= + +Enable UTF-8 mode by default +---------------------------- + +Python will enable UTF-8 mode by default from Python 3.15. + +Users can still disable UTF-8 mode by setting ``PYTHONUTF8=0`` or +``-X utf8=0``. + + +``locale.getencoding()`` +------------------------ + +Since UTF-8 mode affects ``locale.getpreferredencoding(False)``, +we need an API to get locale encoding regardless of UTF-8 mode. + +``locale.getencoding()`` will be added for this purpose. +It returns locale encoding too, but ignores UTF-8 mode. + +When ``warn_default_encoding`` option is specified, +``locale.getpreferredencoding()`` will emit ``EncodingWarning`` like +``open()`` (see also :pep:`597`). + +This API was added in Python 3.11. + + +Fixing ``encoding="locale"`` option +----------------------------------- + +:pep:`597` added the ``encoding="locale"`` option to the ``TextIOWrapper``. +This option is used to specify the locale encoding explicitly. +``TextIOWrapper`` should use locale encoding when the option is specified, +regardless of default text encoding. + +But ``TextIOWrapper`` uses ``"UTF-8"`` in UTF-8 mode even if +``encoding="locale"`` is specified for now. +This behavior is inconsistent with the :pep:`597` motivation. +It is because we didn't expect making UTF-8 mode default when Python +changes its default text encoding. + +This inconsistency should be fixed before making UTF-8 mode default. +``TextIOWrapper`` should use locale encoding when ``encoding="locale"`` is +passed even in UTF-8 mode. + +This issue was fixed in Python 3.11. + + +Backward Compatibility +====================== + +Most Unix systems use UTF-8 locale and Python enables UTF-8 mode when its +locale is C or POSIX. +So this change mostly affects Windows users. + +When a Python program depends on the default encoding, this change may cause +``UnicodeError``, mojibake, or even silent data corruption. +So this change should be announced loudly. + +This is the guideline to fix this backward compatibility issue: + +1. Disable UTF-8 mode. +2. Use ``EncodingWarning`` (:pep:`597`) to find every places UTF-8 mode + affects. + + * If ``encoding`` option is omitted, consider using ``encoding="utf-8"`` + or ``encoding="locale"``. + * If ``locale.getpreferredencoding()`` is used, consider using + ``"utf-8"`` or ``locale.getencoding()``. + +3. Test the application with UTF-8 mode. + + +Preceding examples +================== + +* Ruby `changed `__ the default ``external_encoding`` + to UTF-8 on Windows in Ruby 3.0 (2020). +* Java `changed `__ the default text encoding + to UTF-8 in JDK 18. (2022). + +Both Ruby and Java have an option for backward compatibility. +They don't provide any warning like :pep:`597`'s ``EncodingWarning`` +in Python for use of the default encoding. + + +Rejected Alternative +==================== + +Deprecate implicit encoding +--------------------------- + +Deprecating the use of the default encoding is considered. + +But there are many cases that the default encoding is used for reading/writing +only ASCII text. +Additionally, such warnings are not useful for non-cross platform applications +run on Unix. + +So forcing users to specify the ``encoding`` everywhere is too painful. +Emitting a lot of ``DeprecationWarning`` will lead users ignore warnings. + +:pep:`387` requires adding a warning for backward incompatible changes. +But it doesn't require using ``DeprecationWarning``. +So using optional ``EncodingWarning`` doesn't violate the :pep:`387`. + +Java also rejected this idea in `JEP 400`_. + + +Use ``PYTHONIOENCODING`` for PIPEs +---------------------------------- + +To ease backward compatibility issue, using ``PYTHONIOENCODING`` as the +default encoding of PIPEs in the ``subprocess`` module is considered. + +With this idea, users can use legacy encoding for +``subprocess.Popen(text=True)`` even in UTF-8 mode. + +But this idea makes "default encoding" complicated. +And this idea is also backward incompatible. + +So this idea is rejected. Users can disable UTF-8 mode until they replace +``text=True`` with ``encoding="utf-8"`` or ``encoding="locale"``. + + +How to teach this +================= + +For new users, this change reduces things that need to teach. +Users don't need to learn about text encoding in their first year. +They should learn it when they need to use non-UTF-8 text files. + +For existing users, see the `Backward compatibility`_ section. + + +References +========== + +.. _Feature #16604: https://bugs.ruby-lang.org/issues/16604 + +.. _JEP 400: https://openjdk.java.net/jeps/400 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0687.rst b/peps/pep-0687.rst new file mode 100644 index 000000000..6af4d25de --- /dev/null +++ b/peps/pep-0687.rst @@ -0,0 +1,206 @@ +PEP: 687 +Title: Isolating modules in the standard library +Author: Erlend Egeberg Aasland , Petr Viktorin +Discussions-To: https://discuss.python.org/t/14824 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Requires: 489, 573, 630 +Created: 04-Apr-2022 +Python-Version: 3.12 +Post-History: `04-Apr-2022 `__, + `11-Apr-2022 `__ +Resolution: https://discuss.python.org/t/14824/4 + +Abstract +======== + +Extensions in the standard library will be converted to multi-phase +initialization (:pep:`489`) and where possible, all state will be +stored on module objects rather than in process-global variables. + + +Note on Backdating +================== + +Much of this proposal has already been implemented. +We submit this PEP to explain the changes, seek consensus on +whether they are good, propose the remaining changes, +and set best practices for new modules. + + + +Motivation & Rationale +====================== + +The informational :pep:`630` describes the background, motivation, rationale, +implications and implementation notes of the proposed changes as they apply +generally to any extension module (not just the standard library). + +It is an integral part of this proposal. Read it first. + +This PEP discusses specifics of the standard library. + + +Specification +============= + +The body of :pep:`630` will be converted to a HOWTO in the Python +documentation, and that PEP will be retired (marked Final). + +All extension modules in the standard library will be converted to multi-phase +initialization introduced in :pep:`489`. + +All stdlib extension modules will be *isolated*. That is: + +- Types, functions and other objects defined by the module will either be + immutable, or not shared with other module instances. + +- State specific to the module will not be shared with other module instances, + unless it represents global state. + + For example, ``_csv.field_size_limit`` will get/set a module-specific + number. On the other hand, functions like ``readline.get_history_item`` or + ``os.getpid`` will continue to work with state that is process-global + (external to the module, and possibly shared across other libraries, including + non-Python ones). + +Conversion to heap types +------------------------ + +Static types that do not need module state access, and have no other reason to +be converted, should stay static. + +Types whose methods need access to their module instance will be converted +to heap types following :pep:`630`, with the following considerations: + +- All standard library types that used to be static types should remain + immutable. Heap types must be defined with the ``Py_TPFLAGS_IMMUTABLE_TYPE`` + flag to retain immutability. + See `bpo-43908 `__. + + Tests should ensure ``TypeError`` is raised when trying to create a new + attribute of an immutable type. + +- A static type with ``tp_new = NULL`` does not have a public constructor, but + heap types inherit the constructor from the base class. Make sure types that + previously were impossible to instantiate retain that feature; use + ``Py_TPFLAGS_DISALLOW_INSTANTIATION``. Add tests using + ``test.support.check_disallow_instantiation()``. See + `bpo-43916 `__. + +- Converted heap types may unintentionally become serializable + (``pickle``-able). Test that calling ``pickle.dumps`` has the same result + before and after conversion, and if the test fails, add a ``__reduce__`` + method that raises ``TypeError``. See `PR-21002 `__ + for an example. + +These issues will be added to the Devguide to help any future conversions. + +If another kind of issue is found, the module in question should be unchanged +until a solution is found and added to the Devguide, and already +converted modules are checked and fixed. + + +Process +------- + +The following process should be added to the Devguide, and remain until +all modules are converted. +Any new findings should be documented there or in the general HOWTO. + +Part 1: Preparation +................... + +1. Open a discussion, either on the bug tracker or on Discourse. Involve the + module maintainer and/or code owner. Explain the reason and rationale for + the changes. +2. Identify global state performance bottlenecks. + Create a proof-of-concept implementation, and measure the performance impact. + ``pyperf`` is a good tool for benchmarking. +3. Create an implementation plan. For small modules with few types, a single PR + may do the job. For larger modules with lots of types, and possibly also + external library callbacks, multiple PR's will be needed. + + +Part 2: Implementation +...................... + +Note: this is a suggested implementation plan for a complex module, based on +lessons learned with other modules. Feel free to simplify it for +smaller modules. + +1. Add Argument Clinic where possible; it enables you to easily use the + defining class to fetch module state from type methods. +2. Prepare for module state; establish a module state ``struct``, add an instance + as a static global variable, and create helper stubs for fetching the module + state. +3. Add relevant global variables to the module state ``struct``, and modify code + that accesses the global state to use the module state helpers instead. This + step may be broken into several PR's. +4. Where necessary, convert static types to heap types. +5. Convert the global module state struct to true module state. +6. Implement multi-phase initialisation. + +Steps 4 through 6 should preferably land in a single alpha development phase. + + +Backwards Compatibility +======================= + +Extension modules in the standard library will now be loadable more than once. +For example, deleting such a module from ``sys.modules`` and re-importing it +will result in a fresh module instance, isolated from any previously loaded +instances. + +This may affect code that expected the previous behavior: globals of +extension modules were shallowly copied from the first loaded module. + + +Security Implications +===================== + +None known. + + +How to Teach This +================= + +A large part of this proposal is a HOWTO aimed at experienced users, +which will be moved to the documentation. + +Beginners should not be affected. + + +Reference Implementation +======================== + +Most of the changes are now in the main branch, as commits for these issues: + +- `bpo-40077, Convert static types to heap types: use PyType_FromSpec() `_ +- `bpo-46417, Clear static types in Py_Finalize() for embedded Python `_ +- `bpo-1635741, Py_Finalize() doesn't clear all Python objects at exit `_ + +As an example, changes and fix-ups done in the ``_csv`` module are: + +- `GH-23224, Remove static state from the _csv module `_ +- `GH-26008, Allow subclassing of csv.Error `_ +- `GH-26074, Add GC support to _csv heap types `_ +- `GH-26351, Make heap types converted during 3.10 alpha immutable `_ + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/peps/pep-0688.rst b/peps/pep-0688.rst new file mode 100644 index 000000000..5c7628388 --- /dev/null +++ b/peps/pep-0688.rst @@ -0,0 +1,443 @@ +PEP: 688 +Title: Making the buffer protocol accessible in Python +Author: Jelle Zijlstra +Discussions-To: https://discuss.python.org/t/19756 +Status: Accepted +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 23-Apr-2022 +Python-Version: 3.12 +Post-History: `23-Apr-2022 `__, + `25-Apr-2022 `__, + `06-Oct-2022 `__, + `26-Oct-2022 `__ +Resolution: https://discuss.python.org/t/pep-688-making-the-buffer-protocol-accessible-in-python/15265/35 + + +Abstract +======== + +This PEP proposes a Python-level API for the buffer protocol, +which is currently accessible only to C code. This allows type +checkers to evaluate whether objects implement the protocol. + + +Motivation +========== + +The CPython C API provides a versatile mechanism for accessing the +underlying memory of an object—the `buffer protocol `__ +introduced in :pep:`3118`. +Functions that accept binary data are usually written to handle any +object implementing the buffer protocol. For example, at the time of writing, +there are around 130 functions in CPython using the Argument Clinic +``Py_buffer`` type, which accepts the buffer protocol. + +Currently, there is no way for Python code to inspect whether an object +supports the buffer protocol. Moreover, the static type system +does not provide a type annotation to represent the protocol. +This is a `common problem `__ +when writing type annotations for code that accepts generic buffers. + +Similarly, it is impossible for a class written in Python to support +the buffer protocol. A buffer class in +Python would give users the ability to easily wrap a C buffer object, or to test +the behavior of an API that consumes the buffer protocol. Granted, this is not +a particularly common need. However, there has been a +`CPython feature request `__ +for supporting buffer classes written in Python that has been open since 2012. + + +Rationale +========= + +Current options +--------------- + +There are two known workarounds for annotating buffer types in +the type system, but neither is adequate. + +First, the `current workaround `__ +for buffer types in typeshed is a type alias +that lists well-known buffer types in the standard library, such as +``bytes``, ``bytearray``, ``memoryview``, and ``array.array``. This +approach works for the standard library, but it does not extend to +third-party buffer types. + +Second, the `documentation `__ +for ``typing.ByteString`` currently states: + + This type represents the types ``bytes``, ``bytearray``, and + ``memoryview`` of byte sequences. + + As a shorthand for this type, ``bytes`` can be used to annotate + arguments of any of the types mentioned above. + +Although this sentence has been in the documentation +`since 2015 `__, +the use of ``bytes`` to include these other types is not specified +in any of the typing PEPs. Furthermore, this mechanism has a number of +problems. It does not include all possible buffer types, and it +makes the ``bytes`` type ambiguous in type annotations. After all, +there are many operations that are valid on ``bytes`` objects, but +not on ``memoryview`` objects, and it is perfectly possible for +a function to accept ``bytes`` but not ``memoryview`` objects. +A mypy user +`reports `__ +that this shortcut has caused significant problems for the ``psycopg`` project. + +Kinds of buffers +---------------- + +The C buffer protocol supports +`many options `__, +affecting strides, contiguity, and support for writing to the buffer. Some of these +options would be useful in the type system. For example, typeshed +currently provides separate type aliases for writable and read-only +buffers. + +However, in the C buffer protocol, most of these options cannot be +queried directly on the type object. The only way to figure out +whether an object supports a particular flag is to actually +ask for the buffer. For some types, such as ``memoryview``, +the supported flags depend on the instance. As a result, it would +be difficult to represent support for these flags in the type system. + + +Specification +============= + +Python-level buffer protocol +---------------------------- + +We propose to add two Python-level special methods, ``__buffer__`` +and ``__release_buffer__``. Python +classes that implement these methods are usable as buffers from C +code. Conversely, classes implemented in C that support the +buffer protocol acquire synthesized methods accessible from Python +code. + +The ``__buffer__`` method is called to create a buffer from a Python +object, for example by the ``memoryview()`` constructor. +It corresponds to the ``bf_getbuffer`` C slot. +The Python signature for this method is +``def __buffer__(self, flags: int, /) -> memoryview: ...``. The method +must return a ``memoryview`` object. If the ``bf_getbuffer`` slot +is invoked on a Python class with a ``__buffer__`` method, +the interpreter extracts the underlying ``Py_buffer`` from the +``memoryview`` returned by the method +and returns it to the C caller. Similarly, if Python code calls the +``__buffer__`` method on an instance of a C class that +implements ``bf_getbuffer``, the returned buffer is wrapped in a +``memoryview`` for consumption by Python code. + +The ``__release_buffer__`` method should be called when a caller no +longer needs the buffer returned by ``__buffer__``. It corresponds to the +``bf_releasebuffer`` C slot. This is an +optional part of the buffer protocol. +The Python signature for this method is +``def __release_buffer__(self, buffer: memoryview, /) -> None: ...``. +The buffer to be released is wrapped in a ``memoryview``. When this +method is invoked through CPython's buffer API (for example, through +calling ``memoryview.release`` on a ``memoryview`` returned by +``__buffer__``), the passed ``memoryview`` is the same object +as was returned by ``__buffer__``. It is +also possible to call ``__release_buffer__`` on a C class that +implements ``bf_releasebuffer``. + +If ``__release_buffer__`` exists on an object, +Python code that calls ``__buffer__`` directly on the object must +call ``__release_buffer__`` on the same object when it is done +with the buffer. Otherwise, resources used by the object may +not be reclaimed. Similarly, it is a programming error +to call ``__release_buffer__`` without a previous call to +``__buffer__``, or to call it multiple times for a single call +to ``__buffer__``. For objects that implement the C buffer protocol, +calls to ``__release_buffer__`` where the argument is not a +``memoryview`` wrapping the same object will raise an exception. +After a valid call to ``__release_buffer__``, the ``memoryview`` +is invalidated (as if its ``release()`` method had been called), +and any subsequent calls to ``__release_buffer__`` with the same +``memoryview`` will raise an exception. +The interpreter will ensure that misuse +of the Python API will not break invariants at the C level -- for +example, it will not cause memory safety violations. + +``inspect.BufferFlags`` +----------------------- + +To help implementations of ``__buffer__``, we add ``inspect.BufferFlags``, +a subclass of ``enum.IntFlag``. This enum contains all flags defined in the +C buffer protocol. For example, ``inspect.BufferFlags.SIMPLE`` has the same +value as the ``PyBUF_SIMPLE`` constant. + +``collections.abc.Buffer`` +-------------------------- + +We add a new abstract base classes, ``collections.abc.Buffer``, +which requires the ``__buffer__`` method. +This class is intended primarily for use in type annotations: + +.. code-block:: python + + def need_buffer(b: Buffer) -> memoryview: + return memoryview(b) + + need_buffer(b"xy") # ok + need_buffer("xy") # rejected by static type checkers + + +It can also be used in ``isinstance`` and ``issubclass`` checks: + +.. code-block:: pycon + + >>> from collections.abc import Buffer + >>> isinstance(b"xy", Buffer) + True + >>> issubclass(bytes, Buffer) + True + >>> issubclass(memoryview, Buffer) + True + >>> isinstance("xy", Buffer) + False + >>> issubclass(str, Buffer) + False + +In the typeshed stub files, the class should be defined as a ``Protocol``, +following the precedent of other simple ABCs in ``collections.abc`` such as +``collections.abc.Iterable`` or ``collections.abc.Sized``. + +Example +------- + +The following is an example of a Python class that implements the +buffer protocol: + +.. code-block:: python + + import contextlib + import inspect + + class MyBuffer: + def __init__(self, data: bytes): + self.data = bytearray(data) + self.view = None + + def __buffer__(self, flags: int) -> memoryview: + if flags != inspect.BufferFlags.FULL_RO: + raise TypeError("Only BufferFlags.FULL_RO supported") + if self.view is not None: + raise RuntimeError("Buffer already held") + self.view = memoryview(self.data) + return self.view + + def __release_buffer__(self, view: memoryview) -> None: + assert self.view is view # guaranteed to be true + self.view.release() + self.view = None + + def extend(self, b: bytes) -> None: + if self.view is not None: + raise RuntimeError("Cannot extend held buffer") + self.data.extend(b) + + buffer = MyBuffer(b"capybara") + with memoryview(buffer) as view: + view[0] = ord("C") + + with contextlib.suppress(RuntimeError): + buffer.extend(b"!") # raises RuntimeError + + buffer.extend(b"!") # ok, buffer is no longer held + + with memoryview(buffer) as view: + assert view.tobytes() == b"Capybara!" + + +Equivalent for older Python versions +------------------------------------ + +New typing features are usually backported to older Python versions +in the `typing_extensions `_ +package. Because the buffer protocol +is currently accessible only in C, this PEP cannot be fully implemented +in a pure-Python package like ``typing_extensions``. As a temporary +workaround, an abstract base class ``typing_extensions.Buffer`` +will be provided for Python versions +that do not have ``collections.abc.Buffer`` available. + +After this PEP is implemented, inheriting from ``collections.abc.Buffer`` will +not be necessary to indicate that an object supports the buffer protocol. +However, in older Python versions, it will be necessary to explicitly +inherit from ``typing_extensions.Buffer`` to indicate to type checkers that +a class supports the buffer protocol, since objects supporting the buffer +protocol will not have a ``__buffer__`` method. It is expected that this +will happen primarily in stub files, because buffer classes are necessarily +implemented in C code, which cannot have types defined inline. +For runtime uses, the ``ABC.register`` API can be used to register +buffer classes with ``typing_extensions.Buffer``. + + +No special meaning for ``bytes`` +-------------------------------- + +The special case stating that ``bytes`` may be used as a shorthand +for other ``ByteString`` types will be removed from the ``typing`` +documentation. +With ``collections.abc.Buffer`` available as an alternative, there will be no good +reason to allow ``bytes`` as a shorthand. +Type checkers currently implementing this behavior +should deprecate and eventually remove it. + + +Backwards Compatibility +======================= + +``__buffer__`` and ``__release_buffer__`` attributes +---------------------------------------------------- + +As the runtime changes in this PEP only add new functionality, there are +few backwards compatibility concerns. + +However, code that uses a ``__buffer__`` or ``__release_buffer__`` attribute for +other purposes may be affected. While all dunders are technically reserved for the +language, it is still good practice to ensure that a new dunder does not +interfere with too much existing code, especially widely used packages. A survey +of publicly accessible code found: + +- PyPy `supports `__ + a ``__buffer__`` method with compatible semantics to those proposed in this + PEP. A PyPy core developer `expressed his support `__ + for this PEP. +- pyzmq `implements `__ + a PyPy-compatible ``__buffer__`` method. +- mpi4py `defines `__ + a ``SupportsBuffer`` protocol that would be equivalent to this PEP's ``collections.abc.Buffer``. +- NumPy used to have an undocumented behavior where it would access a ``__buffer__`` attribute + (not method) to get an object's buffer. This was `removed `__ + in 2019 for NumPy 1.17. The behavior would have last worked in NumPy 1.16, which only supported + Python 3.7 and older. Python 3.7 will have reached its end of life by the time this PEP is expected to + be implemented. + +Thus, this PEP's use of the ``__buffer__`` method will improve interoperability with +PyPy and not interfere with the current versions of any major Python packages. + +No publicly accessible code uses the name ``__release_buffer__``. + +Removal of the ``bytes`` special case +------------------------------------- + +Separately, the recommendation to remove the special behavior for +``bytes`` in type checkers does have a backwards compatibility +impact on their users. An `experiment `__ +with mypy shows that several major open source projects that use it +for type checking will see new errors if the ``bytes`` promotion +is removed. Many of these errors can be fixed by improving +the stubs in typeshed, as has already been done for the +`builtins `__, +`binascii `__, +`pickle `__, and +`re `__ modules. +A `review `__ of all +usage of ``bytes`` types in typeshed is in progress. +Overall, the change improves type safety and makes the type system +more consistent, so we believe the migration cost is worth it. + + +How to Teach This +================= + +We will add notes pointing to ``collections.abc.Buffer`` in appropriate places in the +documentation, such as `typing.readthedocs.io `__ +and the `mypy cheat sheet `__. +Type checkers may provide additional pointers in their error messages. For example, +when they encounter a buffer object being passed to a function that +is annotated to only accept ``bytes``, the error message could include a note suggesting +the use of ``collections.abc.Buffer`` instead. + + +Reference Implementation +======================== + +An implementation of this PEP is +`available `__ +in the author's fork. + + +Rejected Ideas +============== + +``types.Buffer`` +---------------- + +An earlier version of this PEP proposed adding a new ``types.Buffer`` type with +an ``__instancecheck__`` implemented in C so that ``isinstance()`` checks can be +used to check whether a type implements the buffer protocol. This avoids the +complexity of exposing the full buffer protocol to Python code, while still +allowing the type system to check for the buffer protocol. + +However, that approach +does not compose well with the rest of the type system, because ``types.Buffer`` +would be a nominal type, not a structural one. For example, there would be no way +to represent "an object that supports both the buffer protocol and ``__len__``". With +the current proposal, ``__buffer__`` is like any other special method, so a +``Protocol`` can be defined combining it with another method. + +More generally, no other part of Python works like the proposed ``types.Buffer``. +The current proposal is more consistent with the rest of the language, where +C-level slots usually have corresponding Python-level special methods. + +Keep ``bytearray`` compatible with ``bytes`` +-------------------------------------------- + +It has been suggested to remove the special case where ``memoryview`` is +always compatible with ``bytes``, but keep it for ``bytearray``, because +the two types have very similar interfaces. However, several standard +library functions (e.g., ``re.compile``, ``socket.getaddrinfo``, and most +functions accepting path-like arguments) accept +``bytes`` but not ``bytearray``. In most codebases, ``bytearray`` is also +not a very common type. We prefer to have users spell out accepted types +explicitly (or use ``Protocol`` from :pep:`544` if only a specific set of +methods is required). This aspect of the proposal was `specifically +discussed `__ +on the typing-sig mailing list, without any strong disagreement from the +typing community. + +Distinguish between mutable and immutable buffers +------------------------------------------------- + +The most frequently used distinction within buffer types is +whether or not the buffer is mutable. Some functions accept only +mutable buffers (e.g., ``bytearray``, some ``memoryview`` objects), +others accept all buffers. + +An earlier version of this PEP proposed using the presence of the +``bf_releasebuffer`` slot to determine whether a buffer type is mutable. +This rule holds for most standard library buffer types, but the relationship +between mutability and the presence of this slot is not absolute. For +example, ``numpy`` arrays are mutable but do not have this slot. + +The current buffer protocol does not provide any way to reliably +determine whether a buffer type represents a mutable or immutable +buffer. Therefore, this PEP does not add type system support +for this distinction. +The question can be revisited in the future if the buffer protocol +is enhanced to provide static introspection support. +A `sketch `_ +for such a mechanism exists. + + +Acknowledgments +=============== + +Many people have provided useful feedback on drafts of this PEP. +Petr Viktorin has been particularly helpful in improving my understanding +of the subtleties of the buffer protocol. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0689.rst b/peps/pep-0689.rst new file mode 100644 index 000000000..01cf6f269 --- /dev/null +++ b/peps/pep-0689.rst @@ -0,0 +1,291 @@ +PEP: 689 +Title: Unstable C API tier +Author: Petr Viktorin +Discussions-To: https://discuss.python.org/t/pep-689-unstable-c-api-tier/20452 +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Requires: 523 +Created: 22-Apr-2022 +Python-Version: 3.12 +Post-History: `27-Apr-2022 `__, + `25-Aug-2022 `__, + `27-Oct-2022 `__, +Resolution: https://discuss.python.org/t/pep-689-unstable-c-api-tier/20452/13 + +.. canonical-doc:: :ref:`devguide:c-api` + + User-facing documentation is at :ref:`py3.12:unstable-c-api`. + + +Abstract +======== + +Some functions and types of the C-API are designated *unstable*, +meaning that they will not change in patch (bugfix/security) releases, +but may change between minor releases (e.g. between 3.11 and 3.12) without +deprecation warnings. + +Any C API with a leading underscore is designated *internal*, meaning that it +may change or disappear without any notice. + + +Motivation & Rationale +====================== + +Unstable C API tier +------------------- + +The Python C-API is currently divided into `three stability tiers `__: + +- Limited API, with high compatibility expectations +- Public API, which follows the :pep:`backwards compatibility policy + <387>`, and requires deprecation warnings before changes +- Internal (private) API, which can change at any time. + +Tools requiring access to CPython internals (e.g. advanced +debuggers and JIT compilers) are often built for minor series releases +of CPython, and assume that the C-API internals used do not change +in patch releases. To support these tools, we need a tier between the +Public and Private C-API, with guarantees on stability throughout +the minor-series release: the proposed *Unstable tier*. + +Some functions, like ``PyCode_New()``, are documented as unstable +(“Calling [it] directly can bind you to a precise Python version”), +and also often change in practice. +The unstable tier should make their status obvious even to people who don't +read the docs carefully enough, making them hard to use accidentally. + + +Reserving leading underscores for Private API +--------------------------------------------- + +Currently, CPython developers don't agree on the exact meaning of a leading +underscore in API names. +It is used to mean two different things: + +- API that may change between minor releases, as in the Unstable tier proposed + here (e.g. functions introduced in :pep:`523`). +- API that is *private* and should not be used outside of CPython at all + (e.g. because it may change without notice, or it relies on undocumented + assumptions that non-CPython code cannot guarantee). + +The unclear meaning makes the underscore less useful than it could be. +If it only marked *private* API, CPython developers could change underscored +functions, or remove unused ones, without researching how they're +documented or used outside CPython. + +With the introduction of a dedicated unstable tier, we can clarify the meaning +of the leading underscore. It should mark private API only. + + +Not breaking code unnecessarily +------------------------------- + +This PEP specifies that API in the unstable tier should have a special name +prefix. This means functions (macros, etc.) will need to be renamed. +After a rename, the old name should continue to be available until +an incompatible change is made (i.e. until call sites need to be updated +anyway). +In other words, just changing the tier of a function shouldn't break users' +code. + + +Specification +============= + +The C API is divided by stability expectations into `three “sections” `__ +(internal, public, and limited). +We'll now call these *stability tiers*, or *tiers* for short. + +An *Unstable tier* will be added. + +APIs (functions, types, etc.) in this tier will named with the ``PyUnstable_`` +prefix, with no leading underscore. + +They will be declared in headers used for public API (``Include/*.h``, +rather than in a subdirectory like ``Include/unstable/``). + +Several rules for dealing with the unstable tier will be introduced: + +- Unstable API should have no backwards-incompatible + changes across patch releases, but may change or be removed in minor + releases (3.x.0, including Alpha and Beta releases of 3.x.0). + Such changes must be documented and mentioned in the What's New document. + +- Backwards-incompatible changes to these APIs should be made so that + code that uses them will need to be updated to compile with + the new version (e.g. arguments should be added/removed, or a function should + be renamed, but the semantic meaning of an argument should not change). + +- Unstable API should be documented and tested. + +- To move an API from the public tier to the unstable tier, it should be + exposed under the new ``PyUnstable_*`` name. + + The old name should be deprecated (e.g. with ``Py_DEPRECATED``), but + continue to be available until an incompatible change is made to the API. + Per Python's backwards compatibility policy (:pep:`387`), this deprecation + needs to last *at least* two releases (without an SC exceptions). + But it can also last indefinitely -- for example, if :pep:`590`'s + :pep:`“provisional” <590#finalizing-the-api>` + ``_PyObject_Vectorcall`` was added today, it would be initially named + ``PyUnstable_Object_Vectorcall`` and there would be no plan to remove + this name. + + In the following cases, an incompatible change (and thus removing the + deprecated name) is allowed without an SC exception, as if the function was + already part of the Unstable tier: + + - Any API introduced before Python 3.12 that is *documented* to be less + stable than default. + - Any API introduced before Python 3.12 that was named with a leading + underscore. + + For examples, see the `initial unstable API`_ + specified in this PEP. + +- To move an *internal* API to the unstable tier, it should be + exposed under the new ``PyUnstable_*`` name. + + If the old name is documented, or widely used externally, + it should continue to be available until an + incompatible change is made (and call sites need to be updated). + It should start raising deprecation warnings (e.g. using ``Py_DEPRECATED``). + +- To move an API from the unstable tier to the public tier, it should be + exposed without the ``PyUnstable_*`` prefix. + + The old name should remain available until the API is deprecated or removed. + +- Adding new unstable API *for existing features* is allowed even after + Beta feature freeze, up until the first Release Candidate. + Consensus on Core Development Discourse or is needed in the Beta period. + +These rules will be documented in the `devguide `__, +and `user documentation `__ +will be updated accordingly. + +Reference docs for C API named ``PyUnstable_*`` will automatically show +notes with links to the unstable tier documentation. + + +Leading underscore +------------------ + +C API named with a leading underscore, as well as API only available with +``Py_BUILD_CORE``, will be considered *internal*. +This means: + +- It may change or be removed *without notice* in minor + releases (3.x.0, including Alpha and Beta releases of 3.x.0). + API changes in patch releases or Release Candidates should only be done if + absolutely necessary. + +- It should be documented in source comments or Devguide only, not in the + public documentation. + +- API introduced before Python 3.12 that is documented or widely used + externally should be moved to the Unstable tier as explained above. + + This might happen long after this PEP is accepted. + Consequently, for a few years core devs should do some research before + changing underscored API, especially if it doesn't need ``Py_BUILD_CORE``. + +Users of the C API are encouraged to search their codebase for ``_Py`` and +``_PY`` identifier prefixes, and treat any hits as issues to be eventually +fixed -- either by switching to an existing alternative, or by opening +a CPython issue to request exposing public API for their use case, +and eventually switching to that. + + +Initial unstable API +-------------------- + +The following API will be moved to the Unstable tier in the initial +implementation as proof of the concept. + +Code object constructors: + +- ``PyUnstable_Code_New()`` (renamed from ``PyCode_New``) +- ``PyUnstable_Code_NewWithPosOnlyArgs()`` (renamed from ``PyCode_NewWithPosOnlyArgs``) + +Code extra information (:pep:`523`): + +- ``PyUnstable_Eval_RequestCodeExtraIndex()`` (renamed from ``_PyEval_RequestCodeExtraIndex``) +- ``PyUnstable_Code_GetExtra()`` (renamed from ``_PyCode_GetExtra``) +- ``PyUnstable_Code_SetExtra()`` (renamed from ``_PyCode_SetExtra``) + +More are expected in Python 3.12, without the need for another PEP. + + +Backwards Compatibility +======================= + +The C API backwards compatibility expectations will be made clearer. + +All renamed API will be available under old names for as long as feasible. + + +How to Teach This +================= + +The changes affect advanced C programmers, who should consult the +updated reference documentation, devguide and/or What's New document. + + +Reference Implementation +======================== + +https://github.com/python/cpython/compare/main...encukou:unstable-tier + + +Rejected Ideas +============== + +No special prefix +----------------- + +In the initial version of this PEP, unstable API didn't have the ``PyUnstable`` +prefix. +Instead, defining ``Py_USING_UNSTABLE_API`` made the API available in a given +source file, signifying acknowledgement that the file as a whole will +potentially need to be revisited for each Python release. + +However, it was decided that unstable-ness needs to be exposed +in the individual names. + +Underscore prefix +----------------- + +It would be possible to mark both private and unstable API with +leading underscores. +However, that would dilute the meaning of ``_Py`` prefix. +Reserving the prefix for internal API only makes it trivial to search for. + + +New header directory +-------------------- + +Other API tiers have dedicated directories for headers +(``Include/cpython/``, ``Include/internal/``). + +Since the unstable tier uses a very obvious naming convention +and the names are always available, +a directory like ``Include/unstable/`` is unnecessary. + + +Python API +---------- + +It might be good to add a similar tier in the Python (not C) API, +e.g. for ``types.CodeType``. +However, the mechanism for that would need to be different. +This is outside the scope of the PEP. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0690.rst b/peps/pep-0690.rst new file mode 100644 index 000000000..593f117cd --- /dev/null +++ b/peps/pep-0690.rst @@ -0,0 +1,870 @@ +PEP: 690 +Title: Lazy Imports +Author: GermĂĄn MĂ©ndez Bravo , Carl Meyer +Sponsor: Barry Warsaw +Discussions-To: https://discuss.python.org/t/pep-690-lazy-imports/15474 +Status: Rejected +Type: Standards Track +Content-Type: text/x-rst +Created: 29-Apr-2022 +Python-Version: 3.12 +Post-History: `03-May-2022 `__, + `03-May-2022 `__ +Resolution: https://discuss.python.org/t/pep-690-lazy-imports-again/19661/26 + + +Abstract +======== + +This PEP proposes a feature to transparently defer the finding and execution of +imported modules until the moment when an imported object is first used. Since +Python programs commonly import many more modules than a single invocation of +the program is likely to use in practice, lazy imports can greatly reduce the +overall number of modules loaded, improving startup time and memory usage. Lazy +imports also mostly eliminate the risk of import cycles. + + +Motivation +========== + +Common Python code style :pep:`prefers <8#imports>` imports at module +level, so they don't have to be repeated within each scope the imported object +is used in, and to avoid the inefficiency of repeated execution of the import +system at runtime. This means that importing the main module of a program +typically results in an immediate cascade of imports of most or all of the +modules that may ever be needed by the program. + +Consider the example of a Python command line program (CLI) with a number of +subcommands. Each subcommand may perform different tasks, requiring the import +of different dependencies. But a given invocation of the program will only +execute a single subcommand, or possibly none (i.e. if just ``--help`` usage +info is requested). Top-level eager imports in such a program will result in the +import of many modules that will never be used at all; the time spent (possibly +compiling and) executing these modules is pure waste. + +To improve startup time, some large Python CLIs make imports lazy by manually +placing imports inline into functions to delay imports of expensive subsystems. +This manual approach is labor-intensive and fragile; one misplaced import or +refactor can easily undo painstaking optimization work. + +The Python standard library already includes built-in support for lazy imports, +via `importlib.util.LazyLoader +`_. +There are also third-party packages such as `demandimport +`_. These provide a "lazy module +object" which delays its own import until first attribute access. This is not +sufficient to make all imports lazy: imports such as ``from foo import a, b`` +will still eagerly import the module ``foo`` since they immediately access an +attribute from it. It also imposes noticeable runtime overhead on every module +attribute access, since it requires a Python-level ``__getattr__`` or +``__getattribute__`` implementation. + +Authors of scientific Python packages have also made extensive use of lazy +imports to allow users to write e.g. ``import scipy as sp`` and then easily +access many different submodules with e.g. ``sp.linalg``, without requiring all +the many submodules to be imported up-front. `SPEC 1 +`_ codifies this practice in the +form of a ``lazy_loader`` library that can be used explicitly in a package +``__init__.py`` to provide lazily accessible submodules. + +Users of static typing also have to import names for use in type annotations +that may never be used at runtime (if :pep:`563` or possibly in future +:pep:`649` are used to avoid eager runtime evaluation of annotations). Lazy +imports are very attractive in this scenario to avoid overhead of unneeded +imports. + +This PEP proposes a more general and comprehensive solution for lazy imports +that can encompass all of the above use cases and does not impose detectable +overhead in real-world use. The implementation in this PEP has already +`demonstrated +`_ +startup time improvements up to 70% and memory-use reductions up to 40% on +real-world Python CLIs. + +Lazy imports also eliminate most import cycles. With eager imports, "false +cycles" can easily occur which are fixed by simply moving an import to the +bottom of a module or inline into a function, or switching from ``from foo +import bar`` to ``import foo``. With lazy imports, these "cycles" just work. +The only cycles which will remain are those where two modules actually each use +a name from the other at module level; these "true" cycles are only fixable by +refactoring the classes or functions involved. + + +Rationale +========= + +The aim of this feature is to make imports transparently lazy. "Lazy" means +that the import of a module (execution of the module body and addition of the +module object to ``sys.modules``) should not occur until the module (or a name +imported from it) is actually referenced during execution. "Transparent" means +that besides the delayed import (and necessarily observable effects of that, +such as delayed import side effects and changes to ``sys.modules``), there is +no other observable change in behavior: the imported object is present in the +module namespace as normal and is transparently loaded whenever first used: its +status as a "lazy imported object" is not directly observable from Python or +from C extension code. + +The requirement that the imported object be present in the module namespace as +usual, even before the import has actually occurred, means that we need some +kind of "lazy object" placeholder to represent the not-yet-imported object. +The transparency requirement dictates that this placeholder must never be +visible to Python code; any reference to it must trigger the import and replace +it with the real imported object. + +Given the possibility that Python (or C extension) code may pull objects +directly out of a module ``__dict__``, the only way to reliably prevent +accidental leakage of lazy objects is to have the dictionary itself be +responsible to ensure resolution of lazy objects on lookup. + +When a lookup finds that the key references a lazy object, it resolves the lazy +object immediately before returning it. To avoid side effects mutating +dictionaries midway through iteration, all lazy objects in a dictionary are +resolved prior to starting an iteration; this could incur a performance penalty +when using bulk iterations (``iter(dict)``, ``reversed(dict)``, +``dict.__reversed__()``, ``dict.keys()``, ``iter(dict.keys())`` and +``reversed(dict.keys())``). To avoid this performance penalty on the vast +majority of dictionaries, which never contain any lazy objects, we steal a bit +from the ``dk_kind`` field for a new ``dk_lazy_imports`` flag to keep track of +whether a dictionary may contain lazy objects or not. + +This implementation comprehensively prevents leakage of lazy objects, ensuring +they are always resolved to the real imported object before anyone can get hold +of them for any use, while avoiding any significant performance impact on +dictionaries in general. + + +Specification +============= + +Lazy imports are opt-in, and they can be globally enabled either via a new +``-L`` flag to the Python interpreter, or via a call to a new +``importlib.set_lazy_imports()`` function. This function takes two arguments, a +boolean ``enabled`` and an ``excluding`` container. If ``enabled`` is true, lazy +imports will be turned on from that point forward. If it is false, they will be +turned off from that point forward. (Use of the ``excluding`` keyword is +discussed below under "Per-module opt-out.") + +When the flag ``-L`` is passed to the Python interpreter, a new +``sys.flags.lazy_imports`` is set to ``True``, otherwise it exists as ``False``. +This flag is used to propagate ``-L`` to new Python subprocesses. + +The flag in ``sys.flags.lazy_imports`` does not necessarily reflect the current +status of lazy imports, only whether the interpreter was started with the ``-L`` +option. Actual current status of whether lazy imports are enabled or not at any +moment can be retrieved using ``importlib.is_lazy_imports_enabled()``, which +will return ``True`` if lazy imports are enabled at the call point or ``False`` +otherwise. + +When lazy imports are enabled, the loading and execution of all (and only) +top-level imports is deferred until the imported name is first used. This could +happen immediately (e.g. on the very next line after the import statement) or +much later (e.g. while using the name inside a function being called by some +other code at some later time.) + +For these top level imports, there are two contexts which will make them eager +(not lazy): imports inside ``try`` / ``except`` / ``finally`` or ``with`` +blocks, and star imports (``from foo import *``.) Imports inside +exception-handling blocks (this includes ``with`` blocks, since those can also +"catch" and handle exceptions) remain eager so that any exceptions arising from +the import can be handled. Star imports must remain eager since performing the +import is the only way to know which names should be added to the namespace. + +Imports inside class definitions or inside functions/methods are not "top +level" and are never lazy. + +Dynamic imports using ``__import__()`` or ``importlib.import_module()`` are +also never lazy. + +Lazy imports state (i.e. whether they have been enabled, and any excluded +modules; see below) is per-interpreter, but global within the interpreter (i.e. +all threads will be affected). + + +Example +------- + +Say we have a module ``spam.py``:: + + # simulate some work + import time + time.sleep(10) + print("spam loaded") + +And a module ``eggs.py`` which imports it:: + + import spam + print("imports done") + +If we run ``python -L eggs.py``, the ``spam`` module will never be imported +(because it is never referenced after the import), ``"spam loaded"`` will never +be printed, and there will be no 10 second delay. + +But if ``eggs.py`` simply references the name ``spam`` after importing it, that +will be enough to trigger the import of ``spam.py``:: + + import spam + print("imports done") + spam + +Now if we run ``python -L eggs.py``, we will see the output ``"imports done"`` +printed first, then a 10 second delay, and then ``"spam loaded"`` printed after +that. + +Of course, in real use cases (especially with lazy imports), it's not +recommended to rely on import side effects like this to trigger real work. This +example is just to clarify the behavior of lazy imports. + +Another way to explain the effect of lazy imports is that it is as if each lazy +import statement had instead been written inline in the source code immediately +before each use of the imported name. So one can think of lazy imports as +similar to transforming this code:: + + import foo + + def func1(): + return foo.bar() + + def func2(): + return foo.baz() + +To this:: + + def func1(): + import foo + return foo.bar() + + def func2(): + import foo + return foo.baz() + +This gives a good sense of when the import of ``foo`` will occur under lazy +imports, but lazy import is not really equivalent to this code transformation. +There are several notable differences: + +* Unlike in the latter code, under lazy imports the name ``foo`` still does + exist in the module's global namespace, and can be imported or referenced by + other modules that import this one. (Such references would also trigger the + import.) + +* The runtime overhead of lazy imports is much lower than the latter code; after + the first reference to the name ``foo`` which triggers the import, subsequent + references will have zero import system overhead; they are indistinguishable + from a normal name reference. + +In a sense, lazy imports turn the import statement into just a declaration of an +imported name or names, to later be fully resolved when referenced. + +An import in the style ``from foo import bar`` can also be made lazy. When the +import occurs, the name ``bar`` will be added to the module namespace as a lazy +import. The first reference to ``bar`` will import ``foo`` and resolve ``bar`` +to ``foo.bar``. + + +Intended usage +-------------- + +Since lazy imports are a potentially-breaking semantic change, they should be +enabled only by the author or maintainer of a Python application, who is +prepared to thoroughly test the application under the new semantics, ensure it +behaves as expected, and opt-out any specific imports as needed (see below). +Lazy imports should not be enabled speculatively by the end user of a Python +application with any expectation of success. + +It is the responsibility of the application developer enabling lazy imports for +their application to opt-out any library imports that turn out to need to be +eager for their application to work correctly; it is not the responsibility of +library authors to ensure that their library behaves exactly the same under lazy +imports. + +The documentation of the feature, the ``-L`` flag, and the new ``importlib`` +APIs will be clear about the intended usage and the risks of adoption without +testing. + + +Implementation +-------------- + +Lazy imports are represented internally by a "lazy import" object. When a lazy +import occurs (say ``import foo`` or ``from foo import bar``), the key ``"foo"`` +or ``"bar"`` is immediately added to the module namespace dictionary, but with +its value set to an internal-only "lazy import" object that preserves all the +necessary metadata to execute the import later. + +A new boolean flag in ``PyDictKeysObject`` (``dk_lazy_imports``) is set to +signal that this particular dictionary may contain lazy import objects. This +flag is only used to efficiently resolve all lazy objects in "bulk" operations, +when a dictionary may contain lazy objects. + +Anytime a key is looked up in a dictionary to extract its value, the +value is checked to see if it is a lazy import object. If so, the lazy object +is immediately resolved, the relevant imported modules executed, the lazy +import object is replaced in the dictionary (if possible) by the actual +imported value, and the resolved value is returned from the lookup function. A +dictionary could mutate as part of an import side effect while resolving a lazy +import object. In this case it is not possible to efficiently replace the key +value with the resolved object. In this case, the lazy import object will gain +a cached pointer to the resolved object. On next access that cached reference +will be returned and the lazy import object will be replaced in the dict with +the resolved value. + +Because this is all handled internally by the dictionary implementation, lazy +import objects can never escape from the module namespace to become visible to +Python code; they are always resolved at their first reference. No stub, dummy +or thunk objects are ever visible to Python code or placed in ``sys.modules``. +If a module is imported lazily, no entry for it will appear in ``sys.modules`` +at all until it is actually imported on first reference. + +If two different modules (``moda`` and ``modb``) both contain a lazy ``import +foo``, each module's namespace dictionary will have an independent lazy import +object under the key ``"foo"``, delaying import of the same ``foo`` module. This +is not a problem. When there is first a reference to, say, ``moda.foo``, the +module ``foo`` will be imported and placed in ``sys.modules`` as usual, and the +lazy object under the key ``moda.__dict__["foo"]`` will be replaced by the +actual module ``foo``. At this point ``modb.__dict__["foo"]`` will remain a lazy +import object. When ``modb.foo`` is later referenced, it will also try to +``import foo``. This import will find the module already present in +``sys.modules``, as is normal for subsequent imports of the same module in +Python, and at this point will replace the lazy import object at +``modb.__dict__["foo"]`` with the actual module ``foo``. + +There are two cases in which a lazy import object can escape a dictionary: + +* Into another dictionary: to preserve the performance of bulk-copy operations + like ``dict.update()`` and ``dict.copy()``, they do not check for or resolve + lazy import objects. However, if the source dict has the ``dk_lazy_imports`` + flag set that indicates it may contain lazy objects, that flag will be + passed on to the updated/copied dictionary. This still ensures that the lazy + import object can't escape into Python code without being resolved. + +* Through the garbage collector: lazy imported objects are still Python objects + and live within the garbage collector; as such, they can be collected and seen + via e.g. ``gc.get_objects()``. If a lazy object becomes + visible to Python code in this way, it is opaque and inert; it has no useful + methods or attributes. A ``repr()`` of it would be shown as something like: + ````. + +When a lazy object is added to a dictionary, the flag ``dk_lazy_imports`` is set. +Once set, the flag is only cleared if *all* lazy import objects in the +dictionary are resolved, e.g. prior to dictionary iteration. + +All dictionary iteration methods involving values (such as ``dict.items()``, +``dict.values()``, ``PyDict_Next()`` etc.) will attempt to resolve *all* lazy +import objects in the dictionary prior to starting the iteration. Since only +(some) module namespace dictionaries will ever have ``dk_lazy_imports`` set, the +extra overhead of resolving all lazy import objects inside a dictionary is only +paid by those dictionaries that need it. Minimizing the overhead on normal +non-lazy dictionaries is the sole purpose of the ``dk_lazy_imports`` flag. + +``PyDict_Next`` will attempt to resolve all lazy import objects the first time +position ``0`` is accessed, and those imports could fail with exceptions. Since +``PyDict_Next`` cannot set an exception, ``PyDict_Next`` will return ``0`` +immediately in this case, and any exception will be printed to stderr as an +unraisable exception. + +For this reason, this PEP introduces ``PyDict_NextWithError``, which works in +the same way as ``PyDict_Next``, but which can set an error when returning ``0`` +and this should be checked via ``PyErr_Occurred()`` after the call. + +The eagerness of imports within ``try`` / ``except`` / ``with`` blocks or within +class or function bodies is handled in the compiler via a new +``EAGER_IMPORT_NAME`` opcode that always imports eagerly. Top-level imports use +``IMPORT_NAME``, which may be lazy or eager depending on ``-L`` and/or +``importlib.set_lazy_imports()``. + + +Debugging +--------- + +Debug logging from ``python -v`` will include logging whenever an import +statement has been encountered but execution of the import will be deferred. + +Python's ``-X importtime`` feature for profiling import costs adapts naturally +to lazy imports; the profiled time is the time spent actually importing. + +Although lazy import objects are not generally visible to Python code, in some +debugging cases it may be useful to check from Python code whether the value at +a given key in a given dictionary is a lazy import object, without triggering +its resolution. For this purpose, ``importlib.is_lazy_import()`` can be used:: + + from importlib import is_lazy_import + + import foo + + is_lazy_import(globals(), "foo") + + foo + + is_lazy_import(globals(), "foo") + +In this example, if lazy imports have been enabled the first call to +``is_lazy_import`` will return ``True`` and the second will return ``False``. + + +Per-module opt-out +------------------ + +Due to the backwards compatibility issues mentioned below, it may be necessary +for an application using lazy imports to force some imports to be eager. + +In first-party code, since imports inside a ``try`` or ``with`` block are never +lazy, this can be easily accomplished:: + + try: # force these imports to be eager + import foo + import bar + finally: + pass + +This PEP proposes to add a new ``importlib.eager_imports()`` context manager, +so the above technique can be less verbose and doesn't require comments to +clarify its intent:: + + from importlib import eager_imports + + with eager_imports(): + import foo + import bar + +Since imports within context managers are always eager, the ``eager_imports()`` +context manager can just be an alias to a null context manager. The context +manager's effect is not transitive: ``foo`` and ``bar`` will be imported +eagerly, but imports within those modules will still follow the usual laziness +rules. + +The more difficult case can occur if an import in third-party code that can't +easily be modified must be forced to be eager. For this purpose, +``importlib.set_lazy_imports()`` takes a second optional keyword-only +``excluding`` argument, which can be set to a container of module names within +which all imports will be eager:: + + from importlib import set_lazy_imports + + set_lazy_imports(excluding=["one.mod", "another"]) + +The effect of this is also shallow: all imports within ``one.mod`` will be +eager, but not imports in all modules imported by ``one.mod``. + +The ``excluding`` parameter of ``set_lazy_imports()`` can be a container of any +kind that will be checked to see whether it contains a module name. If the +module name is contained in the object, imports within it will be eager. Thus, +arbitrary opt-out logic can be encoded in a ``__contains__`` method:: + + import re + from importlib import set_lazy_imports + + class Checker: + def __contains__(self, name): + return re.match(r"foo\.[^.]+\.logger", name) + + set_lazy_imports(excluding=Checker()) + +If Python was executed with the ``-L`` flag, then lazy imports will already be +globally enabled, and the only effect of calling ``set_lazy_imports(True, +excluding=...)`` will be to globally set the eager module names/callback. If +``set_lazy_imports(True)`` is called with no ``excluding`` argument, the +exclusion list/callback will be cleared and all eligible imports (module-level +imports not in ``try/except/with``, and not ``import *``) will be lazy from that +point forward. + +This opt-out system is designed to maintain the possibility of local reasoning +about the laziness of an import. You only need to see the code of one module, +and the ``excluding`` argument to ``set_lazy_imports``, if any, to know whether +a given import will be eager or lazy. + + +Testing +------- + +The CPython test suite will pass with lazy imports enabled (with some tests +skipped). One buildbot should run the test suite with lazy imports enabled. + + +C API +----- + +For authors of C extension modules, the proposed public C API is as follows: + +.. list-table:: + :widths: 50 50 + :header-rows: 1 + + * - C API + - Python API + * - ``PyObject *PyImport_SetLazyImports(PyObject *enabled, PyObject *excluding)`` + - ``importlib.set_lazy_imports(enabled: bool = True, *, excluding: typing.Container[str] | None = None)`` + * - ``int PyDict_IsLazyImport(PyObject *dict, PyObject *name)`` + - ``importlib.is_lazy_import(dict: typing.Dict[str, object], name: str) -> bool`` + * - ``int PyImport_IsLazyImportsEnabled()`` + - ``importlib.is_lazy_imports_enabled() -> bool`` + * - ``void PyDict_ResolveLazyImports(PyObject *dict)`` + - + * - ``PyDict_NextWithError()`` + - + +* ``void PyDict_ResolveLazyImports(PyObject *dict)`` resolves all lazy objects + in a dictionary, if any. To be used prior calling ``PyDict_NextWithError()`` + or ``PyDict_Next()``. + +* ``PyDict_NextWithError()``, works the same way as ``PyDict_Next()``, with + the exception it propagates any errors to the caller by returning ``0`` and + setting an exception. The caller should use ``PyErr_Occurred()`` to check for any + errors. + + +Backwards Compatibility +======================= + +This proposal preserves full backwards compatibility when the feature is +disabled, which is the default. + +Even when enabled, most code will continue to work normally without any +observable change (other than improved startup time and memory usage.) +Namespace packages are not affected: they work just as they do currently, +except lazily. + +In some existing code, lazy imports could produce currently unexpected results +and behaviors. The problems that we may see when enabling lazy imports in an +existing codebase are related to: + + +Import Side Effects +------------------- + +Import side effects that would otherwise be produced by the execution of +imported modules during the execution of import statements will be deferred +until the imported objects are used. + +These import side effects may include: + +* code executing any side-effecting logic during import; +* relying on imported submodules being set as attributes in the parent module. + +A relevant and typical affected case is the `click +`_ library for building Python command-line +interfaces. If e.g. ``cli = click.group()`` is defined in ``main.py``, and +``sub.py`` imports ``cli`` from ``main`` and adds subcommands to it via +decorator (``@cli.command(...)``), but the actual ``cli()`` call is in +``main.py``, then lazy imports may prevent the subcommands from being +registered, since in this case Click is depending on side effects of the import +of ``sub.py``. In this case the fix is to ensure the import of ``sub.py`` is +eager, e.g. by using the ``importlib.eager_imports()`` context manager. + + +Dynamic Paths +------------- + +There could be issues related to dynamic Python import paths; particularly, +adding (and then removing after the import) paths from ``sys.path``:: + + sys.path.insert(0, "/path/to/foo/module") + import foo + del sys.path[0] + foo.Bar() + +In this case, with lazy imports enabled, the import of ``foo`` will not actually +occur while the addition to ``sys.path`` is present. + +An easy fix for this (which also improves the code style and ensures cleanup) +would be to place the ``sys.path`` modifications in a context manager. This +resolves the issue, since imports inside a ``with`` block are always eager. + + +Deferred Exceptions +------------------- + +Exceptions that occur during a lazy import bubble up and erase the +partially-constructed module(s) from ``sys.modules``, just as exceptions during +normal import do. + +Since errors raised during a lazy import will occur later than they would if +the import were eager (i.e. wherever the name is first referenced), it is also +possible that they could be accidentally caught by exception handlers that did +not expect the import to be running within their ``try`` block, leading to +confusion. + + +Drawbacks +========= + +Downsides of this PEP include: + +* It provides a subtly incompatible semantics for the behavior of Python + imports. This is a potential burden on library authors who may be asked by their + users to support both semantics, and is one more possibility for Python + users/readers to be aware of. + +* Some popular Python coding patterns (notably centralized registries populated + by a decorator) rely on import side effects and may require explicit opt-out to + work as expected with lazy imports. + +* Exceptions can be raised at any point while accessing names representing lazy + imports, this could lead to confusion and debugging of unexpected exceptions. + +Lazy import semantics are already possible and even supported today in the +Python standard library, so these drawbacks are not newly introduced by this +PEP. So far, existing usage of lazy imports by some applications has not proven +a problem. But this PEP could make the usage of lazy imports more popular, +potentially exacerbating these drawbacks. + +These drawbacks must be weighed against the significant benefits offered by this +PEP's implementation of lazy imports. Ultimately these costs will be higher if +the feature is widely used; but wide usage also indicates the feature provides a +lot of value, perhaps justifying the costs. + + +Security Implications +===================== + +Deferred execution of code could produce security concerns if process owner, +shell path, ``sys.path``, or other sensitive environment or contextual states +change between the time the ``import`` statement is executed and the time the +imported object is first referenced. + + +Performance Impact +================== + +The reference implementation has shown that the feature has negligible +performance impact on existing real-world codebases (Instagram Server, several +CLI programs at Meta, Jupyter notebooks used by Meta researchers), while +providing substantial improvements to startup time and memory usage. + +The reference implementation shows `no measurable change +`_ +in aggregate performance on the `pyperformance benchmark suite +`_. + + +How to Teach This +================= + +Since the feature is opt-in, beginners should not encounter it by default. +Documentation of the ``-L`` flag and ``importlib.set_lazy_imports()`` can +clarify the behavior of lazy imports. + +The documentation should also clarify that opting into lazy imports is opting +into a non-standard semantics for Python imports, which could cause Python +libraries to break in unexpected ways. The responsibility to identify these +breakages and work around them with an opt-out (or stop using lazy imports) +rests entirely with the person choosing to enable lazy imports for their +application, not with the library author. Python libraries are under no +obligation to support lazy import semantics. Politely reporting an +incompatibility may be useful to the library author, but they may choose to +simply say their library does not support use with lazy imports, and this is a +valid choice. + +Some best practices to deal with some of the issues that could arise and to +better take advantage of lazy imports are: + +* Avoid relying on import side effects. Perhaps the most common reliance on + import side effects is the registry pattern, where population of some external + registry happens implicitly during the importing of modules, often via + decorators. Instead, the registry should be built via an explicit call that does + a discovery process to find decorated functions or classes in explicitly + nominated modules. + +* Always import needed submodules explicitly, don't rely on some other import + to ensure a module has its submodules as attributes. That is, unless there is an + explicit ``from . import bar`` in ``foo/__init__.py``, always do ``import + foo.bar; foo.bar.Baz``, not ``import foo; foo.bar.Baz``. The latter only works + (unreliably) because the attribute ``foo.bar`` is added as a side effect of + ``foo.bar`` being imported somewhere else. With lazy imports this may not always + happen in time. + +* Avoid using star imports, as those are always eager. + + +Reference Implementation +======================== + +The initial implementation is available as part of `Cinder +`_. This reference implementation +is in use within Meta and has proven to achieve improvements in startup time +(and total runtime for some applications) in the range of 40%-70%, as well as +significant reduction in memory footprint (up to 40%), thanks to not needing to +execute imports that end up being unused in the common flow. + +An `updated reference implementation based on CPython main branch +`_ is also available. + + +Rejected Ideas +============== + +Wrapping deferred exceptions +---------------------------- + +To reduce the potential for confusion, exceptions raised in the +course of executing a lazy import could be replaced by a ``LazyImportError`` +exception (a subclass of ``ImportError``), with a ``__cause__`` set to the +original exception. + +Ensuring that all lazy import errors are raised as ``LazyImportError`` would +reduce the likelihood that they would be accidentally caught and mistaken for a +different expected exception. However, in practice we have seen cases, e.g. +inside tests, where failing modules raise ``unittest.SkipTest`` exception and +this too would end up being wrapped in ``LazyImportError``, making such tests +fail because the true exception type is hidden. The drawbacks here seem to +outweigh the hypothetical case where unexpected deferred exceptions are caught +by mistake. + + +Per-module opt-in +----------------- + +A per-module opt-in using future imports (i.e. +``from __future__ import lazy_imports``) does not make sense because +``__future__`` imports are not feature flags, they are for transition to +behaviors which will become default in the future. It is not clear if lazy +imports will ever make sense as the default behavior, so we should not +promise this with a ``__future__`` import. + +There are other cases where a library might desire to locally opt-in to lazy +imports for a particular module; e.g. a lazy top-level ``__init__.py`` for a +large library, to make its subcomponents accessible as lazy attributes. For now, +to keep the feature simpler, this PEP chooses to focus on the "application" use +case and does not address the library use case. The underlying laziness +mechanism introduced in this PEP could be used in the future to address this use +case as well. + + +Explicit syntax for individual lazy imports +------------------------------------------- + +If the primary objective of lazy imports were solely to work around import +cycles and forward references, an explicitly-marked syntax for particular +targeted imports to be lazy would make a lot of sense. But in practice it would +be very hard to get robust startup time or memory use benefits from this +approach, since it would require converting most imports within your code base +(and in third-party dependencies) to use the lazy import syntax. + +It would be possible to aim for a "shallow" laziness where only the top-level +imports of subsystems from the main module are made explicitly lazy, but then +imports within the subsystems are all eager. This is extremely fragile, though +-- it only takes one mis-placed import to undo the carefully constructed +shallow laziness. Globally enabling lazy imports, on the other hand, provides +in-depth robust laziness where you always pay only for the imports you use. + +There may be use cases (e.g. for static typing) where individually-marked lazy +imports are desirable to avoid forward references, but the perf/memory benefits +of globally lazy imports are not needed. Since this is a different set of +motivating use cases and requires new syntax, we prefer not to include it in +this PEP. Another PEP could build on top of this implementation and propose the +additional syntax. + + +Environment variable to enable lazy imports +------------------------------------------- + +Providing an environment variable opt-in lends itself too easily to abuse of the +feature. It may seem tempting for a Python user to, for instance, globally set +the environment variable in their shell in the hopes of speeding up all the +Python programs they run. This usage with untested programs is likely to lead to +spurious bug reports and maintenance burden for the authors of those tools. To +avoid this, we choose not to provide an environment variable opt-in at all. + + +Removing the ``-L`` flag +------------------------ + +We do provide the ``-L`` CLI flag, which could in theory be abused in a similar +way by an end user running an individual Python program that is run with +``python somescript.py`` or ``python -m somescript`` (rather than distributed +via Python packaging tools). But the potential scope for misuse is much less +with ``-L`` than an environment variable, and ``-L`` is valuable for some +applications to maximize startup time benefits by ensuring that all imports from +the start of a process will be lazy, so we choose to keep it. + +It is already the case that running arbitrary Python programs with command line +flags they weren't intended to be used with (e.g. ``-s``, ``-S``, ``-E``, or +``-I``) can have unexpected and breaking results. ``-L`` is nothing new in this +regard. + + +Half-lazy imports +----------------- + +It would be possible to eagerly run the import loader to the point of finding +the module source, but then defer the actual execution of the module and +creation of the module object. The advantage of this would be that certain +classes of import errors (e.g. a simple typo in the module name) would be +caught eagerly instead of being deferred to the use of an imported name. + +The disadvantage would be that the startup time benefits of lazy imports would +be significantly reduced, since unused imports would still require a filesystem +``stat()`` call, at least. It would also introduce a possibly non-obvious split +between *which* import errors are raised eagerly and which are delayed, when +lazy imports are enabled. + +This idea is rejected for now on the basis that in practice, confusion about +import typos has not been an observed problem with the reference +implementation. Generally delayed imports are not delayed forever, and errors +show up soon enough to be caught and fixed (unless the import is truly unused.) + +Another possible motivation for half-lazy imports would be to allow modules +themselves to control via some flag whether they are imported lazily or eagerly. +This is rejected both on the basis that it requires half-lazy imports, giving up +some of the performance benefits of import laziness, and because in general +modules do not decide how or when they are imported, the module importing them +decides that. There isn't clear rationale for this PEP to invert that control; +instead it just provides more options for the importing code to make the +decision. + + +Lazy dynamic imports +-------------------- + +It would be possible to add a ``lazy=True`` or similar option to +``__import__()`` and/or ``importlib.import_module()``, to enable them to +perform lazy imports. That idea is rejected in this PEP for lack of a clear +use case. Dynamic imports are already far outside the :pep:`8` code style +recommendations for imports, and can easily be made precisely as lazy as +desired by placing them at the desired point in the code flow. These aren't +commonly used at module top level, which is where lazy imports applies. + + +Deep eager-imports override +--------------------------- + +The proposed ``importlib.eager_imports()`` context manager and excluded modules +in the ``importlib.set_lazy_imports(excluding=...)`` override all have shallow +effects: they only force eagerness for the location they are applied to, not +transitively. It would be possible to provide a deep/transitive version of one +or both. That idea is rejected in this PEP because the implementation would be +complex (taking into account threads and async code), experience with the +reference implementation has not shown it to be necessary, and because it +prevents local reasoning about laziness of imports. + +A deep override can lead to confusing behavior because the +transitively-imported modules may be imported from multiple locations, some of +which use the "deep eager override" and some of which don't. Thus those modules +may still be imported lazily initially, if they are first imported from a +location that doesn't have the override. + +With deep overrides it is not possible to locally reason about whether a given +import will be lazy or eager. With the behavior specified in this PEP, such +local reasoning is possible. + + +Making lazy imports the default behavior +---------------------------------------- + +Making lazy imports the default/sole behavior of Python imports, instead of +opt-in, would have some long-term benefits, in that library authors would +(eventually) no longer need to consider the possibility of both semantics. + +However, the backwards-incompatibilies are such that this could only be +considered over a long time frame, with a ``__future__`` import. It is not at +all clear that lazy imports should become the default import semantics for +Python. + +This PEP takes the position that the Python community needs more experience with +lazy imports before considering making it the default behavior, so that is +entirely left to a possible future PEP. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0691.rst b/peps/pep-0691.rst new file mode 100644 index 000000000..0de7cfd7f --- /dev/null +++ b/peps/pep-0691.rst @@ -0,0 +1,1047 @@ +PEP: 691 +Title: JSON-based Simple API for Python Package Indexes +Author: Donald Stufft , + Pradyun Gedam , + Cooper Lees , + Dustin Ingram +PEP-Delegate: Brett Cannon +Discussions-To: https://discuss.python.org/t/pep-691-json-based-simple-api-for-python-package-indexes/15553 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 04-May-2022 +Post-History: `05-May-2022 `__ +Resolution: https://discuss.python.org/t/pep-691-json-based-simple-api-for-python-package-indexes/15553/70 + + +Abstract +======== + +The "Simple Repository API" that was defined in :pep:`503` (and was in use much +longer than that) has served us reasonably well for a very long time. However, +the reliance on using HTML as the data exchange mechanism has several +shortcomings. + +There are two major issues with an HTML-based API: + +- While HTML5 is a standard, it's an incredibly complex standard and ensuring + completely correct parsing of it involves complex logic that does not + currently exist within the Python standard library (nor the standard library + of many other languages). + + This means that to actually accept everything that is technically valid, tools + have to pull in large dependencies or they have to rely on the standard library's + ``html.parser`` library, which is lighter weight but potentially doesn't + fully support HTML5. + +- HTML5 is primarily designed as a markup language to present documents for human + consumption. Our use of it is driven largely for historical and accidental + reasons, and it's unlikely anyone would design an API that relied on it if + they were starting from scratch. + + The primary issue with using a markup format designed for human consumption + is that there's not a great way to actually encode data within HTML. We've + gotten around this by limiting the data we put in this API and being creative + with how we can cram data into the API (for instance, hashes are embedded as + URL fragments, adding the ``data-yanked`` attribute in :pep:`592`). + +:pep:`503` was largely an attempt to standardize what was already in use, so it +did not propose any large changes to the API. + +In the intervening years, we've regularly talked about an "API V2" that would +re-envision the entire API of PyPI. However, due to limited time constraints, +that effort has not gained much, if any, traction beyond people thinking that +it would be nice to do. + +This PEP attempts to take a different route. It doesn't fundamentally change +the overall API structure, but instead specifies a new serialization of the +existing data contained in existing :pep:`503` responses in a format that is +easier for software to parse rather than using a human centric document format. + + +Goals +===== + +- **Enable zero configuration discovery.** Clients of the simple API **MUST** be + able to gracefully determine whether a target repository supports this PEP + without relying on any form of out of band communication (configuration, prior + knowledge, etc). Individual clients **MAY** choose to require configuration + to enable the use of this API, however. +- **Enable clients to drop support for "legacy" HTML parsing.** While it is expected + that most clients will keep supporting HTML-only repositories for a while, if not + forever, it should be possible for a client to choose to support only the new + API formats and no longer invoke an HTML parser. +- **Enable repositories to drop support for "legacy" HTML formats.** Similar to + clients, it is expected that most repositories will continue to support HTML + responses for a long time, or forever. It should be possible for a repository to + choose to only support the new formats. +- **Maintain full support for existing HTML-only clients.** We **MUST** not break + existing clients that are accessing the API as a strictly :pep:`503` API. The only + exception to this, is if the repository itself has chosen to no longer support + the HTML format. +- **Minimal additional HTTP requests.** Using this API **MUST** not drastically + increase the amount of HTTP requests an installer must do in order to function. + Ideally it will require 0 additional requests, but if needed it may require one + or two additional requests (total, not per dependency). +- **Minimal additional unique responses.** Due to the nature of how large + repositories like PyPI cache responses, this PEP should not introduce a + significantly or combinatorially large number of additional unique responses + that the repository may produce. +- **Supports TUF.** This PEP **MUST** be able to function within the bounds of + what TUF can support (:pep:`458`), and must be able to be secured using it. +- **Require only the standard library, or small external dependencies for clients.** + Parsing an API response should ideally require nothing but the standard + library, however it would be acceptable to require a small, pure Python + dependency. + + +Specification +============= + +To enable response parsing with only the standard library, this PEP specifies that +all responses (besides the files themselves, and the HTML responses from +:pep:`503`) should be serialized using `JSON `_. + +To enable zero configuration discovery and to minimize the amount of additional HTTP +requests, this PEP extends :pep:`503` such that all of the API endpoints (other than the +files themselves) will utilize HTTP content negotiation to allow client and server to +select the correct serialization format to serve, i.e. either HTML or JSON. + + +Versioning +---------- + +Versioning will adhere to :pep:`629` format (``Major.Minor``), which has defined the +existing HTML responses to be ``1.0``. Since this PEP does not introduce new features +into the API, rather it describes a different serialization format for the existing +features, this PEP does not change the existing ``1.0`` version, and instead just +describes how to serialize that into JSON. + +Similar to :pep:`629`, the major version number **MUST** be incremented if any +changes to the new format would result in no longer being able to expect existing +clients to meaningfully understand the format. + +Likewise, the minor version **MUST** be incremented if features are +added or removed from the format, but existing clients would be expected to continue +to meaningfully understand the format. + +Changes that would not result in existing clients being unable to meaningfully +understand the format and which do not represent features being added or removed +may occur without changing the version number. + +This is intentionally vague, as this PEP believes it is best left up to future PEPs +that make any changes to the API to investigate and decide whether or not that +change should increment the major or minor version. + +Future versions of the API may add things that can only be represented in a subset +of the available serializations of that version. All serializations version numbers, +within a major version, **SHOULD** be kept in sync, but the specifics of how a +feature serializes into each format may differ, including whether or not that feature +is present at all. + +It is the intent of this PEP that the API should be thought of as URL endpoints that +return data, whose interpretation is defined by the version of that data, and then +serialized into the target serialization format. + + +.. _json-serialization: + +JSON Serialization +------------------ + +The URL structure from :pep:`503` still applies, as this PEP only adds an additional +serialization format for the already existing API. + +The following constraints apply to all JSON serialized responses described in this +PEP: + +* All JSON responses will *always* be a JSON object rather than an array or other + type. + +* While JSON doesn't natively support an URL type, any value that represents an + URL in this API may be either absolute or relative as long as they point to + the correct location. If relative, they are relative to the current URL as if + it were HTML. + +* Additional keys may be added to any dictionary objects in the API responses + and clients **MUST** ignore keys that they don't understand. + +* All JSON responses will have a ``meta`` key, which contains information related to + the response itself, rather than the content of the response. + +* All JSON responses will have a ``meta.api-version`` key, which will be a string that + contains the :pep:`629` ``Major.Minor`` version number, with the same fail/warn + semantics as defined in :pep:`629`. + +* All requirements of :pep:`503` that are not HTML specific still apply. + + +Project List +~~~~~~~~~~~~ + +The root URL ``/`` for this PEP (which represents the base URL) will be a JSON encoded +dictionary which has a two keys: + +- ``projects``: An array where each entry is a dictionary with a single key, ``name``, which represents string of the project name. +- ``meta``: The general response metadata as `described earlier `__. + +As an example: + +.. code-block:: json + + { + "meta": { + "api-version": "1.0" + }, + "projects": [ + {"name": "Frob"}, + {"name": "spamspamspam"} + ] + } + + +.. note:: + + The ``name`` field is the same as the one from :pep:`503`, which does not specify + whether it is the non-normalized display name or the normalized name. In practice + different implementations of these PEPs are choosing differently here, so relying + on it being either non-normalized or normalized is relying on an implementation + detail of the repository in question. + + +.. note:: + + While the ``projects`` key is an array, and thus is required to be in some kind + of an order, neither :pep:`503` nor this PEP requires any specific ordering nor + that the ordering is consistent from one request to the next. Mentally this is + best thought of as a set, but both JSON and HTML lack the functionality to have + sets. + + +Project Detail +~~~~~~~~~~~~~~ + +The format of this URL is ``//`` where the ```` is replaced by the +:pep:`503` normalized name for that project, so a project named "Silly_Walk" would +have a URL like ``/silly-walk/``. + +This URL must respond with a JSON encoded dictionary that has three keys: + +- ``name``: The normalized name of the project. +- ``files``: A list of dictionaries, each one representing an individual file. +- ``meta``: The general response metadata as `described earlier `__. + +Each individual file dictionary has the following keys: + +- ``filename``: The filename that is being represented. +- ``url``: The URL that the file can be fetched from. +- ``hashes``: A dictionary mapping a hash name to a hex encoded digest of the file. + Multiple hashes can be included, and it is up to the client to decide what to do + with multiple hashes (it may validate all of them or a subset of them, or nothing + at all). These hash names **SHOULD** always be normalized to be lowercase. + + The ``hashes`` dictionary **MUST** be present, even if no hashes are available + for the file, however it is **HIGHLY** recommended that at least one secure, + guaranteed-to-be-available hash is always included. + + By default, any hash algorithm available via `hashlib + `_ (specifically any that can + be passed to ``hashlib.new()`` and do not require additional parameters) can + be used as a key for the hashes dictionary. At least one secure algorithm from + ``hashlib.algorithms_guaranteed`` **SHOULD** always be included. At the time + of this PEP, ``sha256`` specifically is recommended. +- ``requires-python``: An **optional** key that exposes the *Requires-Python* + metadata field, specified in :pep:`345`. Where this is present, installer tools + **SHOULD** ignore the download when installing to a Python version that + doesn't satisfy the requirement. + + Unlike ``data-requires-python`` in :pep:`503`, the ``requires-python`` key does not + require any special escaping other than anything JSON does naturally. +- ``dist-info-metadata``: An **optional** key that indicates + that metadata for this file is available, via the same location as specified in + :pep:`658` (``{file_url}.metadata``). Where this is present, it **MUST** be + either a boolean to indicate if the file has an associated metadata file, or a + dictionary mapping hash names to a hex encoded digest of the metadata's hash. + + When this is a dictionary of hashes instead of a boolean, then all the same + requirements and recommendations as the ``hashes`` key hold true for this key as + well. + + If this key is missing then the metadata file may or may not exist. If the key + value is truthy, then the metadata file is present, and if it is falsey then it + is not. + + It is recommended that servers make the hashes of the metadata file available if + possible. +- ``gpg-sig``: An **optional** key that acts a boolean to indicate if the file has + an associated GPG signature or not. The URL for the signature file follows what + is specified in :pep:`503` (``{file_url}.asc``). If this key does not exist, then + the signature may or may not exist. +- ``yanked``: An **optional** key which may be either a boolean to indicate if the + file has been yanked, or a non empty, but otherwise arbitrary, string to indicate + that a file has been yanked with a specific reason. If the ``yanked`` key is present + and is a truthy value, then it **SHOULD** be interpreted as indicating that the + file pointed to by the ``url`` field has been "Yanked" as per :pep:`592`. + +As an example: + +.. code-block:: json + + { + "meta": { + "api-version": "1.0" + }, + "name": "holygrail", + "files": [ + { + "filename": "holygrail-1.0.tar.gz", + "url": "https://example.com/files/holygrail-1.0.tar.gz", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "yanked": "Had a vulnerability" + }, + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "https://example.com/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "dist-info-metadata": true + } + ] + } + + +.. note:: + + While the ``files`` key is an array, and thus is required to be in some kind + of an order, neither :pep:`503` nor this PEP requires any specific ordering nor + that the ordering is consistent from one request to the next. Mentally this is + best thought of as a set, but both JSON and HTML lack the functionality to have + sets. + + +Content-Types +------------- + +This PEP proposes that all responses from the Simple API will have a standard +content type that describes what the response is (a Simple API response), what +version of the API it represents, and what serialization format has been used. + +The structure of this content type will be: + +.. code-block:: text + + application/vnd.pypi.simple.$version+format + +Since only major versions should be disruptive to clients attempting to +understand one of these API responses, only the major version will be included +in the content type, and will be prefixed with a ``v`` to clarify that it is a +version number. + +Which means that for the existing 1.0 API, the content types would be: + +- **JSON:** ``application/vnd.pypi.simple.v1+json`` +- **HTML:** ``application/vnd.pypi.simple.v1+html`` + +In addition to the above, a special "meta" version is supported named ``latest``, +whose purpose is to allow clients to request the absolute latest version, without +having to know ahead of time what that version is. It is recommended however, +that clients be explicit about what versions they support. + +To support existing clients which expect the existing :pep:`503` API responses to +use the ``text/html`` content type, this PEP further defines ``text/html`` as an alias +for the ``application/vnd.pypi.simple.v1+html`` content type. + + +Version + Format Selection +-------------------------- + +Now that there is multiple possible serializations, we need a mechanism to allow +clients to indicate what serialization formats they're able to understand. In +addition, it would be beneficial if any possible new major version to the API can +be added without disrupting existing clients expecting the previous API version. + +To enable this, this PEP standardizes on the use of HTTP's +`Server-Driven Content Negotiation `_. + +While this PEP won't fully describe the entirety of server-driven content +negotiation, the flow is roughly: + +1. The client makes an HTTP request containing an ``Accept`` header listing all + of the version+format content types that they are able to understand. +2. The server inspects that header, selects one of the listed content types, + then returns a response using that content type (treating the absence of + an ``Accept`` header as ``Accept: */*``). +3. If the server does not support any of the content types in the ``Accept`` + header then they are able to choose between 3 different options for how to + respond: + + a. Select a default content type other than what the client has requested + and return a response with that. + b. Return a HTTP ``406 Not Acceptable`` response to indicate that none of + the requested content types were available, and the server was unable + or unwilling to select a default content type to respond with. + c. Return a HTTP ``300 Multiple Choices`` response that contains a list of + all of the possible responses that could have been chosen. +4. The client interprets the response, handling the different types of responses + that the server may have responded with. + +This PEP does not specify which choices the server makes in regards to handling +a content type that it isn't able to return, and clients **SHOULD** be prepared +to handle all of the possible responses in whatever way makes the most sense for +that client. + +However, as there is no standard format for how a ``300 Multiple Choices`` +response can be interpreted, this PEP highly discourages servers from utilizing +that option, as clients will have no way to understand and select a different +content-type to request. In addition, it's unlikely that the client *could* +understand a different content type anyways, so at best this response would +likely just be treated the same as a ``406 Not Acceptable`` error. + +This PEP **does** require that if the meta version ``latest`` is being used, the +server **MUST** respond with the content type for the actual version that is +contained in the response +(i.e. A ``Accept: application/vnd.pypi.simple.latest+json`` request that returns +a ``v1.x`` response should have a ``Content-Type`` of +``application/vnd.pypi.simple.v1+json``). + +The ``Accept`` header is a comma separated list of content types that the client +understands and is able to process. It supports three different formats for each +content type that is being requested: + +- ``$type/$subtype`` +- ``$type/*`` +- ``*/*`` + +For the use of selecting a version+format, the most useful of these is +``$type/$subtype``, as that is the only way to actually specify the version +and format you want. + +The order of the content types listed in the ``Accept`` header does not have any +specific meaning, and the server **SHOULD** consider all of them to be equally +valid to respond with. If a client wishes to specify that they prefer a specific +content type over another, they may use the ``Accept`` header's +`quality value `_ +syntax. + +This allows a client to specify a priority for a specific entry in their +``Accept`` header, by appending a ``;q=`` followed by a value between ``0`` and +``1`` inclusive, with up to 3 decimal digits. When interpreting this value, +an entry with a higher quality has priority over an entry with a lower quality, +and any entry without a quality present will default to a quality of ``1``. + +However, clients should keep in mind that a server is free to select **any** of +the content types they've asked for, regardless of their requested priority, and +it may even return a content type that they did **not** ask for. + +To aid clients in determining the content type of the response that they have +received from an API request, this PEP requires that servers always include a +``Content-Type`` header indicating the content type of the response. This is +technically a backwards incompatible change, however in practice +`pip has been enforcing this requirement `_ +so the risks for actual breakages is low. + +An example of how a client can operate would look like: + +.. code-block:: python + + import email.message + import requests + + def parse_content_type(header: str) -> str: + m = email.message.Message() + m["content-type"] = header + return m.get_content_type() + + # Construct our list of acceptable content types, we want to prefer + # that we get a v1 response serialized using JSON, however we also + # can support a v1 response serialized using HTML. For compatibility + # we also request text/html, but we prefer it least of all since we + # don't know if it's actually a Simple API response, or just some + # random HTML page that we've gotten due to a misconfiguration. + CONTENT_TYPES = [ + "application/vnd.pypi.simple.v1+json", + "application/vnd.pypi.simple.v1+html;q=0.2", + "text/html;q=0.01", # For legacy compatibility + ] + ACCEPT = ", ".join(CONTENT_TYPES) + + + # Actually make our request to the API, requesting all of the content + # types that we find acceptable, and letting the server select one of + # them out of the list. + resp = requests.get("https://pypi.org/simple/", headers={"Accept": ACCEPT}) + + # If the server does not support any of the content types you requested, + # AND it has chosen to return a HTTP 406 error instead of a default + # response then this will raise an exception for the 406 error. + resp.raise_for_status() + + + # Determine what kind of response we've gotten to ensure that it is one + # that we can support, and if it is, dispatch to a function that will + # understand how to interpret that particular version+serialization. If + # we don't understand the content type we've gotten, then we'll raise + # an exception. + content_type = parse_content_type(resp.headers.get("content-type", "")) + match content_type: + case "application/vnd.pypi.simple.v1+json": + handle_v1_json(resp) + case "application/vnd.pypi.simple.v1+html" | "text/html": + handle_v1_html(resp) + case _: + raise Exception(f"Unknown content type: {content_type}") + +If a client wishes to only support HTML or only support JSON, then they would +just remove the content types that they do not want from the ``Accept`` header, +and turn receiving them into an error. + + +Alternative Negotiation Mechanisms +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While using HTTP's Content negotiation is considered the standard way for a client +and server to coordinate to ensure that the client is getting an HTTP response that +it is able to understand, there are situations where that mechanism may not be +sufficient. For those cases this PEP has alternative negotiation mechanisms that +may *optionally* be used instead. + + +URL Parameter +^^^^^^^^^^^^^ + +Servers that implement the Simple API may choose to support an URL parameter named +``format`` to allow the clients to request a specific version of the URL. + +The value of the ``format`` parameter should be **one** of the valid content types. +Passing multiple content types, wild cards, quality values, etc... is **not** +supported. + +Supporting this parameter is optional, and clients **SHOULD NOT** rely on it for +interacting with the API. This negotiation mechanism is intended to allow for easier +human based exploration of the API within a browser, or to allow documentation or +notes to link to a specific version+format. + +Servers that do not support this parameter may choose to return an error when it is +present, or they may simple ignore its presence. + +When a server does implement this parameter, it **SHOULD** take precedence over any +values in the client's ``Accept`` header, and if the server does not support the +requested format, it may choose to fall back to the ``Accept`` header, or choose any +of the error conditions that standard server-driven content negotiation typically +has (e.g. ``406 Not Available``, ``303 Multiple Choices``, or selecting a default +type to return). + + +Endpoint Configuration +^^^^^^^^^^^^^^^^^^^^^^ + +This option technically is not a special option at all, it is just a natural +consequence of using content negotiation and allowing servers to select which of the +available content types is their default. + +If a server is unwilling or unable to implement the server-driven content negotiation, +and would instead rather require users to explicitly configure their client to select +the version they want, then that is a supported configuration. + +To enable this, a server should make multiple endpoints (for instance, +``/simple/v1+html/`` and/or ``/simple/v1+json/``) for each version+format that they +wish to support. Under that endpoint, they can host a copy of their repository that +only supports one (or a subset) of the content-types. When a client makes a request +using the ``Accept`` header, the server can ignore it and return the content type +that corresponds to that endpoint. + +For clients that wish to require specific configuration, they can keep track of +which version+format a specific repository URL was configured for, and when making +a request to that server, emit an ``Accept`` header that *only* includes the correct +content type. + + +TUF Support - PEP 458 +--------------------- + +:pep:`458` requires that all API responses are hashable and that they can be uniquely +identified by a path relative to the repository root. For a Simple API repository, the +target path is the Root of our API (e.g. ``/simple/`` on PyPI). This creates +challenges when accessing the API using a TUF client instead of directly using a +standard HTTP client, as the TUF client cannot handle the fact that a target could +have multiple different representations that all hash differently. + +:pep:`458` does not specify what the target path should be for the Simple API, but +TUF requires that the target paths be "file-like", in other words, a path like +``simple/PROJECT/`` is not acceptable, because it technically points to a +directory. + +The saving grace is that the target path does not *have* to actually match the URL +being fetched from the Simple API, and it can just be a sigil that the fetching code +knows how to transform into the actual URL that needs to be fetched. This same thing +can hold true for other aspects of the actual HTTP request, such as the ``Accept`` +header. + +Ultimately figuring out how to map a directory to a filename is out of scope for this +PEP (but it would be in scope for :pep:`458`), and this PEP defers making a decision +about how exactly to represent this inside of :pep:`458` metadata. + +However, it appears that the current WIP branch against pip that attempts to implement +:pep:`458` is using a target path like ``simple/PROJECT/index.html``. This could be +modified to include the API version and serialization format using something like +``simple/PROJECT/vnd.pypi.simple.vN.FORMAT``. So the v1 HTML format would be +``simple/PROJECT/vnd.pypi.simple.v1.html`` and the v1 JSON format would be +``simple/PROJECT/vnd.pypi.simple.v1.json``. + +In this case, since ``text/html`` is an alias to ``application/vnd.pypi.simple.v1+html`` +when interacting through TUF, it likely will make the most sense to normalize to the +more explicit name. + +Likewise the ``latest`` metaversion should not be included in the targets, only +explicitly declared versions should be supported. + + +Recommendations +=============== + +This section is non-normative, and represents what the PEP authors believe to be +the best default implementation decisions for something implementing this PEP, but +it does **not** represent any sort of requirement to match these decisions. + +These decisions have been chosen to maximize the number of requests that can be +moved onto the newest version of an API, while maintaining the greatest amount +of compatibility. In addition, they've also tried to make using the API provide +guardrails that attempt to push clients into making the best choices it can. + +It is recommended that servers: + +- Support all 3 content types described in this PEP, using server-driven + content negotiation, for as long as they reasonably can, or at least as + long as they're receiving non trivial traffic that uses the HTML responses. + +- When encountering an ``Accept`` header that does not contain any content types + that it knows how to work with, the server should not ever return a + ``300 Multiple Choice`` response, and instead return a ``406 Not Acceptable`` + response. + + - However, if choosing to use the endpoint configuration, you should prefer to + return a ``200 OK`` response in the expected content type for that endpoint. + +- When selecting an acceptable version, the server should choose the highest version + that the client supports, with the most expressive/featureful serialization format, + taking into account the specificity of the client requests as well as any + quality priority values they have expressed, and it should only use the + ``text/html`` content type as a last resort. + +It is recommended that clients: + +- Support all 3 content types described in this PEP, using server-driven + content negotiation, for as long as they reasonably can. + +- When constructing an ``Accept`` header, include all of the content types + that you support. + + You should generally *not* include a quality priority value for your content + types, unless you have implementation specific reasons that you want the + server to take into account (for example, if you're using the standard library + HTML parser and you're worried that there may be some kinds of HTML responses + that you're unable to parse in some edge cases). + + The one exception to this recommendation is that it is recommended that you + *should* include a ``;q=0.01`` value on the legacy ``text/html`` content type, + unless it is the only content type that you are requesting. + +- Explicitly select what versions they are looking for, rather than using the + ``latest`` meta version during normal operation. + +- Check the ``Content-Type`` of the response and ensure it matches something + that you were expecting. + + +FAQ +=== + +Does this mean PyPI is planning to drop support for HTML/PEP 503? +----------------------------------------------------------------- + +No, PyPI has no plans at this time to drop support for :pep:`503` or HTML +responses. + +While this PEP does give repositories the flexibility to do that, that largely +exists to ensure that things like using the Endpoint Configuration mechanism is +able to work, and to ensure that clients do not make any assumptions that would +prevent, at some point in the future, gracefully dropping support for HTML. + +The existing HTML responses incur almost no maintenance burden on PyPI and +there is no pressing need to remove them. The only real benefit to dropping them +would be to reduce the number of items cached in our CDN. + +If in the future PyPI *does* wish to drop support for them, doing so would +almost certainly be the topic of a PEP, or at a minimum a public, open, discussion +and would be informed by metrics showing any impact to end users. + + +Why JSON instead of X format? +----------------------------- + +JSON parsers are widely available in most, if not every, language. A JSON +parser is also available in the Python standard library. It's not the perfect +format, but it's good enough. + + +Why not add X feature? +---------------------- + +The general goal of this PEP is to change or add very little. We will instead focus +largely on translating the existing information contained within our HTML responses +into a sensible JSON representation. This will include :pep:`658` metadata required +for packaging tooling. + +The only real new capability that is added in this PEP is the ability to have +multiple hashes for a single file. That was done because the current mechanism being +limited to a single hash has made it painful in the past to migrate hashes +(md5 to sha256) and the cost of making the hashes a dictionary and allowing multiple +is pretty low. + +The API was generally designed to allow further extension through adding new keys, +so if there's some new piece of data that an installer might need, future PEPs can +easily make that available. + + +Why include the filename when the URL has it already? +----------------------------------------------------- + +We could reduce the size of our responses by removing the ``filename`` key and expecting +clients to pull that information out of the URL. + +Currently this PEP chooses not to do that, largely because :pep:`503` explicitly required +that the filename be available via the anchor tag of the links, though that was largely +because *something* had to be there. It's not clear if repositories in the wild always +have a filename as the last part of the URL or if they're relying on the filename in the +anchor tag. + +It also makes the responses slightly nicer to read for a human, as you get a nice short +unique identifier. + +If we got reasonable confidence that mandating the filename is in the URL, then we could +drop this data and reduce the size of the JSON response. + + +Why not break out other pieces of information from the filename? +---------------------------------------------------------------- + +Currently clients are expected to parse a number of pieces of information from the +filename such as project name, version, ABI tags, etc. We could break these out +and add them as keys to the file object. + +This PEP has chosen not to do that because doing so would increase the size of the +API responses, and most clients are going to require the ability to parse that +information out of file names anyways regardless of what the API does. Thus it makes +sense to keep that functionality inside of the clients. + + +Why Content Negotiation instead of multiple URLs? +------------------------------------------------- + +Another reasonable way to implement this would be to duplicate the API routes and +include some marker in the URL itself for JSON. Such as making the URLs be something +like ``/simple/foo.json``, ``/simple/_index.json``, etc. + +This makes some things simpler like TUF integration and fully static serving of a +repository (since ``.json`` files can just be written out). + +However, this is two pretty major issues: + +- Our current URL structure relies on the fact that there is an URL that represents + the "root", ``/`` to serve the list of projects. If we want to have separate URLs + for JSON and HTML, we would need to come up with some way to have two root URLs. + + Something like ``/`` being HTML and ``/_index.json`` being JSON, since ``_index`` + isn't a valid project name could work. But ``/`` being HTML doesn't work great if + a repository wants to remove support for HTML. + + Another option could be moving all of the existing HTML URLs under a namespace while + making a new namespace for JSON. Since ``//`` was defined, we would have to + make these namespaces not valid project names, so something like ``/_html/`` and + ``/_json/`` could work, then just redirect the non namespaced URLs to whatever the + "default" for that repository is (likely HTML, unless they've disabled HTML then JSON). +- With separate URLs, there's no good way to support zero configuration discovery + that a repository supports the JSON URLs without making additional HTTP requests to + determine if the JSON URL exists or not. + + The most naive implementation of this would be to request the JSON URL and fall back + to the HTML URL for *every* single request, but that would be horribly performant + and violate the goal of minimal additional HTTP requests. + + The most likely implementation of this would be to make some sort of repository level + configuration file that somehow indicates what is supported. We would have the same + namespace problem as above, with the same solution, something like ``/_config.json`` + or so could hold that data, and a client could first make an HTTP request to that, + and if it exists pull it down and parse it to learn about the capabilities of this + particular repository. +- The use of ``Accept`` also allows us to add versioning into this field + +All being said, it is the opinion of this PEP that those three issues combined make +using separate API routes a less desirable solution than relying on content +negotiation to select the most ideal representation of the data. + + +Does this mean that static servers are no longer supported? +----------------------------------------------------------- + +In short, no, static servers are still (almost) fully supported by this PEP. + +The specifics of how they are supported will depend on the static server in +question. For example: + +- **S3:** S3 fully supports custom content types, however it does not support + any form of content negotiation. In order to have a server hosted on S3, you + would have to use the "Endpoint configuration" style of negotiation, and + users would have to configure their clients explicitly. +- **GitHub Pages:** GitHub pages does not support custom content types, so the + S3 solution is not currently workable, which means that only ``text/html`` + repositories would function. +- **Apache:** Apache fully supports server-driven content negotiation, and would + just need to be configured to map the custom content types to specific extension. + + +Why not add an ``application/json`` alias like ``text/html``? +------------------------------------------------------------- + +This PEP believes that it is best for both clients and servers to be explicit +about the types of the API responses that are being used, and a content type +like ``application/json`` is the exact opposite of explicit. + +The existence of the ``text/html`` alias exists as a compromise primarily to +ensure that existing consumers of the API continue to function as they already +do. There is no such expectation of existing clients using the Simple API with +a ``application/json`` content type. + +In addition, ``application/json`` has no versioning in it, which means that +if there is ever a ``2.x`` version of the Simple API, we will be forced to make +a decision. Should ``application/json`` preserve backwards compatibility and +continue to be an alias for ``application/vnd.pypi.simple.v1+json``, or should +it be updated to be an alias for ``application/vnd.pypi.simple.v2+json``? + +This problem doesn't exist for ``text/html``, because the assumption is that +HTML will remain a legacy format, and will likely not gain *any* new features, +much less features that require breaking compatibility. So having it be an +alias for ``application/vnd.pypi.simple.v1+html`` is effectively the same as +having it be an alias for ``application/vnd.pypi.simple.latest+html``, since +``1.x`` will likely be the only HTML version to exist. + +The largest benefit to adding the ``application/json`` content type is that +there do things that do not allow you to have custom content types, and require +you to select one of their preset content types. The main example of this being +GitHub Pages, which the lack of ``application/json`` support in this PEP means +that static repositories will no longer be able to be hosted on GitHub Pages +unless GitHub adds the ``application/vnd.pypi.simple.v1+json`` content type. + +This PEP believes that the benefits are not large enough to add that content +type alias at this time, and that its inclusion would likely be a footgun +waiting for unsuspecting people to accidentally pick it up. Especially given +that we can always add it in the future, but removing things is a lot harder +to do. + + +Why add a ``application/vnd.pypi.simple.v1+html``? +-------------------------------------------------- + +The PEP expects the HTML version of the API to become legacy, so one option it +could take is not add the ``application/vnd.pypi.simple.v1+html`` content type, +and just use ``text/html`` for that. + +This PEP has decided that adding the new content type is better overall, since it +makes even the legacy format more self describing and makes them both more consistent +with each other. Overall I think it's more confusing if the ``+html`` version doesn't +exist. + + +Why v1.0 and not v1.1 or v2.0? +------------------------------ + +This PEP is still wholly backwards compatible with clients that could read the +existing v1.0 API, can still continue to read the API after these changes have +been made. In :pep:`629`, the qualification for a major version bump is: + + Incrementing the major version is used to signal a backwards incompatible + change such that existing clients would no longer be expected to be able to + meaningfully use the API. + +The changes in this PEP do not meet that bar, nothing has changed in a way that +existing clients would no longer be expected to be able to meaningfully use the +API. + +That means we should still be within the v1.x version line. + +The question of whether we should be v1.1 or v1.0 is a more interesting one, and +there are a few ways of looking at it: + +- We've exposed new features to the API (the project name on the project + page, multiple hashes), which is a sign that we should increment the minor + version. +- The new features exist wholly within the JSON serialization, which means that + no client that currently is requesting the HTML 1.0 page, would ever see any + of the new features anyways, so for them it is effectively still v1.0. +- No major client has implemented support for PEP 629 yet, which means that the + minor version numbering is largely academic at this point anyways, since it + exists to let clients provide feedback to end users. + +The second and third points above end up making the first point kind of +meaningless, and with that, it makes more sense to just call everything v1.0 +and be stricter about updating versions into the future. + + +Appendix 1: Survey of use cases to cover +======================================== + +This was done through a discussion between ``pip``, ``PyPI``, and ``bandersnarch`` +maintainers, who are the two first potential users for the new API. This is +how they use the Simple + JSON APIs today or how they currently plan to use it: + +- ``pip``: + + - List of all files for a particular release + - Metadata of each individual artifact: + + - was it yanked? (``data-yanked``) + - what's the python-requires? (``data-python-requires``) + - what's the hash of this file? (currently, hash in URL) + - Full metadata (``data-dist-info-metadata``) + - [Bonus] what are the declared dependencies, if available (list-of-strings, null if unavailable)? + +- ``bandersnatch`` - Only uses legacy JSON API + XMLRPC today: + + - Generates Simple HTML rather than copying from PyPI + + - Maybe this changes with the new API and we verbatim pull these API assets from PyPI + + - List of all files for a particular release. + + - Workout URL for release files to download + + - Metadata of each individual artifact. + + - Write out the JSON to mirror storage today (disk/S3) + + - Required metadata used + (via `Package class `__): + + - ``metadata["info"]`` + - ``metadata["last_serial"]`` + - ``metadata["releases"]`` + + - digests + - URL + + - XML-RPC calls (we'd love to deprecate - but we don't think should go in the Simple API) + + - [Bonus] Get packages since serial X (or all) + + - XML-RPC Call: ``changelog_since_serial`` + + - [Bonus] Get all packages with serial + + - XML-RPC Call: ``list_packages_with_serial`` + + +Appendix 2: Rough Underlying Data Models +======================================== + +These are not intended to perfectly match the server, client, or wire +formats. Rather, these are conceptual models, put to code to make them +more explicit as to the abstract models underlining the repository API +as it evolved through :pep:`503`, :pep:`529`, :pep:`629`, :pep:`658`, +and now this PEP, :pep:`691`. + +The existing HTML, and the new JSON serialization of these models then +represent how these underlying conceptual models get mapped onto the +actual wire formats. + +How servers or clients choose to model this data is out of scope for +this PEP. + +.. code-block:: python + + @dataclass + class Meta: + api_version: Literal["1.0"] + + + @dataclass + class Project: + # Normalized or Non-Normalized Name + name: str + # Computed in JSON, Included in HTML + url: str | None + + + @dataclass + class File: + filename: str + url: str + # Limited to a len() of 1 in HTML + hashes: dict[str, str] + gpg_sig: bool | None + requires_python: str | None + + + @dataclass + class PEP529File(File): + yanked: bool | str + + @dataclass + class PEP658File(PEP529File): + # Limited to a len() of 1 in HTML + dist_info_metadata: bool | dict[str, str] + + + # Simple Index page (/simple/) + @dataclass + class PEP503_Index: + projects: set[Project] + + + @dataclass + class PEP629_Index(PEP503_Index): + meta: Meta + + + @dataclass + class Index(PEP629_Index): + pass + + + # Simple Detail page (/simple/$PROJECT/) + @dataclass + class PEP503_Detail: + files: set[File] + + + @dataclass + class PEP529_Detail(PEP503_Detail): + files: set[PEP529File] + + + @dataclass + class PEP629_Detail(PEP529_Detail): + meta: Meta + + + @dataclass + class PEP658_Detail(PEP629_Detail): + files: set[PEP658File] + + + @dataclass + class PEP691_Detail(PEP658_Detail): + name: str # Normalized Name + + + @dataclass + class Detail(PEP691_Detail): + pass + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0692.rst b/peps/pep-0692.rst new file mode 100644 index 000000000..bb12250eb --- /dev/null +++ b/peps/pep-0692.rst @@ -0,0 +1,553 @@ +PEP: 692 +Title: Using TypedDict for more precise \*\*kwargs typing +Author: Franek Magiera +Sponsor: Jelle Zijlstra +Discussions-To: https://discuss.python.org/t/pep-692-using-typeddict-for-more-precise-kwargs-typing/17314 +Status: Accepted +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 29-May-2022 +Python-Version: 3.12 +Post-History: `29-May-2022 `__, + `12-Jul-2022 `__, + `12-Jul-2022 `__, +Resolution: https://discuss.python.org/t/pep-692-using-typeddict-for-more-precise-kwargs-typing/17314/81 + + +Abstract +======== + +Currently ``**kwargs`` can be type hinted as long as all of the keyword +arguments specified by them are of the same type. However, that behaviour can +be very limiting. Therefore, in this PEP we propose a new way to enable more +precise ``**kwargs`` typing. The new approach revolves around using +``TypedDict`` to type ``**kwargs`` that comprise keyword arguments of different +types. + +Motivation +========== + +Currently annotating ``**kwargs`` with a type ``T`` means that the ``kwargs`` +type is in fact ``dict[str, T]``. For example:: + + def foo(**kwargs: str) -> None: ... + +means that all keyword arguments in ``foo`` are strings (i.e., ``kwargs`` is +of type ``dict[str, str]``). This behaviour limits the ability to type +annotate ``**kwargs`` only to the cases where all of them are of the same type. +However, it is often the case that keyword arguments conveyed by ``**kwargs`` +have different types that are dependent on the keyword's name. In those cases +type annotating ``**kwargs`` is not possible. This is especially a problem for +already existing codebases where the need of refactoring the code in order to +introduce proper type annotations may be considered not worth the effort. This +in turn prevents the project from getting all of the benefits that type hinting +can provide. + +Moreover, ``**kwargs`` can be used to reduce the amount of code needed in +cases when there is a top-level function that is a part of a public API and it +calls a bunch of helper functions, all of which expect the same keyword +arguments. Unfortunately, if those helper functions were to use ``**kwargs``, +there is no way to properly type hint them if the keyword arguments they expect +are of different types. In addition, even if the keyword arguments are of the +same type, there is no way to check whether the function is being called with +keyword names that it actually expects. + +As described in the `Intended Usage`_ section, +using ``**kwargs`` is not always the best tool for the job. Despite that, it is +still a widely used pattern. As a consequence, there has been a lot of +discussion around supporting more precise ``**kwargs`` typing and it became a +feature that would be valuable for a large part of the Python community. This +is best illustrated by the `mypy GitHub issue 4441 `__ which +contains a lot of real world cases that could benefit from this propsal. + +One more use case worth mentioning for which ``**kwargs`` are also convenient, +is when a function should accommodate optional keyword-only arguments that +don't have default values. A need for a pattern like that can arise when values +that are usually used as defaults to indicate no user input, such as ``None``, +can be passed in by a user and should result in a valid, non-default behavior. +For example, this issue `came up `__ in the popular ``httpx`` library. + +Rationale +========= + +:pep:`589` introduced the ``TypedDict`` type constructor that supports dictionary +types consisting of string keys and values of potentially different types. A +function's keyword arguments represented by a formal parameter that begins with +double asterisk, such as ``**kwargs``, are received as a dictionary. +Additionally, such functions are often called using unpacked dictionaries to +provide keyword arguments. This makes ``TypedDict`` a perfect candidate to be +used for more precise ``**kwargs`` typing. In addition, with ``TypedDict`` +keyword names can be taken into account during static type analysis. However, +specifying ``**kwargs`` type with a ``TypedDict`` means, as mentioned earlier, +that each keyword argument specified by ``**kwargs`` is a ``TypedDict`` itself. +For instance:: + + class Movie(TypedDict): + name: str + year: int + + def foo(**kwargs: Movie) -> None: ... + +means that each keyword argument in ``foo`` is itself a ``Movie`` dictionary +that has a ``name`` key with a string type value and a ``year`` key with an +integer type value. Therefore, in order to support specifying ``kwargs`` type +as a ``TypedDict`` without breaking current behaviour, a new construct has to +be introduced. + +To support this use case, we propose reusing ``Unpack`` which +was initially introduced in :pep:`646`. There are several reasons for doing so: + +* Its name is quite suitable and intuitive for the ``**kwargs`` typing use case + as our intention is to "unpack" the keywords arguments from the supplied + ``TypedDict``. +* The current way of typing ``*args`` would be extended to ``**kwargs`` + and those are supposed to behave similarly. +* There would be no need to introduce any new special forms. +* The use of ``Unpack`` for the purposes described in this PEP does not + interfere with the use cases described in :pep:`646`. + +Specification +============= + +With ``Unpack`` we introduce a new way of annotating ``**kwargs``. +Continuing the previous example:: + + def foo(**kwargs: Unpack[Movie]) -> None: ... + +would mean that the ``**kwargs`` comprise two keyword arguments specified by +``Movie`` (i.e. a ``name`` keyword of type ``str`` and a ``year`` keyword of +type ``int``). This indicates that the function should be called as follows:: + + kwargs: Movie = {"name": "Life of Brian", "year": 1979} + + foo(**kwargs) # OK! + foo(name="The Meaning of Life", year=1983) # OK! + +When ``Unpack`` is used, type checkers treat ``kwargs`` inside the +function body as a ``TypedDict``:: + + def foo(**kwargs: Unpack[Movie]) -> None: + assert_type(kwargs, Movie) # OK! + + +Using the new annotation will not have any runtime effect - it should only be +taken into account by type checkers. Any mention of errors in the following +sections relates to type checker errors. + +Function calls with standard dictionaries +----------------------------------------- + +Passing a dictionary of type ``dict[str, object]`` as a ``**kwargs`` argument +to a function that has ``**kwargs`` annotated with ``Unpack`` must generate a +type checker error. On the other hand, the behaviour for functions using +standard, untyped dictionaries can depend on the type checker. For example:: + + def foo(**kwargs: Unpack[Movie]) -> None: ... + + movie: dict[str, object] = {"name": "Life of Brian", "year": 1979} + foo(**movie) # WRONG! Movie is of type dict[str, object] + + typed_movie: Movie = {"name": "The Meaning of Life", "year": 1983} + foo(**typed_movie) # OK! + + another_movie = {"name": "Life of Brian", "year": 1979} + foo(**another_movie) # Depends on the type checker. + +Keyword collisions +------------------ + +A ``TypedDict`` that is used to type ``**kwargs`` could potentially contain +keys that are already defined in the function's signature. If the duplicate +name is a standard parameter, an error should be reported by type checkers. +If the duplicate name is a positional-only parameter, no errors should be +generated. For example:: + + def foo(name, **kwargs: Unpack[Movie]) -> None: ... # WRONG! "name" will + # always bind to the + # first parameter. + + def foo(name, /, **kwargs: Unpack[Movie]) -> None: ... # OK! "name" is a + # positional-only parameter, + # so **kwargs can contain + # a "name" keyword. + +Required and non-required keys +------------------------------ + +By default all keys in a ``TypedDict`` are required. This behaviour can be +overridden by setting the dictionary's ``total`` parameter as ``False``. +Moreover, :pep:`655` introduced new type qualifiers - ``typing.Required`` and +``typing.NotRequired`` - that enable specifying whether a particular key is +required or not:: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + +When using a ``TypedDict`` to type ``**kwargs`` all of the required and +non-required keys should correspond to required and non-required function +keyword parameters. Therefore, if a required key is not supported by the +caller, then an error must be reported by type checkers. + +Assignment +---------- + +Assignments of a function typed with ``**kwargs: Unpack[Movie]`` and +another callable type should pass type checking only if they are compatible. +This can happen for the scenarios described below. + +Source and destination contain ``**kwargs`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Both destination and source functions have a ``**kwargs: Unpack[TypedDict]`` +parameter and the destination function's ``TypedDict`` is assignable to the +source function's ``TypedDict`` and the rest of the parameters are +compatible:: + + class Animal(TypedDict): + name: str + + class Dog(Animal): + breed: str + + def accept_animal(**kwargs: Unpack[Animal]): ... + def accept_dog(**kwargs: Unpack[Dog]): ... + + accept_dog = accept_animal # OK! Expression of type Dog can be + # assigned to a variable of type Animal. + + accept_animal = accept_dog # WRONG! Expression of type Animal + # cannot be assigned to a variable of type Dog. + +.. _PEP 692 assignment dest no kwargs: + +Source contains ``**kwargs`` and destination doesn't +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The destination callable doesn't contain ``**kwargs``, the source callable +contains ``**kwargs: Unpack[TypedDict]`` and the destination function's keyword +arguments are assignable to the corresponding keys in source function's +``TypedDict``. Moreover, not required keys should correspond to optional +function arguments, whereas required keys should correspond to required +function arguments. Again, the rest of the parameters have to be compatible. +Continuing the previous example:: + + class Example(TypedDict): + animal: Animal + string: str + number: NotRequired[int] + + def src(**kwargs: Unpack[Example]): ... + def dest(*, animal: Dog, string: str, number: int = ...): ... + + dest = src # OK! + +It is worth pointing out that the destination function's parameters that are to +be compatible with the keys and values from the ``TypedDict`` must be keyword +only:: + + def dest(dog: Dog, string: str, number: int = ...): ... + + dog: Dog = {"name": "Daisy", "breed": "labrador"} + + dest(dog, "some string") # OK! + + dest = src # Type checker error! + dest(dog, "some string") # The same call fails at + # runtime now because 'src' expects + # keyword arguments. + +The reverse situation where the destination callable contains +``**kwargs: Unpack[TypedDict]`` and the source callable doesn't contain +``**kwargs`` should be disallowed. This is because, we cannot be sure that +additional keyword arguments are not being passed in when an instance of a +subclass had been assigned to a variable with a base class type and then +unpacked in the destination callable invocation:: + + def dest(**kwargs: Unpack[Animal]): ... + def src(name: str): ... + + dog: Dog = {"name": "Daisy", "breed": "Labrador"} + animal: Animal = dog + + dest = src # WRONG! + dest(**animal) # Fails at runtime. + +Similar situation can happen even without inheritance as compatibility +between ``TypedDict``\s is based on structural subtyping. + +Source contains untyped ``**kwargs`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The destination callable contains ``**kwargs: Unpack[TypedDict]`` and the +source callable contains untyped ``**kwargs``:: + + def src(**kwargs): ... + def dest(**kwargs: Unpack[Movie]): ... + + dest = src # OK! + +Source contains traditionally typed ``**kwargs: T`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The destination callable contains ``**kwargs: Unpack[TypedDict]``, the source +callable contains traditionally typed ``**kwargs: T`` and each of the +destination function ``TypedDict``'s fields is assignable to a variable of +type ``T``:: + + class Vehicle: + ... + + class Car(Vehicle): + ... + + class Motorcycle(Vehicle): + ... + + class Vehicles(TypedDict): + car: Car + moto: Motorcycle + + def dest(**kwargs: Unpack[Vehicles]): ... + def src(**kwargs: Vehicle): ... + + dest = src # OK! + +On the other hand, if the destination callable contains either untyped or +traditionally typed ``**kwargs: T`` and the source callable is typed using +``**kwargs: Unpack[TypedDict]`` then an error should be generated, because +traditionally typed ``**kwargs`` aren't checked for keyword names. + +To summarize, function parameters should behave contravariantly and function +return types should behave covariantly. + +Passing kwargs inside a function to another function +---------------------------------------------------- + +`A previous point `_ +mentions the problem of possibly passing additional keyword arguments by +assigning a subclass instance to a variable that has a base class type. Let's +consider the following example:: + + class Animal(TypedDict): + name: str + + class Dog(Animal): + breed: str + + def takes_name(name: str): ... + + dog: Dog = {"name": "Daisy", "breed": "Labrador"} + animal: Animal = dog + + def foo(**kwargs: Unpack[Animal]): + print(kwargs["name"].capitalize()) + + def bar(**kwargs: Unpack[Animal]): + takes_name(**kwargs) + + def baz(animal: Animal): + takes_name(**animal) + + def spam(**kwargs: Unpack[Animal]): + baz(kwargs) + + foo(**animal) # OK! foo only expects and uses keywords of 'Animal'. + + bar(**animal) # WRONG! This will fail at runtime because 'breed' keyword + # will be passed to 'takes_name' as well. + + spam(**animal) # WRONG! Again, 'breed' keyword will be eventually passed + # to 'takes_name'. + +In the example above, the call to ``foo`` will not cause any issues at +runtime. Even though ``foo`` expects ``kwargs`` of type ``Animal`` it doesn't +matter if it receives additional arguments because it only reads and uses what +it needs completely ignoring any additional values. + +The calls to ``bar`` and ``spam`` will fail because an unexpected keyword +argument will be passed to the ``takes_name`` function. + +Therefore, ``kwargs`` hinted with an unpacked ``TypedDict`` can only be passed +to another function if the function to which unpacked kwargs are being passed +to has ``**kwargs`` in its signature as well, because then additional keywords +would not cause errors at runtime during function invocation. Otherwise, the +type checker should generate an error. + +In cases similar to the ``bar`` function above the problem could be worked +around by explicitly dereferencing desired fields and using them as arguments +to perform the function call:: + + def bar(**kwargs: Unpack[Animal]): + name = kwargs["name"] + takes_name(name) + +Using ``Unpack`` with types other than ``TypedDict`` +---------------------------------------------------- + +As described in the Rationale_ section, +``TypedDict`` is the most natural candidate for typing ``**kwargs``. +Therefore, in the context of typing ``**kwargs``, using ``Unpack`` with types +other than ``TypedDict`` should not be allowed and type checkers should +generate errors in such cases. + +Changes to ``Unpack`` +--------------------- + +Currently using ``Unpack`` in the context of +typing is interchangeable with using the asterisk syntax:: + + >>> Unpack[Movie] + * + +Therefore, in order to be compatible with the new use case, ``Unpack``'s +``repr`` should be changed to simply ``Unpack[T]``. + +Intended Usage +============== +The intended use cases for this proposal are described in the +Motivation_ section. In summary, more precise ``**kwargs`` typing +can bring benefits to already existing codebases that decided to use +``**kwargs`` initially, but now are mature enough to use a stricter contract +via type hints. Using ``**kwargs`` can also help in reducing code duplication +and the amount of copy-pasting needed when there is a bunch of functions that +require the same set of keyword arguments. Finally, ``**kwargs`` are useful for +cases when a function needs to facilitate optional keyword arguments that don't +have obvious default values. + +However, it has to be pointed out that in some cases there are better tools +for the job than using ``TypedDict`` to type ``**kwargs`` as proposed in this +PEP. For example, when writing new code if all the keyword arguments are +required or have default values then writing everything explicitly is better +than using ``**kwargs`` and a ``TypedDict``:: + + def foo(name: str, year: int): ... # Preferred way. + def foo(**kwargs: Unpack[Movie]): ... + +Similarly, when type hinting third party libraries via stubs it is again better +to state the function signature explicitly - this is the only way to type such +a function if it has default arguments. Another issue that may arise in this +case when trying to type hint the function with a ``TypedDict`` is that some +standard function parameters may be treated as keyword only:: + + def foo(name, year): ... # Function in a third party library. + + def foo(Unpack[Movie]): ... # Function signature in a stub file. + + foo("Life of Brian", 1979) # This would be now failing type + # checking but is fine. + + foo(name="Life of Brian", year=1979) # This would be the only way to call + # the function now that passes type + # checking. + +Therefore, in this case it is again preferred to type hint such function +explicitly as:: + + def foo(name: str, year: int): ... + +Also, for the benefit of IDEs and documentation pages, functions that are part +of the public API should prefer explicit keyword parameters whenever possible. + +How to Teach This +================= + +This PEP could be linked in the ``typing`` module's documentation. Moreover, a +new section on using ``Unpack`` could be added to the aforementioned docs. +Similar sections could be also added to the +`mypy documentation `_ and the +`typing RTD documentation `_. + +Reference Implementation +======================== + +The `mypy type checker `_ already +`supports `_ more precise +``**kwargs`` typing using ``Unpack``. + +`Pyright type checker `_ also +`provides provisional support `__ +for `this feature `__. + +Rejected Ideas +============== + +``TypedDict`` unions +-------------------- + +It is possible to create unions of typed dictionaries. However, supporting +typing ``**kwargs`` with a union of typed dicts would greatly increase the +complexity of the implementation of this PEP and there seems to be no +compelling use case to justify the support for this. Therefore, using unions of +typed dictionaries to type ``**kwargs`` as described in the context of this PEP +can result in an error:: + + class Book(TypedDict): + genre: str + pages: int + + TypedDictUnion = Movie | Book + + def foo(**kwargs: Unpack[TypedDictUnion]) -> None: ... # WRONG! Unsupported use + # of a union of + # TypedDicts to type + # **kwargs + +Instead, a function that expects a union of ``TypedDict``\s can be +overloaded:: + + @overload + def foo(**kwargs: Unpack[Movie]): ... + + @overload + def foo(**kwargs: Unpack[Book]): ... + +Changing the meaning of ``**kwargs`` annotations +------------------------------------------------ + +One way to achieve the purpose of this PEP would be to change the +meaning of ``**kwargs`` annotations, so that the annotations would +apply to the entire ``**kwargs`` dict, not to individual elements. +For consistency, we would have to make an analogous change to ``*args`` +annotations. + +This idea was discussed in a meeting of the typing community, and the +consensus was that the change would not be worth the cost. There is no +clear migration path, the current meaning of ``*args`` and ``**kwargs`` +annotations is well-established in the ecosystem, and type checkers +would have to introduce new errors for code that is currently legal. + +Introducing a new syntax +------------------------ + +In the previous versions of this PEP, using a double asterisk syntax was +proposed to support more precise ``**kwargs`` typing. Using this syntax, +functions could be annotated as follows:: + + def foo(**kwargs: **Movie): ... + +Which would have the same meaning as:: + + def foo(**kwargs: Unpack[Movie]): ... + +This greatly increased the scope of the PEP, as it would require a grammar +change and adding a new dunder for the ``Unpack`` special form. At the same +the justification for introducing a new syntax was not strong enough and +became a blocker for the whole PEP. Therefore, we decided to abandon the idea +of introducing a new syntax as a part of this PEP and may propose it again in a +separate one. + +References +========== + +.. _httpxIssue1384: https://github.com/encode/httpx/issues/1384 +.. _mypyIssue4441: https://github.com/python/mypy/issues/4441 +.. _pyrightIssue3002: https://github.com/microsoft/pyright/issues/3002 +.. _pyrightProvisionalImplementation: https://github.com/microsoft/pyright/commit/5bee749eb171979e3f526cd8e5bf66b00593378a + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0693.rst b/peps/pep-0693.rst new file mode 100644 index 000000000..092c798e2 --- /dev/null +++ b/peps/pep-0693.rst @@ -0,0 +1,93 @@ +PEP: 693 +Title: Python 3.12 Release Schedule +Author: Thomas Wouters +Status: Active +Type: Informational +Topic: Release +Content-Type: text/x-rst +Created: 24-May-2022 +Python-Version: 3.12 + + +Abstract +======== + +This document describes the development and release schedule for +Python 3.12. The schedule primarily concerns itself with PEP-sized +items. + +.. Small features may be added up to the first beta + release. Bugs may be fixed until the final release, + which is planned for October 2023. + + +Release Manager and Crew +======================== + +- 3.12 Release Manager: Thomas Wouters +- Windows installers: Steve Dower +- Mac installers: Ned Deily +- Documentation: Julien Palard + + +Release Schedule +================ + +3.12.0 schedule +--------------- + +Note: the dates below use a 17-month development period that results +in a 12-month release cadence between feature versions, as defined by +:pep:`602`. + +Actual: + +- 3.12 development begins: Sunday, 2022-05-08 +- 3.12.0 alpha 1: Monday, 2022-10-24 +- 3.12.0 alpha 2: Monday, 2022-11-14 +- 3.12.0 alpha 3: Tuesday, 2022-12-06 +- 3.12.0 alpha 4: Tuesday, 2023-01-10 +- 3.12.0 alpha 5: Tuesday, 2023-02-07 +- 3.12.0 alpha 6: Tuesday, 2023-03-07 +- 3.12.0 alpha 7: Tuesday, 2023-04-04 +- 3.12.0 beta 1: Monday, 2023-05-22 + (No new features beyond this point.) +- 3.12.0 beta 2: Tuesday, 2023-06-06 +- 3.12.0 beta 3: Monday, 2023-06-19 +- 3.12.0 beta 4: Tuesday, 2023-07-11 +- 3.12.0 candidate 1: Sunday, 2023-08-06 +- 3.12.0 candidate 2: Wednesday, 2023-09-06 +- 3.12.0 candidate 3: Tuesday, 2023-09-19 + +Expected: + +- 3.12.0 final: Monday, 2023-10-02 + +Subsequent bugfix releases every two months. + + +3.12 Lifespan +------------- + +3.12 will receive bugfix updates approximately every 2 months for +approximately 18 months. Some time after the release of 3.13.0 final, +the ninth and final 3.12 bugfix update will be released. After that, +it is expected that security updates (source only) will be released +until 5 years after the release of 3.12.0 final, so until approximately +October 2028. + + +Features for 3.12 +================= + +New features can be found in `What’s New In Python 3.12 +`__. + + +Copyright +========= + +This document is placed in the public domain or under the CC0-1.0-Universal +license, whichever is more permissive. + + diff --git a/peps/pep-0694.rst b/peps/pep-0694.rst new file mode 100644 index 000000000..1f5471222 --- /dev/null +++ b/peps/pep-0694.rst @@ -0,0 +1,732 @@ +PEP: 694 +Title: Upload 2.0 API for Python Package Repositories +Author: Donald Stufft +Discussions-To: https://discuss.python.org/t/pep-694-upload-2-0-api-for-python-package-repositories/16879 +Status: Draft +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 11-Jun-2022 +Post-History: `27-Jun-2022 `__ + + +Abstract +======== + +There is currently no standardized API for uploading files to a Python package +repository such as PyPI. Instead, everyone has been forced to reverse engineer +the non-standard API from PyPI. + +That API, while functional, leaks a lot of implementation details of the original +PyPI code base, which have now had to have been faithfully replicated in the new +code base, and alternative implementations. + +Beyond the above, there are a number of major issues with the current API: + +- It is a fully synchronous API, which means that we're forced to have a single + request being held open for potentially a long time, both for the upload itself, + and then while the repository processes the uploaded file to determine success + or failure. + +- It does not support any mechanism for resuming an upload, with the largest file + size on PyPI being just under 1GB in size, that's a lot of wasted bandwidth if + a large file has a network blip towards the end of an upload. + +- It treats a single file as the atomic unit of operation, which can be problematic + when a release might have multiple binary wheels which can cause people to get + different versions while the files are uploading, and if the sdist happens to + not go last, possibly some hard to build packages are attempting to be built + from source. + +- It has very limited support for communicating back to the user, with no support + for multiple errors, warnings, deprecations, etc. It is limited entirely to the + HTTP status code and reason phrase, of which the reason phrase has been + deprecated since HTTP/2 (:rfc:`RFC 7540 <7540#section-8.1.2.4>`). + +- The metadata for a release/file is submitted alongside the file, however this + metadata is famously unreliable, and most installers instead choose to download + the entire file and read that in part due to that unreliability. + +- There is no mechanism for allowing a repository to do any sort of sanity + checks before bandwidth starts getting expended on an upload, whereas a lot + of the cases of invalid metadata or incorrect permissions could be checked + prior to upload. + +- It has no support for "staging" a draft release prior to publishing it to the + repository. + +- It has no support for creating new projects, without uploading a file. + +This PEP proposes a new API for uploads, and deprecates the existing non standard +API. + + +Status Quo +========== + +This does not attempt to be a fully exhaustive documentation of the current API, but +give a high level overview of the existing API. + + +Endpoint +-------- + +The existing upload API (and the now removed register API) lives at an url, currently +``https://upload.pypi.org/legacy/``, and to communicate which specific API you want +to call, you add a ``:action`` url parameter with a value of ``file_upload``. The values +of ``submit``, ``submit_pkg_info``, and ``doc_upload`` also used to be supported, but +no longer are. + +It also has a ``protocol_version`` parameter, in theory to allow new versions of the +API to be written, but in practice that has never happened, and the value is always +``1``. + +So in practice, on PyPI, the endpoint is +``https://upload.pypi.org/legacy/?:action=file_upload&protocol_version=1``. + + + +Encoding +-------- + +The data to be submitted is submitted as a ``POST`` request with the content type +of ``multipart/form-data``. This is due to the historical nature, that this API +was not actually designed as an API, but rather was a form on the initial PyPI +implementation, then client code was written to programmatically submit that form. + + +Content +------- + +Roughly speaking, the metadata contained within the package is submitted as parts +where the content-disposition is ``form-data``, and the name is the name of the +field. The names of these various pieces of metadata are not documented, and they +sometimes, but not always match the names used in the ``METADATA`` files. The casing +rarely matches though, but overall the ``METADATA`` to ``form-data`` conversion is +extremely inconsistent. + +The file itself is then sent as a ``application/octet-stream`` part with the name +of ``content``, and if there is a PGP signature attached, then it will be included +as a ``application/octet-stream`` part with the name of ``gpg_signature``. + + +Specification +============= + +This PEP traces the root cause of most of the issues with the existing API to be +roughly two things: + +- The metadata is submitted alongside the file, rather than being parsed from the + file itself. + + - This is actually fine if used as a pre-check, but it should be validated + against the actual ``METADATA`` or similar files within the distribution. + +- It supports a single request, using nothing but form data, that either succeeds + or fails, and everything is done and contained within that single request. + +We then propose a multi-request workflow, that essentially boils down to: + +1. Initiate an upload session. +2. Upload the file(s) as part of the upload session. +3. Complete the upload session. +4. (Optional) Check the status of an upload session. + +All URLs described here will be relative to the root endpoint, which may be +located anywhere within the url structure of a domain. So it could be at +``https://upload.example.com/``, or ``https://example.com/upload/``. + + +Versioning +---------- + +This PEP uses the same ``MAJOR.MINOR`` versioning system as used in :pep:`691`, +but it is otherwise independently versioned. The existing API is considered by +this spec to be version ``1.0``, but it otherwise does not attempt to modify +that API in any way. + + +Endpoints +--------- + +Create an Upload Session +~~~~~~~~~~~~~~~~~~~~~~~~ + +To create a new upload session, you can send a ``POST`` request to ``/``, +with a payload that looks like: + +.. code-block:: json + + { + "meta": { + "api-version": "2.0" + }, + "name": "foo", + "version": "1.0" + } + + +This currently has three keys, ``meta``, ``name``, and ``version``. + +The ``meta`` key is included in all payloads, and it describes information about the +payload itself. + +The ``name`` key is the name of the project that this session is attempting to +add files to. + +The ``version`` key is the version of the project that this session is attepmting to +add files to. + +If creating the session was successful, then the server must return a response +that looks like: + +.. code-block:: json + + { + "meta": { + "api-version": "2.0" + }, + "urls": { + "upload": "...", + "draft": "...", + "publish": "..." + }, + "valid-for": 604800, + "status": "pending", + "files": {}, + "notices": [ + "a notice to display to the user" + ] + } + + +Besides the ``meta`` key, this response has five keys, ``urls``, ``valid-for``, +``status``, ``files``, and ``notices``. + +The ``urls`` key is a dictionary mapping identifiers to related URLs to this +session. + +The ``valid-for`` key is an integer representing how long, in seconds, until the +server itself will expire this session (and thus all of the URLs contained in it). +The session **SHOULD** live at least this much longer unless the client itself +has canceled the session. Servers **MAY** choose to *increase* this time, but should +never *decrease* it, except naturally through the passage of time. + +The ``status`` key is a string that contains one of ``pending``, ``published``, +``errored``, or ``canceled``, this string represents the overall status of +the session. + +The ``files`` key is a mapping containing the filenames that have been uploaded +to this session, to a mapping containing details about each file. + +The ``notices`` key is an optional key that points to an array of notices that +the server wishes to communicate to the end user that are not specific to any +one file. + +For each filename in ``files`` the mapping has three keys, ``status``, ``url``, +and ``notices``. + +The ``status`` key is the same as the top level ``status`` key, except that it +indicates the status of a specific file. + +The ``url`` key is the *absolute* URL that the client should upload that specific +file to (or use to delete that file). + +The ``notices`` key is an optional key, that is an array of notices that the server +wishes to communicate to the end user that are specific to this file. + +The required response code to a successful creation of the session is a +``201 Created`` response and it **MUST** include a ``Location`` header that is the +URL for this session, which may be used to check its status or cancel it. + +For the ``urls`` key, there are currently three keys that may appear: + +The ``upload`` key, which is the upload endpoint for this session to initiate +a file upload. + +The ``draft`` key, which is the repository URL that these files are available at +prior to publishing. + +The ``publish`` key, which is the endpoint to trigger publishing the session. + + +In addition to the above, if a second session is created for the same name+version +pair, then the upload server **MUST** return the already existing session rather +than creating a new, empty one. + + +Upload Each File +~~~~~~~~~~~~~~~~ + +Once you have initiated an upload session for one or more files, then you have +to actually upload each of those files. + +There is no set endpoint for actually uploading the file, that is given to the +client by the server as part of the creation of the upload session, and clients +**MUST NOT** assume that there is any commonality to what those URLs look like from +one session to the next. + +To initiate a file upload, a client sends a ``POST`` request to the upload URL +in the session, with a request body that looks like: + +.. code-block:: json + + { + "meta": { + "api-version": "2.0" + }, + "filename": "foo-1.0.tar.gz", + "size": 1000, + "hashes": {"sha256": "...", "blake2b": "..."}, + "metadata": "..." + } + + +Besides the standard ``meta`` key, this currently has 4 keys: + +- ``filename``: The filename of the file being uploaded. +- ``size``: The size, in bytes, of the file that is being uploaded. +- ``hashes``: A mapping of hash names to hex encoded digests, each of these digests + are the digests of that file, when hashed by the hash identified in the name. + + By default, any hash algorithm available via `hashlib + `_ (specifically any that can + be passed to ``hashlib.new()`` and do not require additional parameters) can + be used as a key for the hashes dictionary. At least one secure algorithm from + ``hashlib.algorithms_guaranteed`` **MUST** always be included. At the time + of this PEP, ``sha256`` specifically is recommended. + + Multiple hashes may be passed at a time, but all hashes must be valid for the + file. +- ``metadata``: An optional key that is a string containing the file's + `core metadata `_. + +Servers **MAY** use the data provided in this response to do some sanity checking +prior to allowing the file to be uploaded, which may include but is not limited +to: + +- Checking if the ``filename`` already exists. +- Checking if the ``size`` would invalidate some quota. +- Checking if the contents of the ``metadata``, if provided, are valid. + +If the server determines that the client should attempt the upload, it will return +a ``201 Created`` response, with an empty body, and a ``Location`` header pointing +to the URL that the file itself should be uploaded to. + +At this point, the status of the session should show the filename, with the above url +included in it. + + +Upload Data ++++++++++++ + +To upload the file, a client has two choices, they may upload the file as either +a single chunk, or as multiple chunks. Either option is acceptable, but it is +recommended that most clients should choose to upload each file as a single chunk +as that requires fewer requests and typically has better performance. + +However for particularly large files, uploading within a single request may result +in timeouts, so larger files may need to be uploaded in multiple chunks. + +In either case, the client must generate a unique token (or nonce) for each upload +attempt for a file, and **MUST** include that token in each request in the ``Upload-Token`` +header. The ``Upload-Token`` is a binary blob encoded using base64 surrounded by +a ``:`` on either side. Clients **SHOULD** use at least 32 bytes of cryptographically +random data. You can generate it using the following: + +.. code-block:: python + + import base64 + import secrets + + header = ":" + base64.b64encode(secrets.token_bytes(32)).decode() + ":" + +The one time that it is permissible to omit the ``Upload-Token`` from an upload +request is when a client wishes to opt out of the resumable or chunked file upload +feature completely. In that case, they **MAY** omit the ``Upload-Token``, and the +file must be successfully uploaded in a single HTTP request, and if it fails, the +entire file must be resent in another single HTTP request. + +To upload in a single chunk, a client sends a ``POST`` request to the URL from the +session response for that filename. The client **MUST** include a ``Content-Length`` +header that is equal to the size of the file in bytes, and this **MUST** match the +size given in the original session creation. + +As an example, if uploading a 100,000 byte file, you would send headers like:: + + Content-Length: 100000 + Upload-Token: :nYuc7Lg2/Lv9S4EYoT9WE6nwFZgN/TcUXyk9wtwoABg=: + +If the upload completes successfully, the server **MUST** respond with a +``201 Created`` status. At this point this file **MUST** not be present in the +repository, but merely staged until the upload session has completed. + +To upload in multiple chunks, a client sends multiple ``POST`` requests to the same +URL as before, one for each chunk. + +This time however, the ``Content-Length`` is equal to the size, in bytes, of the +chunk that they are sending. In addition, the client **MUST** include a +``Upload-Offset`` header which indicates a byte offset that the content included +in this request starts at and a ``Upload-Incomplete`` header set to ``1``. + +As an example, if uploading a 100,000 byte file in 1000 byte chunks, and this chunk +represents bytes 1001 through 2000, you would send headers like:: + + Content-Length: 1000 + Upload-Token: :nYuc7Lg2/Lv9S4EYoT9WE6nwFZgN/TcUXyk9wtwoABg=: + Upload-Offset: 1001 + Upload-Incomplete: 1 + +However, the **final** chunk of data omits the ``Upload-Incomplete`` header, since +at that point the upload is no longer incomplete. + +For each successful chunk, the server **MUST** respond with a ``202 Accepted`` +header, except for the final chunk, which **MUST** be a ``201 Created``. + +The following constraints are placed on uploads regardless of whether they are +single chunk or multiple chunks: + +- A client **MUST NOT** perform multiple ``POST`` requests in parallel for the + same file to avoid race conditions and data loss or corruption. The server + **MAY** terminate any ongoing ``POST`` request that utilizes the same + ``Upload-Token``. +- If the offset provided in ``Upload-Offset`` is not ``0`` or the next chunk + in an incomplete upload, then the server **MUST** respond with a 409 Conflict. +- Once an upload has started with a specific token, you may not use another token + for that file without deleting the in progress upload. +- Once a file has uploaded successfully, you may initiate another upload for + that file, and doing so will replace that file. + + +Resume Upload ++++++++++++++ + +To resume an upload, you first have to know how much of the data the server has +already received, regardless of if you were originally uploading the file as +a single chunk, or in multiple chunks. + +To get the status of an individual upload, a client can make a ``HEAD`` request +with their existing ``Upload-Token`` to the same URL they were uploading to. + +The server **MUST** respond back with a ``204 No Content`` response, with an +``Upload-Offset`` header that indicates what offset the client should continue +uploading from. If the server has not received any data, then this would be ``0``, +if it has received 1007 bytes then it would be ``1007``. + +Once the client has retrieved the offset that they need to start from, they can +upload the rest of the file as described above, either in a single request +containing all of the remaining data or in multiple chunks. + + +Canceling an In Progress Upload ++++++++++++++++++++++++++++++++ + +If a client wishes to cancel an upload of a specific file, for instance because +they need to upload a different file, they may do so by issuing a ``DELETE`` +request to the file upload URL with the ``Upload-Token`` used to upload the +file in the first place. + +A successful cancellation request **MUST** response with a ``204 No Content``. + + +Delete an uploaded File ++++++++++++++++++++++++ + +Already uploaded files may be deleted by issuing a ``DELETE`` request to the file +upload URL without the ``Upload-Token``. + +A successful deletion request **MUST** response with a ``204 No Content``. + + +Session Status +~~~~~~~~~~~~~~ + +Similarly to file upload, the session URL is provided in the response to +creating the upload session, and clients **MUST NOT** assume that there is any +commonality to what those URLs look like from one session to the next. + +To check the status of a session, clients issue a ``GET`` request to the +session URL, to which the server will respond with the same response that +they got when they initially created the upload session, except with any +changes to ``status``, ``valid-for``, or updated ``files`` reflected. + + +Session Cancellation +~~~~~~~~~~~~~~~~~~~~ + +To cancel an upload session, a client issues a ``DELETE`` request to the +same session URL as before. At which point the server marks the session as +canceled, **MAY** purge any data that was uploaded as part of that session, +and future attempts to access that session URL or any of the file upload URLs +**MAY** return a ``404 Not Found``. + +To prevent a lot of dangling sessions, servers may also choose to cancel a +session on their own accord. It is recommended that servers expunge their +sessions after no less than a week, but each server may choose their own +schedule. + + +Session Completion +~~~~~~~~~~~~~~~~~~ + +To complete a session, and publish the files that have been included in it, +a client **MUST** send a ``POST`` request to the ``publish`` url in the +session status payload. + +If the server is able to immediately complete the session, it may do so +and return a ``201 Created`` response. If it is unable to immediately +complete the session (for instance, if it needs to do processing that may +take longer than reasonable in a single HTTP request), then it may return +a ``202 Accepted`` response. + +In either case, the server should include a ``Location`` header pointing +back to the session status url, and if the server returned a ``202 Accepted``, +the client may poll that URL to watch for the status to change. + + +Errors +------ + +All Error responses that contain a body will have a body that looks like: + +.. code-block:: json + + { + "meta": { + "api-version": "2.0" + }, + "message": "...", + "errors": [ + { + "source": "...", + "message": "..." + } + ] + } + +Besides the standard ``meta`` key, this has two top level keys, ``message`` +and ``errors``. + +The ``message`` key is a singular message that encapsulates all errors that +may have happened on this request. + +The ``errors`` key is an array of specific errors, each of which contains +a ``source`` key, which is a string that indicates what the source of the +error is, and a ``messasge`` key for that specific error. + +The ``message`` and ``source`` strings do not have any specific meaning, and +are intended for human interpretation to figure out what the underlying issue +was. + + +Content-Types +------------- + +Like :pep:`691`, this PEP proposes that all requests and responses from the +Upload API will have a standard content type that describes what the content +is, what version of the API it represents, and what serialization format has +been used. + +The structure of this content type will be: + +.. code-block:: text + + application/vnd.pypi.upload.$version+format + +Since only major versions should be disruptive to systems attempting to +understand one of these API content bodies, only the major version will be +included in the content type, and will be prefixed with a ``v`` to clarify +that it is a version number. + +Unlike :pep:`691`, this PEP does not change the existing ``1.0`` API in any +way, so servers will be required to host the new API described in this PEP at +a different endpoint than the existing upload API. + +Which means that for the new 2.0 API, the content types would be: + +- **JSON:** ``application/vnd.pypi.upload.v2+json`` + +In addition to the above, a special "meta" version is supported named ``latest``, +whose purpose is to allow clients to request the absolute latest version, without +having to know ahead of time what that version is. It is recommended however, +that clients be explicit about what versions they support. + +These content types **DO NOT** apply to the file uploads themselves, only to the +other API requests/responses in the upload API. The files themselves should use +the ``application/octet-stream`` content-type. + + +Version + Format Selection +-------------------------- + +Again similar to :pep:`691`, this PEP standardizes on using server-driven +content negotiation to allow clients to request different versions or +serialization formats, which includes the ``format`` url parameter. + +Since this PEP expects the existing legacy ``1.0`` upload API to exist at a +different endpoint, and it currently only provides for JSON serialization, this +mechanism is not particularly useful, and clients only have a single version and +serialization they can request. However clients **SHOULD** be setup to handle +content negotiation gracefully in the case that additional formats or versions +are added in the future. + + +FAQ +=== + +Does this mean PyPI is planning to drop support for the existing upload API? +---------------------------------------------------------------------------- + +At this time PyPI does not have any specific plans to drop support for the +existing upload API. + +Unlike with :pep:`691` there are wide benefits to doing so, so it is likely +that we will want to drop support for it at some point in the future, but +until this API is implemented, and receiving broad use it would be premature +to make any plans for actually dropping support for it. + + +Is this Resumable Upload protocol based on anything? +---------------------------------------------------- + +Yes! + +It's actually the protocol specified in an +`Active Internet-Draft `_, +where the authors took what they learned implementing `tus `_ +to provide the idea of resumable uploads in a wholly generic, standards based +way. + +The only deviation we've made from that spec is that we don't use the +``104 Upload Resumption Supported`` informational response in the first +``POST`` request. This decision was made for a few reasons: + +- The ``104 Upload Resumption Supported`` is the only part of that draft + which does not rely entirely on things that are already supported in the + existing standards, since it was adding a new informational status. +- Many clients and web frameworks don't support ``1xx`` informational + responses in a very good way, if at all, adding it would complicate + implementation for very little benefit. +- The purpose of the ``104 Upload Resumption Supported`` support is to allow + clients to determine that an arbitrary endpoint that they're interacting + with supports resumable uploads. Since this PEP is mandating support for + that in servers, clients can just assume that the server they are + interacting with supports it, which makes using it unneeded. +- In theory, if the support for ``1xx`` responses got resolved and the draft + gets accepted with it in, we can add that in at a later date without + changing the overall flow of the API. + +There is a risk that the above draft doesn't get accepted, but even if it +does not, that doesn't actually affect us. It would just mean that our +support for resumable uploads is an application specific protocol, but is +still wholly standards compliant. + + +Open Questions +============== + + +Multipart Uploads vs tus +------------------------ + +This PEP currently bases the actual uploading of files on an internet draft +from tus.io that supports resumable file uploads. + +That protocol requires a few things: + +- That the client selects a secure ``Upload-Token`` that they use to identify + uploading a single file. +- That if clients don't upload the entire file in one shot, that they have + to submit the chunks serially, and in the correct order, with all but the + final chunk having a ``Upload-Incomplete: 1`` header. +- Resumption of an upload is essentially just querying the server to see how + much data they've gotten, then sending the remaining bytes (either as a single + request, or in chunks). +- The upload implicitly is completed when the server successfully gets all of + the data from the client. + +This has one big benefit, that if a client doesn't care about resuming their +download, the work to support, from a client side, resumable uploads is able +to be completely ignored. They can just ``POST`` the file to the URL, and if +it doesn't succeed, they can just ``POST`` the whole file again. + +The other benefit is that even if you do want to support resumption, you can +still just ``POST`` the file, and unless you *need* to resume the download, +that's all you have to do. + +Another, possibly theoretical, benefit is that for hashing the uploaded files, +the serial chunks requirement means that the server can maintain hashing state +between requests, update it for each request, then write that file back to +storage. Unfortunately this isn't actually possible to do with Python's hashlib, +though there are some libraries like `Rehash `_ +that implement it, but they don't support every hash that hashlib does +(specifically not blake2 or sha3 at the time of writing). + +We might also need to reconstitute the download for processing anyways to do +things like extract metadata, etc from it, which would make it a moot point. + +The downside is that there is no ability to parallelize the upload of a single +file because each chunk has to be submitted serially. + +AWS S3 has a similar API (and most blob stores have copied it either wholesale +or something like it) which they call multipart uploading. + +The basic flow for a multipart upload is: + +1. Initiate a Multipart Upload to get an Upload ID. +2. Break your file up into chunks, and upload each one of them individually. +3. Once all chunks have been uploaded, finalize the upload. + - This is the step where any errors would occur. + +It does not directly support resuming an upload, but it allows clients to +control the "blast radius" of failure by adjusting the size of each part +they upload, and if any of the parts fail, they only have to resend those +specific parts. + +This has a big benefit in that it allows parallelization in uploading files, +allowing clients to maximize their bandwidth using multiple threads to send +the data. + +We wouldn't need an explicit step (1), because our session would implicitly +initiate a multipart upload for each file. + +It does have its own downsides: + +- Clients have to do more work on every request to have something resembling + resumable uploads. They would *have* to break the file up into multiple parts + rather than just making a single POST request, and only needing to deal + with the complexity if something fails. + +- Clients that don't care about resumption at all still have to deal with + the third explicit step, though they could just upload the file all as a + single part. + + - S3 works around this by having another API for one shot uploads, but + I'd rather not have two different APIs for uploading the same file. + +- Verifying hashes gets somewhat more complicated. AWS implements hashing + multipart uploads by hashing each part, then the overall hash is just a + hash of those hashes, not of the content itself. We need to know the + actual hash of the file itself for PyPI, so we would have to reconstitute + the file and read its content and hash it once it's been fully uploaded, + though we could still use the hash of hashes trick for checksumming the + upload itself. + + - See above about whether this is actually a downside in practice, or + if it's just in theory. + +I lean towards the tus style resumable uploads as I think they're simpler +to use and to implement, and the main downside is that we possibly leave +some multi-threaded performance on the table, which I think that I'm +personally fine with? + +I guess one additional benefit of the S3 style multi part uploads is that +you don't have to try and do any sort of protection against parallel uploads, +since they're just supported. That alone might erase most of the server side +implementation simplification. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0695.rst b/peps/pep-0695.rst new file mode 100644 index 000000000..182a3eb52 --- /dev/null +++ b/peps/pep-0695.rst @@ -0,0 +1,1626 @@ +PEP: 695 +Title: Type Parameter Syntax +Author: Eric Traut +Sponsor: Guido van Rossum +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/BB2BGYJY2YG5IWESKGTAPUQL3N27ZKVW/ +Status: Accepted +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 15-Jun-2022 +Python-Version: 3.12 +Post-History: `20-Jun-2022 `__, + `04-Dec-2022 `__ +Resolution: https://discuss.python.org/t/pep-695-type-parameter-syntax/21646/92 + + +Abstract +======== + +This PEP specifies an improved syntax for specifying type parameters within +a generic class, function, or type alias. It also introduces a new statement +for declaring type aliases. + + +Motivation +========== + +:pep:`484` introduced type variables into the language. :pep:`612` built +upon this concept by introducing parameter specifications, and +:pep:`646` added variadic type variables. + +While generic types and type parameters have grown in popularity, the +syntax for specifying type parameters still feels "bolted on" to Python. +This is a source of confusion among Python developers. + +There is consensus within the Python static typing community that it is time +to provide a formal syntax that is similar to other modern programming +languages that support generic types. + +An analysis of 25 popular typed Python libraries revealed that type +variables (in particular, the ``typing.TypeVar`` symbol) were used in +14% of modules. + + +Points of Confusion +------------------- + +While the use of type variables has become widespread, the manner in which +they are specified within code is the source of confusion among many +Python developers. There are a couple of factors that contribute to this +confusion. + +The scoping rules for type variables are difficult to understand. Type +variables are typically allocated within the global scope, but their semantic +meaning is valid only when used within the context of a generic class, +function, or type alias. A single runtime instance of a type variable may be +reused in multiple generic contexts, and it has a different semantic meaning +in each of these contexts. This PEP proposes to eliminate this source of +confusion by declaring type parameters at a natural place within a class, +function, or type alias declaration statement. + +Generic type aliases are often misused because it is not clear to developers +that a type argument must be supplied when the type alias is used. This leads +to an implied type argument of ``Any``, which is rarely the intent. This PEP +proposes to add new syntax that makes generic type alias declarations +clear. + +:pep:`483` and :pep:`484` introduced the concept of "variance" for a type +variable used within a generic class. Type variables can be invariant, +covariant, or contravariant. The concept of variance is an advanced detail +of type theory that is not well understood by most Python developers, yet +they must confront this concept today when defining their first generic +class. This PEP largely eliminates the need for most developers +to understand the concept of variance when defining generic classes. + +When more than one type parameter is used with a generic class or type alias, +the rules for type parameter ordering can be confusing. It is normally based on +the order in which they first appear within a class or type alias declaration +statement. However, this can be overridden in a class definition by +including a "Generic" or "Protocol" base class. For example, in the class +declaration ``class ClassA(Mapping[K, V])``, the type parameters are +ordered as ``K`` and then ``V``. However, in the class declaration +``class ClassB(Mapping[K, V], Generic[V, K])``, the type parameters are +ordered as ``V`` and then ``K``. This PEP proposes to make type parameter +ordering explicit in all cases. + +The practice of sharing a type variable across multiple generic contexts +creates other problems today. Modern editors provide features like "find +all references" and "rename all references" that operate on symbols at the +semantic level. When a type parameter is shared among multiple generic +classes, functions, and type aliases, all references are semantically +equivalent. + +Type variables defined within the global scope also need to be given a name +that starts with an underscore to indicate that the variable is private to +the module. Globally-defined type variables are also often given names to +indicate their variance, leading to cumbersome names like "_T_contra" and +"_KT_co". The current mechanisms for allocating type variables also requires +the developer to supply a redundant name in quotes (e.g. ``T = TypeVar("T")``). +This PEP eliminates the need for the redundant name and cumbersome +variable names. + +Defining type parameters today requires importing the ``TypeVar`` and +``Generic`` symbols from the ``typing`` module. Over the past several releases +of Python, efforts have been made to eliminate the need to import ``typing`` +symbols for common use cases, and the PEP furthers this goal. + + +Summary Examples +================ + +Defining a generic class prior to this PEP looks something like this. + +:: + + from typing import Generic, TypeVar + + _T_co = TypeVar("_T_co", covariant=True, bound=str) + + class ClassA(Generic[_T_co]): + def method1(self) -> _T_co: + ... + + +With the new syntax, it looks like this. + +:: + + class ClassA[T: str]: + def method1(self) -> T: + ... + + +Here is an example of a generic function today. + +:: + + from typing import TypeVar + + _T = TypeVar("_T") + + def func(a: _T, b: _T) -> _T: + ... + +And the new syntax. + +:: + + def func[T](a: T, b: T) -> T: + ... + + +Here is an example of a generic type alias today. + +:: + + from typing import TypeAlias + + _T = TypeVar("_T") + + ListOrSet: TypeAlias = list[_T] | set[_T] + + +And with the new syntax. + +:: + + type ListOrSet[T] = list[T] | set[T] + + +Specification +============= + +Type Parameter Declarations +--------------------------- + +Here is a new syntax for declaring type parameters for generic +classes, functions, and type aliases. The syntax adds support for +a comma-delimited list of type parameters in square brackets after +the name of the class, function, or type alias. + +Simple (non-variadic) type variables are declared with an unadorned name. +Variadic type variables are preceded by ``*`` (see :pep:`646` for details). +Parameter specifications are preceded by ``**`` (see :pep:`612` for details). + +:: + + # This generic class is parameterized by a TypeVar T, a + # TypeVarTuple Ts, and a ParamSpec P. + class ChildClass[T, *Ts, **P]: ... + +There is no need to include ``Generic`` as a base class. Its inclusion as +a base class is implied by the presence of type parameters, and it will +automatically be included in the ``__mro__`` and ``__orig_bases__`` attributes +for the class. The explicit use of a ``Generic`` base class will result in a +runtime error. + +:: + + class ClassA[T](Generic[T]): ... # Runtime error + + +A ``Protocol`` base class with type arguments may generate a runtime +error. Type checkers should generate an error in this case because +the use of type arguments is not needed, and the order of type parameters +for the class are no longer dictated by their order in the ``Protocol`` +base class. + +:: + + class ClassA[S, T](Protocol): ... # OK + + class ClassB[S, T](Protocol[S, T]): ... # Recommended type checker error + + +Type parameter names within a generic class, function, or type alias must be +unique within that same class, function, or type alias. A duplicate name +generates a syntax error at compile time. This is consistent with the +requirement that parameter names within a function signature must be unique. + +:: + + class ClassA[T, *T]: ... # Syntax Error + + def func1[T, **T](): ... # Syntax Error + + +Class type parameter names are mangled if they begin with a double +underscore, to avoid complicating the name lookup mechanism for names used +within the class. However, the ``__name__`` attribute of the type parameter +will hold the non-mangled name. + + +Upper Bound Specification +------------------------- + +For a non-variadic type parameter, an "upper bound" type can be specified +through the use of a type annotation expression. If an upper bound is +not specified, the upper bound is assumed to be ``object``. + +:: + + class ClassA[T: str]: ... + +The specified upper bound type must use an expression form that is allowed in +type annotations. More complex expression forms should be flagged +as an error by a type checker. Quoted forward references are allowed. + +The specified upper bound type must be concrete. An attempt to use a generic +type should be flagged as an error by a type checker. This is consistent with +the existing rules enforced by type checkers for a ``TypeVar`` constructor call. + +:: + + class ClassA[T: dict[str, int]]: ... # OK + + class ClassB[T: "ForwardReference"]: ... # OK + + class ClassC[V]: + class ClassD[T: dict[str, V]]: ... # Type checker error: generic type + + class ClassE[T: [str, int]]: ... # Type checker error: illegal expression form + + +Constrained Type Specification +------------------------------ + +:pep:`484` introduced the concept of a "constrained type variable" which is +constrained to a set of two or more types. The new syntax supports this type +of constraint through the use of a literal tuple expression that contains +two or more types. + +:: + + class ClassA[AnyStr: (str, bytes)]: ... # OK + + class ClassB[T: ("ForwardReference", bytes)]: ... # OK + + class ClassC[T: ()]: ... # Type checker error: two or more types required + + class ClassD[T: (str, )]: ... # Type checker error: two or more types required + + t1 = (bytes, str) + class ClassE[T: t1]: ... # Type checker error: literal tuple expression required + + +If the specified type is not a tuple expression or the tuple expression includes +complex expression forms that are not allowed in a type annotation, a type +checker should generate an error. Quoted forward references are allowed. + +:: + + class ClassF[T: (3, bytes)]: ... # Type checker error: invalid expression form + + +The specified constrained types must be concrete. An attempt to use a generic +type should be flagged as an error by a type checker. This is consistent with +the existing rules enforced by type checkers for a ``TypeVar`` constructor call. + +:: + + class ClassG[T: (list[S], str)]: ... # Type checker error: generic type + + +Runtime Representation of Bounds and Constraints +------------------------------------------------ + +The upper bounds and constraints of ``TypeVar`` objects are accessible at +runtime through the ``__bound__`` and ``__constraints__`` attributes. +For ``TypeVar`` objects defined through the new syntax, these attributes +become lazily evaluated, as discussed under `Lazy Evaluation`_ below. + + +Generic Type Alias +------------------ + +We propose to introduce a new statement for declaring type aliases. Similar +to ``class`` and ``def`` statements, a ``type`` statement defines a scope +for type parameters. + +:: + + # A non-generic type alias + type IntOrStr = int | str + + # A generic type alias + type ListOrSet[T] = list[T] | set[T] + + +Type aliases can refer to themselves without the use of quotes. + +:: + + # A type alias that includes a forward reference + type AnimalOrVegetable = Animal | "Vegetable" + + # A generic self-referential type alias + type RecursiveList[T] = T | list[RecursiveList[T]] + + +The ``type`` keyword is a new soft keyword. It is interpreted as a keyword +only in this part of the grammar. In all other locations, it is assumed to +be an identifier name. + +Type parameters declared as part of a generic type alias are valid only +when evaluating the right-hand side of the type alias. + +As with ``typing.TypeAlias``, type checkers should restrict the right-hand +expression to expression forms that are allowed within type annotations. +The use of more complex expression forms (call expressions, ternary operators, +arithmetic operators, comparison operators, etc.) should be flagged as an +error. + +Type alias expressions are not allowed to use traditional type variables (i.e. +those allocated with an explicit ``TypeVar`` constructor call). Type checkers +should generate an error in this case. + +:: + + T = TypeVar("T") + type MyList = list[T] # Type checker error: traditional type variable usage + + +We propose to deprecate the existing ``typing.TypeAlias`` introduced in +:pep:`613`. The new syntax eliminates its need entirely. + + +Runtime Type Alias Class +------------------------ + +At runtime, a ``type`` statement will generate an instance of +``typing.TypeAliasType``. This class represents the type. Its attributes +include: + +* ``__name__`` is a str representing the name of the type alias +* ``__type_params__`` is a tuple of ``TypeVar``, ``TypeVarTuple``, or + ``ParamSpec`` objects that parameterize the type alias if it is generic +* ``__value__`` is the evaluated value of the type alias + +All of these attributes are read-only. + +The value of the type alias is evaluated lazily (see `Lazy Evaluation`_ below). + + +Type Parameter Scopes +--------------------- + +When the new syntax is used, a new lexical scope is introduced, and this scope +includes the type parameters. Type parameters can be accessed by name +within inner scopes. As with other symbols in Python, an inner scope can +define its own symbol that overrides an outer-scope symbol of the same name. +This section provides a verbal description of the new scoping rules. +The `Scoping Behavior`_ section below specifies the behavior in terms +of a translation to near-equivalent existing Python code. + +Type parameters are visible to other +type parameters declared elsewhere in the list. This allows type parameters +to use other type parameters within their definition. While there is currently +no use for this capability, it preserves the ability in the future to support +upper bound expressions or type argument defaults that depend on earlier +type parameters. + +A compiler error or runtime exception is generated if the definition of an +earlier type parameter references a later type parameter even if the name is +defined in an outer scope. + +:: + + # The following generates no compiler error, but a type checker + # should generate an error because an upper bound type must be concrete, + # and ``Sequence[S]`` is generic. Future extensions to the type system may + # eliminate this limitation. + class ClassA[S, T: Sequence[S]]: ... + + # The following generates no compiler error, because the bound for ``S`` + # is lazily evaluated. However, type checkers should generate an error. + class ClassB[S: Sequence[T], T]: ... + + +A type parameter declared as part of a generic class is valid within the +class body and inner scopes contained therein. Type parameters are also +accessible when evaluating the argument list (base classes and any keyword +arguments) that comprise the class definition. This allows base classes +to be parameterized by these type parameters. Type parameters are not +accessible outside of the class body, including class decorators. + +:: + + class ClassA[T](BaseClass[T], param = Foo[T]): ... # OK + + print(T) # Runtime error: 'T' is not defined + + @dec(Foo[T]) # Runtime error: 'T' is not defined + class ClassA[T]: ... + + +A type parameter declared as part of a generic function is valid within +the function body and any scopes contained therein. It is also valid within +parameter and return type annotations. Default argument values for function +parameters are evaluated outside of this scope, so type parameters are +not accessible in default value expressions. Likewise, type parameters are not +in scope for function decorators. + +:: + + def func1[T](a: T) -> T: ... # OK + + print(T) # Runtime error: 'T' is not defined + + def func2[T](a = list[T]): ... # Runtime error: 'T' is not defined + + @dec(list[T]) # Runtime error: 'T' is not defined + def func3[T](): ... + +A type parameter declared as part of a generic type alias is valid within +the type alias expression. + +:: + + type Alias1[K, V] = Mapping[K, V] | Sequence[K] + + +Type parameter symbols defined in outer scopes cannot be bound with +``nonlocal`` statements in inner scopes. + +:: + + S = 0 + + def outer1[S](): + S = 1 + T = 1 + + def outer2[T](): + + def inner1(): + nonlocal S # OK because it binds variable S from outer1 + nonlocal T # Syntax error: nonlocal binding not allowed for type parameter + + def inner2(): + global S # OK because it binds variable S from global scope + + +The lexical scope introduced by the new type parameter syntax is unlike +traditional scopes introduced by a ``def`` or ``class`` statement. A type +parameter scope acts more like a temporary "overlay" to the containing scope. +The only new symbols contained +within its symbol table are the type parameters defined using the new syntax. +References to all other symbols are treated as though they were found within +the containing scope. This allows base class lists (in class definitions) and +type annotation expressions (in function definitions) to reference symbols +defined in the containing scope. + +:: + + class Outer: + class Private: + pass + + # If the type parameter scope was like a traditional scope, + # the base class 'Private' would not be accessible here. + class Inner[T](Private, Sequence[T]): + pass + + # Likewise, 'Inner' would not be available in these type annotations. + def method1[T](self, a: Inner[T]) -> Inner[T]: + return a + + +The compiler allows inner scopes to define a local symbol that overrides an +outer-scoped type parameter. + +Consistent with the scoping rules defined in :pep:`484`, type checkers should +generate an error if inner-scoped generic classes, functions, or type aliases +reuse the same type parameter name as an outer scope. + +:: + + T = 0 + + @decorator(T) # Argument expression `T` evaluates to 0 + class ClassA[T](Sequence[T]): + T = 1 + + # All methods below should result in a type checker error + # "type parameter 'T' already in use" because they are using the + # type parameter 'T', which is already in use by the outer scope + # 'ClassA'. + def method1[T](self): + ... + + def method2[T](self, x = T): # Parameter 'x' gets default value of 1 + ... + + def method3[T](self, x: T): # Parameter 'x' has type T (scoped to method3) + ... + + +Symbols referenced in inner scopes are resolved using existing rules except +that type parameter scopes are also considered during name resolution. + +:: + + T = 0 + + # T refers to the global variable + print(T) # Prints 0 + + class Outer[T]: + T = 1 + + # T refers to the local variable scoped to class 'Outer' + print(T) # Prints 1 + + class Inner1: + T = 2 + + # T refers to the local type variable within 'Inner1' + print(T) # Prints 2 + + def inner_method(self): + # T refers to the type parameter scoped to class 'Outer'; + # If 'Outer' did not use the new type parameter syntax, + # this would instead refer to the global variable 'T' + print(T) # Prints 'T' + + def outer_method(self): + T = 3 + + # T refers to the local variable within 'outer_method' + print(T) # Prints 3 + + def inner_func(): + # T refers to the variable captured from 'outer_method' + print(T) # Prints 3 + + +When the new type parameter syntax is used for a generic class, assignment +expressions are not allowed within the argument list for the class definition. +Likewise, with functions that use the new type parameter syntax, assignment +expressions are not allowed within parameter or return type annotations, nor +are they allowed within the expression that defines a type alias, or within +the bounds and constraints of a ``TypeVar``. Similarly, ``yield``, ``yield from``, +and ``await`` expressions are disallowed in these contexts. + +This restriction is necessary because expressions evaluated within the +new lexical scope should not introduce symbols within that scope other than +the defined type parameters, and should not affect whether the enclosing function +is a generator or coroutine. + +:: + + class ClassA[T]((x := Sequence[T])): ... # Syntax error: assignment expression not allowed + + def func1[T](val: (x := int)): ... # Syntax error: assignment expression not allowed + + def func2[T]() -> (x := Sequence[T]): ... # Syntax error: assignment expression not allowed + + type Alias1[T] = (x := list[T]) # Syntax error: assignment expression not allowed + + +Accessing Type Parameters at Runtime +------------------------------------ + +A new read-only attribute called ``__type_params__`` is available on generic classes, +functions, and type aliases. This attribute is a tuple of the +type parameters that parameterize the class, function, or alias. +The tuple contains ``TypeVar``, ``ParamSpec``, and ``TypeVarTuple`` instances. + +Type parameters declared using the new syntax will not appear within the +dictionary returned by ``globals()`` or ``locals()``. + + +Variance Inference +------------------ + +This PEP eliminates the need for variance to be specified for type +parameters. Instead, type checkers will infer the variance of type parameters +based on their usage within a class. Type parameters are inferred to be +invariant, covariant, or contravariant depending on how they are used. + +Python type checkers already include the ability to determine the variance of +type parameters for the purpose of validating variance within a generic +protocol class. This capability can be used for all classes (whether or not +they are protocols) to calculate the variance of each type parameter. + +The algorithm for computing the variance of a type parameter is as follows. + +For each type parameter in a generic class: + +1. If the type parameter is variadic (``TypeVarTuple``) or a parameter +specification (``ParamSpec``), it is always considered invariant. No further +inference is needed. + +2. If the type parameter comes from a traditional ``TypeVar`` declaration and +is not specified as ``infer_variance`` (see below), its variance is specified +by the ``TypeVar`` constructor call. No further inference is needed. + +3. Create two specialized versions of the class. We'll refer to these as +``upper`` and ``lower`` specializations. In both of these specializations, +replace all type parameters other than the one being inferred by a dummy type +instance (a concrete anonymous class that is type compatible with itself and +assumed to meet the bounds or constraints of the type parameter). In +the ``upper`` specialized class, specialize the target type parameter with +an ``object`` instance. This specialization ignores the type parameter's +upper bound or constraints. In the ``lower`` specialized class, specialize +the target type parameter with itself (i.e. the corresponding type argument +is the type parameter itself). + +4. Determine whether ``lower`` can be assigned to ``upper`` using normal type +compatibility rules. If so, the target type parameter is covariant. If not, +determine whether ``upper`` can be assigned to ``lower``. If so, the target +type parameter is contravariant. If neither of these combinations are +assignable, the target type parameter is invariant. + +Here is an example. + +:: + + class ClassA[T1, T2, T3](list[T1]): + def method1(self, a: T2) -> None: + ... + + def method2(self) -> T3: + ... + +To determine the variance of ``T1``, we specialize ``ClassA`` as follows: + +:: + + upper = ClassA[object, Dummy, Dummy] + lower = ClassA[T1, Dummy, Dummy] + +We find that ``upper`` is not assignable to ``lower`` using normal type +compatibility rules defined in :pep:`484`. Likewise, ``lower`` is not assignable +to ``upper``, so we conclude that ``T1`` is invariant. + +To determine the variance of ``T2``, we specialize ``ClassA`` as follows: + +:: + + upper = ClassA[Dummy, object, Dummy] + lower = ClassA[Dummy, T2, Dummy] + +Since ``upper`` is assignable to ``lower``, ``T2`` is contravariant. + +To determine the variance of ``T3``, we specialize ``ClassA`` as follows: + +:: + + upper = ClassA[Dummy, Dummy, object] + lower = ClassA[Dummy, Dummy, T3] + +Since ``lower`` is assignable to ``upper``, ``T3`` is covariant. + + +Auto Variance For TypeVar +------------------------- + +The existing ``TypeVar`` class constructor accepts keyword parameters named +``covariant`` and ``contravariant``. If both of these are ``False``, the +type variable is assumed to be invariant. We propose to add another keyword +parameter named ``infer_variance`` indicating that a type checker should use +inference to determine whether the type variable is invariant, covariant or +contravariant. A corresponding instance variable ``__infer_variance__`` can be +accessed at runtime to determine whether the variance is inferred. Type +variables that are implicitly allocated using the new syntax will always +have ``__infer_variance__`` set to ``True``. + +A generic class that uses the traditional syntax may include combinations of +type variables with explicit and inferred variance. + +:: + + T1 = TypeVar("T1", infer_variance=True) # Inferred variance + T2 = TypeVar("T2") # Invariant + T3 = TypeVar("T3", covariant=True) # Covariant + + # A type checker should infer the variance for T1 but use the + # specified variance for T2 and T3. + class ClassA(Generic[T1, T2, T3]): ... + + +Compatibility with Traditional TypeVars +--------------------------------------- + +The existing mechanism for allocating ``TypeVar``, ``TypeVarTuple``, and +``ParamSpec`` is retained for backward compatibility. However, these +"traditional" type variables should not be combined with type parameters +allocated using the new syntax. Such a combination should be flagged as +an error by type checkers. This is necessary because the type parameter +order is ambiguous. + +It is OK to combine traditional type variables with new-style type parameters +if the class, function, or type alias does not use the new syntax. The +new-style type parameters must come from an outer scope in this case. + +:: + + K = TypeVar("K") + + class ClassA[V](dict[K, V]): ... # Type checker error + + class ClassB[K, V](dict[K, V]): ... # OK + + class ClassC[V]: + # The use of K and V for "method1" is OK because it uses the + # "traditional" generic function mechanism where type parameters + # are implicit. In this case V comes from an outer scope (ClassC) + # and K is introduced implicitly as a type parameter for "method1". + def method1(self, a: V, b: K) -> V | K: ... + + # The use of M and K are not allowed for "method2". A type checker + # should generate an error in this case because this method uses the + # new syntax for type parameters, and all type parameters associated + # with the method must be explicitly declared. In this case, ``K`` + # is not declared by "method2", nor is it supplied by a new-style + # type parameter defined in an outer scope. + def method2[M](self, a: M, b: K) -> M | K: ... + + +Runtime Implementation +====================== + +Grammar Changes +--------------- + +This PEP introduces a new soft keyword ``type``. It modifies the grammar +in the following ways: + +1. Addition of optional type parameter clause in ``class`` and ``def`` statements. + +:: + + type_params: '[' t=type_param_seq ']' + + type_param_seq: a[asdl_typeparam_seq*]=','.type_param+ [','] + + type_param: + | a=NAME b=[type_param_bound] + | '*' a=NAME + | '**' a=NAME + + type_param_bound: ":" e=expression + + # Grammar definitions for class_def_raw and function_def_raw are modified + # to reference type_params as an optional syntax element. The definitions + # of class_def_raw and function_def_raw are simplified here for brevity. + + class_def_raw: 'class' n=NAME t=[type_params] ... + + function_def_raw: a=[ASYNC] 'def' n=NAME t=[type_params] ... + + +2. Addition of new ``type`` statement for defining type aliases. + +:: + + type_alias: "type" n=NAME t=[type_params] '=' b=expression + + +AST Changes +----------- + +This PEP introduces a new AST node type called ``TypeAlias``. + +:: + + TypeAlias(expr name, typeparam* typeparams, expr value) + +It also adds an AST node type that represents a type parameter. + +:: + + typeparam = TypeVar(identifier name, expr? bound) + | ParamSpec(identifier name) + | TypeVarTuple(identifier name) + +Bounds and constraints are represented identically in the AST. In the implementation, +any expression that is a ``Tuple`` AST node is treated as a constraint, and any other +expression is treated as a bound. + +It also modifies existing AST node types ``FunctionDef``, ``AsyncFunctionDef`` +and ``ClassDef`` to include an additional optional attribute called +``typeparams`` that includes a list of type parameters associated with the +function or class. + +Lazy Evaluation +--------------- + +This PEP introduces three new contexts where expressions may occur that represent +static types: ``TypeVar`` bounds, ``TypeVar`` constraints, and the value of type +aliases. These expressions may contain references to names +that are not yet defined. For example, type aliases may be recursive, or even mutually +recursive, and type variable bounds may refer back to the current class. If these +expressions were evaluated eagerly, users would need to enclose such expressions in +quotes to prevent runtime errors. :pep:`563` and :pep:`649` detail the problems with +this situation for type annotations. + +To prevent a similar situation with the new syntax proposed in this PEP, we propose +to use lazy evaluation for these expressions, similar to the approach in :pep:`649`. +Specifically, each expression will be saved in a code object, and the code object +is evaluated only when the corresponding attribute is accessed (``TypeVar.__bound__``, +``TypeVar.__constraints__``, or ``TypeAlias.__value__``). After the value is +successfully evaluated, the value is saved and later calls will return the same value +without re-evaluating the code object. + +If :pep:`649` is implemented, additional evaluation mechanisms should be added to +mirror the options that PEP provides for annotations. In the current version of the +PEP, that might include adding an ``__evaluate_bound__`` method to ``TypeVar`` taking +a ``format`` parameter with the same meaning as in PEP 649's ``__annotate__`` method +(and a similar ``__evaluate_constraints__`` method, as well as an ``__evaluate_value__`` +method on ``TypeAliasType``). +However, until PEP 649 is accepted and implemented, only the default evaluation format +(PEP 649's "VALUE" format) will be supported. + +As a consequence of lazy evaluation, the value observed for an attribute may +depend on the time the attribute is accessed. + +:: + + X = int + + class Foo[T: X, U: X]: + t, u = T, U + + print(Foo.t.__bound__) # prints "int" + X = str + print(Foo.u.__bound__) # prints "str" + +Similar examples affecting type annotations can be constructed using the +semantics of PEP 563 or PEP 649. + +A naive implementation of lazy evaluation would handle class namespaces +incorrectly, because functions within a class do not normally have access to +the enclosing class namespace. The implementation will retain a reference to +the class namespace so that class-scoped names are resolved correctly. + +.. _695-scoping-behavior: + +Scoping Behavior +---------------- + +The new syntax requires a new kind of scope that behaves differently +from existing scopes in Python. Thus, the new syntax cannot be described exactly in terms of +existing Python scoping behavior. This section specifies these scopes +further by reference to existing scoping behavior: the new scopes behave +like function scopes, except for a number of minor differences listed below. + +All examples include functions introduced with the pseudo-keyword ``def695``. +This keyword will not exist in the actual language; it is used to +clarify that the new scopes are for the most part like function scopes. + +``def695`` scopes differ from regular function scopes in the following ways: + +- If a ``def695`` scope is immediately within a class scope, or within another + ``def695`` scope that is immediately within a class scope, then names defined + in that class scope can be accessed within the ``def695`` scope. (Regular functions, + by contrast, cannot access names defined within an enclosing class scope.) +- The following constructs are disallowed directly within a ``def695`` scope, though + they may be used within other scopes nested inside a ``def695`` scope: + + - ``yield`` + - ``yield from`` + - ``await`` + - ``:=`` (walrus operator) + +- The qualified name (``__qualname__``) of objects (classes and functions) defined within ``def695`` scopes + is as if the objects were defined within the closest enclosing scope. +- Names bound within ``def695`` scopes cannot be rebound with a ``nonlocal`` statement in nested scopes. + +``def695`` scopes are used for the evaluation of several new syntactic constructs proposed +in this PEP. Some are evaluated eagerly (when a type alias, function, or class is defined); others are +evaluated lazily (only when evaluation is specifically requested). In all cases, the scoping semantics are identical: + +- Eagerly evaluated values: + + - The type parameters of generic type aliases + - The type parameters and annotations of generic functions + - The type parameters and base class expressions of generic classes +- Lazily evaluated values: + + - The value of generic type aliases + - The bounds of type variables + - The constraints of type variables + +In the below translations, names that start with two underscores are internal to the implementation +and not visible to actual Python code. We use the following intrinsic functions, which in the real +implementation are defined directly in the interpreter: + +- ``__make_typealias(*, name, type_params=(), evaluate_value)``: Creates a new ``typing.TypeAlias`` object with the given + name, type parameters, and lazily evaluated value. The value is not evaluated until the ``__value__`` attribute + is accessed. +- ``__make_typevar_with_bound(*, name, evaluate_bound)``: Creates a new ``typing.TypeVar`` object with the given + name and lazily evaluated bound. The bound is not evaluated until the ``__bound__`` attribute is accessed. +- ``__make_typevar_with_constraints(*, name, evaluate_constraints)``: Creates a new ``typing.TypeVar`` object with the given + name and lazily evaluated constraints. The constraints are not evaluated until the ``__constraints__`` attribute + is accessed. + +Non-generic type aliases are translated as follows:: + + type Alias = int + +Equivalent to:: + + def695 __evaluate_Alias(): + return int + + Alias = __make_typealias(name='Alias', evaluate_value=__evaluate_Alias) + +Generic type aliases:: + + type Alias[T: int] = list[T] + +Equivalent to:: + + def695 __generic_parameters_of_Alias(): + def695 __evaluate_T_bound(): + return int + T = __make_typevar_with_bound(name='T', evaluate_bound=__evaluate_T_bound) + + def695 __evaluate_Alias(): + return list[T] + return __make_typealias(name='Alias', type_params=(T,), evaluate_value=__evaluate_Alias) + + Alias = __generic_parameters_of_Alias() + +Generic functions:: + + def f[T](x: T) -> T: + return x + +Equivalent to:: + + def695 __generic_parameters_of_f(): + T = typing.TypeVar(name='T') + + def f(x: T) -> T: + return x + f.__type_params__ = (T,) + return f + + f = __generic_parameters_of_f() + +A fuller example of generic functions, illustrating the scoping behavior of defaults, decorators, and bounds. +Note that this example does not use ``ParamSpec`` correctly, so it should be rejected by a static type checker. +It is however valid at runtime, and it us used here to illustrate the runtime semantics. + +:: + + @decorator + def f[T: int, U: (int, str), *V, **P]( + x: T = SOME_CONSTANT, + y: U, + *args: *Ts, + **kwargs: P.kwargs, + ) -> T: + return x + +Equivalent to:: + + __default_of_x = SOME_CONSTANT # evaluated outside the def695 scope + def695 __generic_parameters_of_f(): + def695 __evaluate_T_bound(): + return int + T = __make_typevar_with_bound(name='T', evaluate_bound=__evaluate_T_bound) + + def695 __evaluate_U_constraints(): + return (int, str) + U = __make_typevar_with_constraints(name='U', evaluate_constraints=__evaluate_U_constraints) + + Ts = typing.TypeVarTuple("Ts") + P = typing.ParamSpec("P") + + def f(x: T = __default_of_x, y: U, *args: *Ts, **kwargs: P.kwargs) -> T: + return x + f.__type_params__ = (T, U, Ts, P) + return f + + f = decorator(__generic_parameters_of_f()) + +Generic classes:: + + class C[T](Base): + def __init__(self, x: T): + self.x = x + +Equivalent to:: + + def695 __generic_parameters_of_C(): + T = typing.TypeVar('T') + class C(Base): + __type_params__ = (T,) + def __init__(self, x: T): + self.x = x + return C + + C = __generic_parameters_of_C() + +The biggest divergence from existing behavior for ``def695`` scopes +is the behavior within class scopes. This divergence is necessary +so that generics defined within classes behave in an intuitive way:: + + class C: + class Nested: ... + def generic_method[T](self, x: T, y: Nested) -> T: ... + +Equivalent to:: + + class C: + class Nested: ... + + def695 __generic_parameters_of_generic_method(): + T = typing.TypeVar('T') + + def generic_method(self, x: T, y: Nested) -> T: ... + return generic_method + + generic_method = __generic_parameters_of_generic_method() + +In this example, the annotations for ``x`` and ``y`` are evaluated within +a ``def695`` scope, because they need access to the type parameter ``T`` +for the generic method. However, they also need access to the ``Nested`` +name defined within the class namespace. If ``def695`` scopes behaved +like regular function scopes, ``Nested`` would not be visible within the +function scope. Therefore, ``def695`` scopes that are immediately within +class scopes have access to that class scope, as described above. + + +Library Changes +--------------- + +Several classes in the ``typing`` module that are currently implemented in +Python must be partially implemented in C. This includes ``TypeVar``, +``TypeVarTuple``, ``ParamSpec``, and ``Generic``, and the new class +``TypeAliasType`` (described above). The implementation may delegate to the +Python version of ``typing.py`` for some behaviors that interact heavily with +the rest of the module. The +documented behaviors of these classes should not change. + + +Reference Implementation +======================== + +This proposal is prototyped in +`CPython PR #103764 `_. + +The Pyright type checker supports the behavior described in this PEP. + + +Rejected Ideas +============== + +Prefix Clause +------------- +We explored various syntactic options for specifying type parameters that +preceded ``def`` and ``class`` statements. One such variant we considered +used a ``using`` clause as follows: + +:: + + using S, T + class ClassA: ... + +This option was rejected because the scoping rules for the type parameters +were less clear. Also, this syntax did not interact well with class and +function decorators, which are common in Python. Only one other popular +programming language, C++, uses this approach. + +We likewise considered prefix forms that looked like decorators (e.g., +``@using(S, T)``). This idea was rejected because such forms would be confused +with regular decorators, and they would not compose well with existing +decorators. Furthermore, decorators are logically executed after the statement +they are decorating, so it would be confusing for them to introduce symbols +(type parameters) that are visible within the "decorated" statement, which is +logically executed before the decorator itself. + + +Angle Brackets +-------------- +Many languages that support generics make use of angle brackets. (Refer to +the table at the end of Appendix A for a summary.) We explored the use of +angle brackets for type parameter declarations in Python, but we ultimately +rejected it for two reasons. First, angle brackets are not considered +"paired" by the Python scanner, so end-of-line characters between a ``<`` +and ``>`` token are retained. That means any line breaks within a list of +type parameters would require the use of unsightly and cumbersome ``\`` escape +sequences. Second, Python has already established the use of square brackets +for explicit specialization of a generic type (e.g., ``list[int]``). We +concluded that it would be inconsistent and confusing to use angle brackets +for generic declarations but square brackets for explicit specialization. All +other languages that we surveyed were consistent in this regard. + + +Bounds Syntax +------------- +We explored various syntactic options for specifying the bounds and constraints +for a type variable. We considered, but ultimately rejected, the use +of a ``<:`` token like in Scala, the use of an ``extends`` or ``with`` +keyword like in various other languages, and the use of a function call +syntax similar to today's ``typing.TypeVar`` constructor. The simple colon +syntax is consistent with many other programming languages (see Appendix A), +and it was heavily preferred by a cross section of Python developers who +were surveyed. + + +Explicit Variance +----------------- +We considered adding syntax for specifying whether a type parameter is intended +to be invariant, covariant, or contravariant. The ``typing.TypeVar`` mechanism +in Python requires this. A few other languages including Scala and C# +also require developers to specify the variance. We rejected this idea because +variance can generally be inferred, and most modern programming languages +do infer variance based on usage. Variance is an advanced topic that +many developers find confusing, so we want to eliminate the need to +understand this concept for most Python developers. + + +Name Mangling +------------- +When considering implementation options, we considered a "name mangling" +approach where each type parameter was given a unique "mangled" name by the +compiler. This mangled name would be based on the qualified name of the +generic class, function or type alias it was associated with. This approach +was rejected because qualified names are not necessarily unique, which means +the mangled name would need to be based on some other randomized value. +Furthermore, this approach is not compatible with techniques used for +evaluating quoted (forward referenced) type annotations. + + +Appendix A: Survey of Type Parameter Syntax +=========================================== + +Support for generic types is found in many programming languages. In this +section, we provide a survey of the options used by other popular programming +languages. This is relevant because familiarity with other languages will +make it easier for Python developers to understand this concept. We provide +additional details here (for example, default type argument support) that +may be useful when considering future extensions to the Python type system. + + +C++ +--- + +C++ uses angle brackets in combination with keywords ``template`` and +``typename`` to declare type parameters. It uses angle brackets for +specialization. + +C++20 introduced the notion of generalized constraints, which can act +like protocols in Python. A collection of constraints can be defined in +a named entity called a ``concept``. + +Variance is not explicitly specified, but constraints can enforce variance. + +A default type argument can be specified using the ``=`` operator. + +.. code-block:: c++ + + // Generic class + template + class ClassA + { + // Constraints are supported through compile-time assertions. + static_assert(std::is_base_of::value); + + public: + Container t; + }; + + // Generic function with default type argument + template + S func1(ClassA a, S b) {}; + + // C++20 introduced a more generalized notion of "constraints" + // and "concepts", which are named constraints. + + // A sample concept + template + concept Hashable = requires(T a) + { + { std::hash{}(a) } -> std::convertible_to; + }; + + // Use of a concept in a template + template + void func2(T value) {} + + // Alternative use of concept + template requires Hashable + void func3(T value) {} + + // Alternative use of concept + template + void func3(T value) requires Hashable {} + + +Java +---- + +Java uses angle brackets to declare type parameters and for specialization. +By default, type parameters are invariant. +The ``extends`` keyword is used to specify an upper bound. The ``super`` keyword +is used to specify a contravariant bound. + +Java uses use-site variance. The compiler places limits on which methods and +members can be accessed based on the use of a generic type. Variance is +not specified explicitly. + +Java provides no way to specify a default type argument. + +.. code-block:: java + + // Generic class + public class ClassA { + public Container t; + + // Generic method + public void method1(S value) { } + + // Use site variance + public void method1(ClassA value) { } + } + + +C# +-- + +C# uses angle brackets to declare type parameters and for specialization. +The ``where`` keyword and a colon is used to specify the bound for a type +parameter. + +C# uses declaration-site variance using the keywords ``in`` and ``out`` for +contravariance and covariance, respectively. By default, type parameters are +invariant. + +C# provides no way to specify a default type argument. + +.. code-block:: c# + + // Generic class with bounds on type parameters + public class ClassA + where T : SomeClass1 + where S : SomeClass2 + { + // Generic method + public void MyMethod(U value) where U : SomeClass3 { } + } + + // Contravariant and covariant type parameters + public class ClassB + { + public T MyMethod(S value) { } + } + + + +TypeScript +---------- + +TypeScript uses angle brackets to declare type parameters and for +specialization. The ``extends`` keyword is used to specify a bound. It can be +combined with other type operators such as ``keyof``. + +TypeScript uses declaration-site variance. Variance is inferred from +usage, not specified explicitly. TypeScript 4.7 introduced the ability +to specify variance using ``in`` and ``out`` keywords. This was added to handle +extremely complex types where inference of variance was expensive. + +A default type argument can be specified using the ``=`` operator. + +TypeScript supports the ``type`` keyword to declare a type alias, and this +syntax supports generics. + +.. code-block:: typescript + + // Generic interface + interface InterfaceA { + val1: S; + val2: T; + + method1(val: U): S + } + + // Generic function + function func1(ojb: T, key: K) { } + + // Contravariant and covariant type parameters (TypeScript 4.7) + interface InterfaceB { } + + // Type parameter with default + interface InterfaceC { } + + // Generic type alias + type MyType = Array + + +Scala +----- + +In Scala, square brackets are used to declare type parameters. Square +brackets are also used for specialization. The ``<:`` and ``>:`` operators +are used to specify upper and lower bounds, respectively. + +Scala uses use-site variance but also allows declaration-site variance +specification. It uses a ``+`` or ``-`` prefix operator for covariance and +contravariance, respectively. + +Scala provides no way to specify a default type argument. + +It does support higher-kinded types (type parameters that accept type +type parameters). + +.. code-block:: scala + + + // Generic class; type parameter has upper bound + class ClassA[A <: SomeClass1] + { + // Generic method; type parameter has lower bound + def method1[B >: A](val: B) ... + } + + // Use of an upper and lower bound with the same type parameter + class ClassB[A >: SomeClass1 <: SomeClass2] { } + + // Contravariant and covariant type parameters + class ClassC[+A, -B] { } + + // Higher-kinded type + trait Collection[T[_]] + { + def method1[A](a: A): T[A] + def method2[B](b: T[B]): B + } + + // Generic type alias + type MyType[T <: Int] = Container[T] + + +Swift +----- + +Swift uses angle brackets to declare type parameters and for specialization. +The upper bound of a type parameter is specified using a colon. + +Swift doesn't support generic variance; all type parameters are invariant. + +Swift provides no way to specify a default type argument. + +.. code-block:: swift + + // Generic class + class ClassA { + // Generic method + func method1(val: T) -> X { } + } + + // Type parameter with upper bound constraint + class ClassB {} + + // Generic type alias + typealias MyType = Container + + +Rust +---- + +Rust uses angle brackets to declare type parameters and for specialization. +The upper bound of a type parameter is specified using a colon. Alternatively +a ``where`` clause can specify various constraints. + +Rust does not have traditional object oriented inheritance or variance. +Subtyping in Rust is very restricted and occurs only due to variance with +respect to lifetimes. + +A default type argument can be specified using the ``=`` operator. + +.. code-block:: rust + + // Generic class + struct StructA { // T's lifetime is inferred as covariant + x: T + } + + fn f<'a>( + mut short_lifetime: StructA<&'a i32>, + mut long_lifetime: StructA<&'static i32>, + ) { + long_lifetime = short_lifetime; + // error: StructA<&'a i32> is not a subtype of StructA<&'static i32> + short_lifetime = long_lifetime; + // valid: StructA<&'static i32> is a subtype of StructA<&'a i32> + } + + // Type parameter with bound + struct StructB {} + + // Type parameter with additional constraints + struct StructC + where + T: Iterator, + T::Item: Copy + {} + + // Generic function + fn func1(val: &[T]) -> T { } + + // Generic type alias + type MyType = StructC; + + +Kotlin +------ + +Kotlin uses angle brackets to declare type parameters and for specialization. +By default, type parameters are invariant. The upper bound of a type is +specified using a colon. +Alternatively, a ``where`` clause can specify various constraints. + +Kotlin supports declaration-site variance where variance of type parameters is +explicitly declared using ``in`` and ``out`` keywords. It also supports use-site +variance which limits which methods and members can be used. + +Kotlin provides no way to specify a default type argument. + +.. code-block:: kotlin + + // Generic class + class ClassA + + // Type parameter with upper bound + class ClassB + + // Contravariant and covariant type parameters + class ClassC + + // Generic function + fun func1(): T { + + // Use site variance + val covariantA: ClassA + val contravariantA: ClassA + } + + // Generic type alias + typealias TypeAliasFoo = ClassA + + +Julia +----- + +Julia uses curly braces to declare type parameters and for specialization. +The ``<:`` operator can be used within a ``where`` clause to declare +upper and lower bounds on a type. + +.. code-block:: julia + + # Generic struct; type parameter with upper and lower bounds + # Valid for T in (Int64, Signed, Integer, Real, Number) + struct Container{Int <: T <: Number} + x::T + end + + # Generic function + function func1(v::Container{T}) where T <: Real end + + # Alternate forms of generic function + function func2(v::Container{T} where T <: Real) end + function func3(v::Container{<: Real}) end + + # Tuple types are covariant + # Valid for func4((2//3, 3.5)) + function func4(t::Tuple{Real,Real}) end + +Dart +---- + +Dart uses angle brackets to declare type parameters and for specialization. +The upper bound of a type is specified using the ``extends`` keyword. +By default, type parameters are covariant. + +Dart supports declaration-site variance, where variance of type parameters is +explicitly declared using ``in``, ``out`` and ``inout`` keywords. +It does not support use-site variance. + +Dart provides no way to specify a default type argument. + +.. code-block:: dart + + // Generic class + class ClassA { } + + // Type parameter with upper bound + class ClassB { } + + // Contravariant and covariant type parameters + class ClassC { } + + // Generic function + T func1() { } + + // Generic type alias + typedef TypeDefFoo = ClassA; + +Go +-- + +Go uses square brackets to declare type parameters and for specialization. +The upper bound of a type is specified after the name of the parameter, and +must always be specified. The keyword ``any`` is used for an unbound type parameter. + +Go doesn't support variance; all type parameters are invariant. + +Go provides no way to specify a default type argument. + +Go does not support generic type aliases. + +.. code-block:: go + + // Generic type without a bound + type TypeA[T any] struct { + t T + } + + // Type parameter with upper bound + type TypeB[T SomeType1] struct { } + + // Generic function + func func1[T any]() { } + + +Summary +------- + ++------------+----------+---------+--------+----------+-----------+-----------+ +| | Decl | Upper | Lower | Default | Variance | Variance | +| | Syntax | Bound | Bound | Value | Site | | ++============+==========+=========+========+==========+===========+===========+ +| C++ | template | n/a | n/a | = | n/a | n/a | +| | <> | | | | | | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Java | <> | extends | | | use | super, | +| | | | | | | extends | ++------------+----------+---------+--------+----------+-----------+-----------+ +| C# | <> | where | | | decl | in, out | ++------------+----------+---------+--------+----------+-----------+-----------+ +| TypeScript | <> | extends | | = | decl | inferred, | +| | | | | | | in, out | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Scala | [] | T <: X | T >: X | | use, decl | +, - | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Swift | <> | T: X | | | n/a | n/a | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Rust | <> | T: X, | | = | n/a | n/a | +| | | where | | | | | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Kotlin | <> | T: X, | | | use, decl | in, out | +| | | where | | | | | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Julia | {} | T <: X | X <: T | | n/a | n/a | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Dart | <> | extends | | | decl | in, out, | +| | | | | | | inout | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Go | [] | T X | | | n/a | n/a | ++------------+----------+---------+--------+----------+-----------+-----------+ +| Python | [] | T: X | | | decl | inferred | +| (proposed) | | | | | | | ++------------+----------+---------+--------+----------+-----------+-----------+ + + +Acknowledgements +================ + +Thanks to Sebastian Rittau for kick-starting the discussions that led to this +proposal, to Jukka Lehtosalo for proposing the syntax for type alias +statements and to Jelle Zijlstra, Daniel Moisset, and Guido van Rossum +for their valuable feedback and suggested improvements to the specification +and implementation. + + +Copyright +========= + +This document is placed in the public domain or under the CC0-1.0-Universal +license, whichever is more permissive. diff --git a/peps/pep-0696.rst b/peps/pep-0696.rst new file mode 100644 index 000000000..7e8f7f1e1 --- /dev/null +++ b/peps/pep-0696.rst @@ -0,0 +1,586 @@ +PEP: 696 +Title: Type defaults for TypeVarLikes +Author: James Hilton-Balfe +Sponsor: Jelle Zijlstra +Discussions-To: https://mail.python.org/archives/list/typing-sig@python.org/thread/7VWBZWXTCX6RAJO6GG67BAXUPFZ24NTC +Status: Draft +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 14-Jul-2022 +Python-Version: 3.12 +Post-History: `22-Mar-2022 `__, + `08-Jan-2023 `__, + +Abstract +-------- + +This PEP introduces the concept of type defaults for +``TypeVarLike``\ s (``TypeVar``, ``ParamSpec`` and ``TypeVarTuple``), +which act as defaults for a type parameter when one is not specified or +the constraint solver isn't able to solve a type parameter to anything. + +Default type argument support is available in some popular languages +such as C++, TypeScript, and Rust. A survey of type parameter syntax in +some common languages has been conducted by the author of :pep:`695` +and can be found in its +:pep:`Appendix A <695#appendix-a-survey-of-type-parameter-syntax>`. + + +Motivation +---------- + +.. code-block:: py + + T = TypeVar("T", default=int) # This means that if no type is specified T = int + + @dataclass + class Box(Generic[T]): + value: T | None = None + + reveal_type(Box()) # type is Box[int] + reveal_type(Box(value="Hello World!")) # type is Box[str] + +One place this `regularly comes +up `__ is ``Generator``. I +propose changing the *stub definition* to something like: + +.. code-block:: py + + YieldT = TypeVar("YieldT") + SendT = TypeVar("SendT", default=None) + ReturnT = TypeVar("ReturnT", default=None) + + class Generator(Generic[YieldT, SendT, ReturnT]): ... + + Generator[int] == Generator[int, None] == Generator[int, None, None] + +This is also useful for a ``Generic`` that is commonly over one type. + +.. code-block:: py + + class Bot: ... + + BotT = TypeVar("BotT", bound=Bot, default=Bot) + + class Context(Generic[BotT]): + bot: BotT + + class MyBot(Bot): ... + + reveal_type(Context().bot) # type is Bot # notice this is not Any which is what it would be currently + reveal_type(Context[MyBot]().bot) # type is MyBot + +Not only does this improve typing for those who explicitly use it, it +also helps non-typing users who rely on auto-complete to speed up their +development. + +This design pattern is common in projects like: + - `discord.py `__ — where the + example above was taken from. + - `NumPy `__ — the default for types + like ``ndarray``'s ``dtype`` would be ``float64``. Currently it's + ``Unknown`` or ``Any``. + - `TensorFlow `__ — this + could be used for Tensor similarly to ``numpy.ndarray`` and would be + useful to simplify the definition of ``Layer``. + + +Specification +------------- + +Default Ordering and Subscription Rules +''''''''''''''''''''''''''''''''''''''' + +The order for defaults should follow the standard function parameter +rules, so a ``TypeVarLike`` with no ``default`` cannot follow one with +a ``default`` value. Doing so should ideally raise a ``TypeError`` in +``typing._GenericAlias``/``types.GenericAlias``, and a type checker +should flag this an error. + +.. code-block:: py + + DefaultStrT = TypeVar("DefaultStrT", default=str) + DefaultIntT = TypeVar("DefaultIntT", default=int) + DefaultBoolT = TypeVar("DefaultBoolT", default=bool) + T = TypeVar("T") + T2 = TypeVar("T2") + + class NonDefaultFollowsDefault(Generic[DefaultStrT, T]): ... # Invalid: non-default TypeVars cannot follow ones with defaults + + + class NoNonDefaults(Generic[DefaultStrT, DefaultIntT]): ... + + ( + NoNoneDefaults == + NoNoneDefaults[str] == + NoNoneDefaults[str, int] + ) # All valid + + + class OneDefault(Generic[T, DefaultBoolT]): ... + + OneDefault[float] == OneDefault[float, bool] # Valid + reveal_type(OneDefault) # type is type[OneDefault[T, DefaultBoolT = bool]] + reveal_type(OneDefault[float]()) # type is OneDefault[float, bool] + + + class AllTheDefaults(Generic[T1, T2, DefaultStrT, DefaultIntT, DefaultBoolT]): ... + + reveal_type(AllTheDefaults) # type is type[AllTheDefaults[T1, T2, DefaultStrT = str, DefaultIntT = int, DefaultBoolT = bool]] + reveal_type(AllTheDefaults[int, complex]()) # type is AllTheDefaults[int, complex, str, int, bool] + AllTheDefaults[int] # Invalid: expected 2 arguments to AllTheDefaults + ( + AllTheDefaults[int, complex] == + AllTheDefaults[int, complex, str] == + AllTheDefaults[int, complex, str, int] == + AllTheDefaults[int, complex, str, int, bool] + ) # All valid + +This cannot be enforced at runtime for functions, for now, but in the +future, this might be possible (see `Interaction with PEP +695 <#interaction-with-pep-695>`__). + +``ParamSpec`` Defaults +'''''''''''''''''''''' + +``ParamSpec`` defaults are defined using the same syntax as +``TypeVar`` \ s but use a ``list`` of types or an ellipsis +literal "``...``" or another in-scope ``ParamSpec`` (see `Scoping Rules`_). + +.. code-block:: py + + DefaultP = ParamSpec("DefaultP", default=[str, int]) + + class Foo(Generic[DefaultP]): ... + + reveal_type(Foo) # type is type[Foo[DefaultP = [str, int]]] + reveal_type(Foo()) # type is Foo[[str, int]] + reveal_type(Foo[[bool, bool]]()) # type is Foo[[bool, bool]] + +``TypeVarTuple`` Defaults +''''''''''''''''''''''''' + +``TypeVarTuple`` defaults are defined using the same syntax as +``TypeVar`` \ s but use an unpacked tuple of types instead of a single type +or another in-scope ``TypeVarTuple`` (see `Scoping Rules`_). + +.. code-block:: py + + DefaultTs = TypeVarTuple("DefaultTs", default=Unpack[tuple[str, int]]) + + class Foo(Generic[*DefaultTs]): ... + + reveal_type(Foo) # type is type[Foo[DefaultTs = *tuple[str, int]]] + reveal_type(Foo()) # type is Foo[str, int] + reveal_type(Foo[int, bool]()) # type is Foo[int, bool] + +Using Another ``TypeVarLike`` as ``default`` +'''''''''''''''''''''''''''''''''''''''''''' + +This allows for a value to be used again when the constraints solver +fails to solve a constraint for a type, or the type parameter to a +generic is missing but another type parameter is specified. + +To use another ``TypeVarLike`` as a default the ``default`` and the +``TypeVarLike`` must be the same type (a ``TypeVar``'s default must be +a ``TypeVar``, etc.). + +`This could be used on builtins.slice `__ +where the ``start`` parameter should default to ``int``, ``stop`` +default to the type of ``start`` and step default to ``int | None``. + +.. code-block:: py + + StartT = TypeVar("StartT", default=int) + StopT = TypeVar("StopT", default=StartT) + StepT = TypeVar("StepT", default=int | None) + + class slice(Generic[StartT, StopT, StepT]): ... + + reveal_type(slice) # type is type[slice[StartT = int, StopT = StartT, StepT = int | None]] + reveal_type(slice()) # type is slice[int, int, int | None] + reveal_type(slice[str]()) # type is slice[str, str, int | None] + reveal_type(slice[str, bool, timedelta]()) # type is slice[str, bool, timedelta] + + T2 = TypeVar("T2", default=DefaultStrT) + + class Foo(Generic[DefaultStrT, T2]): + def __init__(self, a: DefaultStrT, b: T2) -> None: ... + + reveal_type(Foo(1, "")) # type is Foo[int, str] + Foo[int](1, "") # Invalid: Foo[int, str] cannot be assigned to self: Foo[int, int] in Foo.__init__ + Foo[int]("", 1) # Invalid: Foo[str, int] cannot be assigned to self: Foo[int, int] in Foo.__init__ + +When using a ``TypeVarLike`` as the default to another ``TypeVarLike``. +Where ``T1`` is the default for ``T2`` the following rules apply. + +``TypeVarTuple``\s are not supported because: + +- `Scoping Rules`_ does not allow usage of ``TypeVarLikes`` + from outer scopes. +- Multiple ``TypeVarTuple``\s cannot appear in the type + parameter list for a single class, as specified in + :pep:`646#multiple-type-variable-tuples-not-allowed`. +- ``TypeVarLike`` defaults in functions are not supported. + +These reasons leave no current valid location where a +``TypeVarTuple`` could have a default. + +Scoping Rules +~~~~~~~~~~~~~ + +``T1`` must be used before ``T2`` in the parameter list of the generic. + +.. code-block:: py + + DefaultT = TypeVar("DefaultT", default=T) + + class Foo(Generic[T, DefaultT]): ... # Valid + class Foo(Generic[T]): + class Bar(Generic[DefaultT]): ... # Valid + + StartT = TypeVar("StartT", default="StopT") # Swapped defaults around from previous example + StopT = TypeVar("StopT", default=int) + class slice(Generic[StartT, StopT, StepT]): ... + # ^^^^^^ Invalid: ordering does not allow StopT to be bound + +Using a ``TypeVarLike`` from an outer scope as a default is not supported. + +Bound Rules +~~~~~~~~~~~ + +``T2``'s bound must be a subtype of ``T1``'s bound. + +.. code-block:: py + + T = TypeVar("T", bound=float) + TypeVar("Ok", default=T, bound=int) # Valid + TypeVar("AlsoOk", default=T, bound=float) # Valid + TypeVar("Invalid", default=T, bound=str) # Invalid: str is not a subtype of float + +Constraint Rules +~~~~~~~~~~~~~~~~ + +The constraints of ``T2`` must be a superset of the constraints of ``T1``. + +.. code-block:: py + + T1 = TypeVar("T1", bound=int) + TypeVar("Invalid", float, str, default=T1) # Invalid: upper bound int is incompatible with constraints float or str + + T1 = TypeVar("T1", int, str) + TypeVar("AlsoOk", int, str, bool, default=T1) # Valid + TypeVar("AlsoInvalid", bool, complex, default=T1) # Invalid: {bool, complex} is not a superset of {int, str} + + +``TypeVarLike``\s as Parameters to Generics +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``TypeVarLike``\ s are valid as parameters to generics inside of a +``default`` when the first parameter is in scope as determined by the +`previous section `_. + +.. code-block:: py + + T = TypeVar("T") + ListDefaultT = TypeVar("ListDefaultT", default=list[T]) + + class Bar(Generic[T, ListDefaultT]): + def __init__(self, x: T, y: ListDefaultT): ... + + reveal_type(Bar) # type is type[Bar[T, ListDefaultT = list[T]]] + reveal_type(Bar[int]) # type is type[Bar[int, list[int]]] + reveal_type(Bar[int]()) # type is Bar[int, list[int]] + reveal_type(Bar[int, list[str]]()) # type is Bar[int, list[str]] + reveal_type(Bar[int, str]()) # type is Bar[int, str] + +Specialisation Rules +~~~~~~~~~~~~~~~~~~~~ + +``TypeVarLike``\ s currently cannot be further subscripted. This might +change if `Higher Kinded TypeVars `__ +are implemented. + + +``Generic`` ``TypeAlias``\ es +''''''''''''''''''''''''''''' + +``Generic`` ``TypeAlias``\ es should be able to be further subscripted +following normal subscription rules. If a ``TypeVarLike`` has a default +that hasn't been overridden it should be treated like it was +substituted into the ``TypeAlias``. However, it can be specialised +further down the line. + +.. code-block:: py + + class SomethingWithNoDefaults(Generic[T, T2]): ... + + MyAlias: TypeAlias = SomethingWithNoDefaults[int, DefaultStrT] # Valid + reveal_type(MyAlias) # type is type[SomethingWithNoDefaults[int, DefaultStrT]] + reveal_type(MyAlias[bool]()) # type is SomethingWithNoDefaults[int, bool] + + MyAlias[bool, int] # Invalid: too many arguments passed to MyAlias + +Subclassing +''''''''''' + +Subclasses of ``Generic``\ s with ``TypeVarLike``\ s that have defaults +behave similarly to ``Generic`` ``TypeAlias``\ es. + +.. code-block:: py + + class SubclassMe(Generic[T, DefaultStrT]): + x: DefaultStrT + + class Bar(SubclassMe[int, DefaultStrT]): ... + reveal_type(Bar) # type is type[Bar[DefaultStrT = str]] + reveal_type(Bar()) # type is Bar[str] + reveal_type(Bar[bool]()) # type is Bar[bool] + + class Foo(SubclassMe[float]): ... + + reveal_type(Foo().x) # type is str + + Foo[str] # Invalid: Foo cannot be further subscripted + + class Baz(Generic[DefaultIntT, DefaultStrT]): ... + + class Spam(Baz): ... + reveal_type(Spam()) # type is + +Using ``bound`` and ``default`` +''''''''''''''''''''''''''''''' + +If both ``bound`` and ``default`` are passed ``default`` must be a +subtype of ``bound``. Otherwise the type checker should generate an +error. + +.. code-block:: py + + TypeVar("Ok", bound=float, default=int) # Valid + TypeVar("Invalid", bound=str, default=int) # Invalid: the bound and default are incompatible + +Constraints +''''''''''' + +For constrained ``TypeVar``\ s, the default needs to be one of the +constraints. A type checker should generate an error even if it is a +subtype of one of the constraints. + +.. code-block:: py + + TypeVar("Ok", float, str, default=float) # Valid + TypeVar("Invalid", float, str, default=int) # Invalid: expected one of float or str got int + +.. _696-function-defaults: + +Function Defaults +''''''''''''''''' + +``TypeVarLike``\ s currently are not supported in the signatures of +functions as ensuring the ``default`` is returned in every code path +where the ``TypeVarLike`` can go unsolved is too hard to implement. + +Binding rules +------------- + +``TypeVarLikes`` defaults should be bound by attribute access +(including call and subscript). + +.. code-block:: python + + class Foo[T = int]: + def meth(self) -> Self: + return self + + reveal_type(Foo.meth) # type is (self: Foo[int]) -> Foo[int] + + +Implementation +-------------- + +At runtime, this would involve the following changes to the ``typing`` +module. + +- The classes ``TypeVar``, ``ParamSpec``, and ``TypeVarTuple`` should + expose the type passed to ``default``. This would be available as + a ``__default__`` attribute, which would be ``None`` if no argument + is passed and ``NoneType`` if ``default=None``. + +The following changes would be required to both ``GenericAlias``\ es: + +- logic to determine the defaults required for a subscription. +- ideally, logic to determine if subscription (like + ``Generic[T, DefaultT]``) would be valid. + +A reference implementation of the runtime changes can be found at +https://github.com/Gobot1234/cpython/tree/pep-696 + +A reference implementation of the type checker can be found at +https://github.com/Gobot1234/mypy/tree/TypeVar-defaults + +Pyright currently supports this functionality. + + +Interaction with PEP 695 +------------------------ + +The syntax proposed in :pep:`695` will be extended to introduce a way +to specify defaults for type parameters using the "=" operator inside +of the square brackets like so: + +.. code-block:: py + + # TypeVars + class Foo[T = str]: ... + + # ParamSpecs + class Baz[**P = [int, str]]: ... + + # TypeVarTuples + class Qux[*Ts = *tuple[int, bool]]: ... + + # TypeAliases + type Foo[T, U = str] = Bar[T, U] + type Baz[**P = [int, str]] = Spam[**P] + type Qux[*Ts = *tuple[str]] = Ham[*Ts] + type Rab[U, T = str] = Bar[T, U] + +:ref:`Similarly to the bound for a type parameter <695-scoping-behavior>`, +defaults should be lazily evaluated, with the same scoping rules to +avoid the unnecessary usage of quotes around them. + +This functionality was included in the initial draft of :pep:`695` but +was removed due to scope creep. + +Grammar Changes +''''''''''''''' + +:: + + type_param: + | a=NAME b=[type_param_bound] d=[type_param_default] + | a=NAME c=[type_param_constraint] d=[type_param_default] + | '*' a=NAME d=[type_param_default] + | '**' a=NAME d=[type_param_default] + + type_param_default: + | '=' e=expression + | '=' e=starred_expression + +This would mean that ``TypeVarLike``\ s with defaults proceeding those +with non-defaults can be checked at compile time. + + +Rejected Alternatives +--------------------- + +Allowing the ``TypeVarLike``\s Defaults to Be Passed to ``type.__new__``'s ``**kwargs`` +''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +.. code-block:: py + + T = TypeVar("T") + + @dataclass + class Box(Generic[T], T=int): + value: T | None = None + +While this is much easier to read and follows a similar rationale to the +``TypeVar`` `unary +syntax `__, it would not be +backwards compatible as ``T`` might already be passed to a +metaclass/superclass or support classes that don't subclass ``Generic`` +at runtime. + +Ideally, if :pep:`637` wasn't rejected, the following would be acceptable: + +.. code-block:: py + + T = TypeVar("T") + + @dataclass + class Box(Generic[T = int]): + value: T | None = None + +Allowing Non-defaults to Follow Defaults +'''''''''''''''''''''''''''''''''''''''' + +.. code-block:: py + + YieldT = TypeVar("YieldT", default=Any) + SendT = TypeVar("SendT", default=Any) + ReturnT = TypeVar("ReturnT") + + class Coroutine(Generic[YieldT, SendT, ReturnT]): ... + + Coroutine[int] == Coroutine[Any, Any, int] + +Allowing non-defaults to follow defaults would alleviate the issues with +returning types like ``Coroutine`` from functions where the most used +type argument is the last (the return). Allowing non-defaults to follow +defaults is too confusing and potentially ambiguous, even if only the +above two forms were valid. Changing the argument order now would also +break a lot of codebases. This is also solvable in most cases using a +``TypeAlias``. + +.. code-block:: py + + Coro: TypeAlias = Coroutine[Any, Any, T] + Coro[int] == Coroutine[Any, Any, int] + +Having ``default`` Implicitly Be ``bound`` +'''''''''''''''''''''''''''''''''''''''''' + +In an earlier version of this PEP, the ``default`` was implicitly set +to ``bound`` if no value was passed for ``default``. This while +convenient, could have a ``TypeVarLike`` with no default follow a +``TypeVarLike`` with a default. Consider: + +.. code-block:: py + + T = TypeVar("T", bound=int) # default is implicitly int + U = TypeVar("U") + + class Foo(Generic[T, U]): + ... + + # would expand to + + T = TypeVar("T", bound=int, default=int) + U = TypeVar("U") + + class Foo(Generic[T, U]): + ... + +This would have also been a breaking change for a small number of cases +where the code relied on ``Any`` being the implicit default. + +Allowing ``TypeVarLike``\s with defaults to be used in function signatures +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +A previous version of this PEP allowed ``TypeVarLike``\s with defaults to be used in +function signatures. This was removed for the reasons described in +`Function Defaults`_. Hopefully, this can be added in the future if +a way to get the runtime value of a type parameter is added. + +Allowing ``TypeVarLikes`` from outer scopes in ``default`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +This was deemed too niche a feature to be worth the added complexity. +If any cases arise where this is needed, it can be added in a future PEP. + +Acknowledgements +---------------- + +Thanks to the following people for their feedback on the PEP: + +Eric Traut, Jelle Zijlstra, Joshua Butt, Danny Yamamoto, Kaylynn Morgan +and Jakub Kuczys + + +Copyright +--------- +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0697.rst b/peps/pep-0697.rst new file mode 100644 index 000000000..b44300cca --- /dev/null +++ b/peps/pep-0697.rst @@ -0,0 +1,556 @@ +PEP: 697 +Title: Limited C API for Extending Opaque Types +Author: Petr Viktorin +Discussions-To: https://discuss.python.org/t/19743 +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Created: 23-Aug-2022 +Python-Version: 3.12 +Post-History: `24-May-2022 `__, + `06-Oct-2022 `__, +Resolution: https://discuss.python.org/t/19743/30 + + +.. canonical-doc:: + :external+py3.12:c:member:`PyType_Spec.basicsize`, + :external+py3.12:c:func:`PyObject_GetTypeData`, + :external+py3.12:c:macro:`Py_TPFLAGS_ITEMS_AT_END`, + :external+py3.12:c:macro:`Py_RELATIVE_OFFSET`, + :external+py3.12:c:func:`PyObject_GetItemData` + + +Abstract +======== + +Add `Limited C API `__ +support for extending some types with opaque data +by allowing code to only deal with data specific to a particular (sub)class. + +This mechanism is required to be usable with ``PyHeapTypeObject``. + +This PEP does not propose allowing to extend non-dynamically sized variable +sized objects such as ``tuple`` or ``int`` due to their different memory layout +and perceived lack of demand for doing so. This PEP leaves room to do so in +the future via the same mechanism if ever desired. + + +Motivation +========== + +The motivating problem this PEP solves is attaching C-level state +to custom types --- i.e. metaclasses (subclasses of +:py:class:`python:type`). + +This is often needed in “wrappers” that expose another type +system (e.g. C++, Java, Rust) as Python classes. +These typically need to attach information about the “wrapped” non-Python +class to the Python type object. + +This should be possible to do in the Limited API, so that the language wrappers +or code generators can be used to create Stable ABI extensions. +(See :pep:`652` for the benefits of providing a stable ABI.) + +Extending ``type`` is an instance of a more general problem: +extending a class while maintaining loose coupling – that is, +not depending on the memory layout used by the superclass. +(That's a lot of jargon; see Rationale for a concrete example of extending +``list``.) + + +Rationale +========= + +Extending opaque types +---------------------- + +In the Limited API, most ``struct``\ s are opaque: their size and memory layout +are not exposed, so they can be changed in new versions of CPython (or +alternate implementations of the C API). + +This means that the usual subclassing pattern -- making the ``struct`` +used for instances of the *base* type be the first element of the ``struct`` +used for instances of the *derived* type -- does not work. +To illustrate with code, the `example from the tutorial `_ +extends :external+python:c:type:`PyListObject` (:py:class:`python:list`) +using the following ``struct``: + +.. code-block:: c + + typedef struct { + PyListObject list; + int state; + } SubListObject; + +This won't compile in the Limited API, since ``PyListObject`` is opaque (to +allow changes as features and optimizations are implemented). + +Instead, this PEP proposes using a ``struct`` with only the state needed +in the subclass, that is: + +.. code-block:: c + + typedef struct { + int state; + } SubListState; + + // (or just `typedef int SubListState;` in this case) + +The subclass can now be completely decoupled from the memory layout (and size) +of the superclass. + +This is possible today. To use such a struct: + +* when creating the class, use ``PyListObject->tp_basicsize + sizeof(SubListState)`` + as ``PyType_Spec.basicsize``; +* when accessing the data, use ``PyListObject->tp_basicsize`` as the offset + into the instance (``PyObject*``). + +However, this has disadvantages: + +* The base's ``basicsize`` may not be properly aligned, causing issues + on some architectures if not mitigated. (These issues can be particularly + nasty if alignment changes in a new release.) +* ``PyTypeObject.tp_basicsize`` is not exposed in the + Limited API, so extensions that support Limited API need to + use ``PyObject_GetAttrString(obj, "__basicsize__")``. + This is cumbersome, and unsafe in edge cases (the Python attribute can + be overridden). +* Variable-size objects are not handled + (see `Extending variable-size objects`_ below). + +To make this easy (and even *best practice* for projects that choose loose +coupling over maximum performance), this PEP proposes an API to: + +1. During class creation, specify that ``SubListState`` + should be “appended” to ``PyListObject``, without passing any additional + details about ``list``. (The interpreter itself gets all necessary info, + like ``tp_basicsize``, from the base). + + This will be specified by a negative ``PyType_Spec.basicsize``: + ``-sizeof(SubListState)``. + +2. Given an instance, and the subclass ``PyTypeObject*``, + get a pointer to the ``SubListState``. + A new function, ``PyObject_GetTypeData``, will be added for this. + +The base class is not limited to ``PyListObject``, of course: it can be used to +extend any base class whose instance ``struct`` is opaque, unstable across +releases, or not exposed at all -- including :py:class:`python:type` +(``PyHeapTypeObject``) or third-party extensions +(for example, NumPy arrays [#f1]_). + +For cases where no additional state is needed, a zero ``basicsize`` will be +allowed: in that case, the base's ``tp_basicsize`` will be inherited. +(This currently works, but lacks explicit documentation and tests.) + +The ``tp_basicsize`` of the new class will be set to the computed total size, +so code that inspects classes will continue working as before. + + +Extending variable-size objects +------------------------------- + +Additional considerations are needed to subclass +:external+python:c:type:`variable-sized objects ` +while maintaining loose coupling: +the variable-sized data can collide with subclass data (``SubListState`` in +the example above). + +Currently, CPython doesn't provide a way to prevent such collisions. +So, the proposed mechanism of extending opaque classes (negative +``base->tp_itemsize``) will *fail* by default. + +We could stop there, but since the motivating type --- ``PyHeapTypeObject`` --- +is variable sized, we need a safe way to allow subclassing it. +A bit of background first: + +Variable-size layouts +..................... + +There are two main memory layouts for variable-sized objects. + +In types such as ``int`` or ``tuple``, the variable data is stored at a fixed +offset. +If subclasses need additional space, it must be added after any variable-sized +data:: + + PyTupleObject: + ┌───────────────────┬───┬───┬╌╌╌╌┐ + │ PyObject_VAR_HEAD │var. data │ + └───────────────────┮───┮───┮╌╌╌╌┘ + + tuple subclass: + ┌───────────────────┬───┬───┬╌╌╌╌┬─────────────┐ + │ PyObject_VAR_HEAD │var. data │subclass data│ + └───────────────────┮───┮───┮╌╌╌╌┮─────────────┘ + +In other types, like ``PyHeapTypeObject``, variable-sized data always lives at +the end of the instance's memory area:: + + heap type: + ┌───────────────────┬──────────────┬───┬───┬╌╌╌╌┐ + │ PyObject_VAR_HEAD │Heap type data│var. data │ + └───────────────────┮──────────────┮───┮───┮╌╌╌╌┘ + + type subclass: + ┌───────────────────┬──────────────┬─────────────┬───┬───┬╌╌╌╌┐ + │ PyObject_VAR_HEAD │Heap type data│subclass data│var. data │ + └───────────────────┮──────────────┮─────────────┮───┮───┮╌╌╌╌┘ + +The first layout enables fast access to the items array. +The second allows subclasses to ignore the variable-sized array (assuming +they use offsets from the start of the object to access their data). + +Since this PEP focuses on ``PyHeapTypeObject``, it proposes an API to allow +subclassing for the second variant. +Support for the first can be added later *as an API-compatible change* +(though your PEP author doubts it'd be worth the effort). + + +Extending classes with the ``PyHeapTypeObject``-like layout +........................................................... + +This PEP proposes a type flag, ``Py_TPFLAGS_ITEMS_AT_END``, which will indicate +the ``PyHeapTypeObject``-like layout. +This can be set in two ways: + +* the superclass can set the flag, allowing subclass authors to not care about + the fact that ``itemsize`` is involved, or +* the new subclass sets the flag, asserting that the author knows the + superclass is suitable (but perhaps hasn't been updated to use the flag yet). + +This flag will be necessary to extend a variable-sized type using negative +``basicsize``. + +An alternative to a flag would be to require subclass authors to know that the +base uses a compatible layout (e.g. from documentation). +A past version of this PEP proposed a new +``PyType_Slot`` for it. +This turned out to be hard to explain, and goes against the idea of decoupling +the subclass from the base layout. + +The new flag will be used to allow safely extending variable-sized types: +creating a type with ``spec->basicsize < 0`` and ``base->tp_itemsize > 0`` +will require the flag. + +Additionally, this PEP proposes a helper function to get the variable-sized +data of a given instance, if it uses the new ``Py_TPFLAGS_ITEMS_AT_END`` flag. +This hides the necessary pointer arithmetic behind an API +that can potentially be adapted to other layouts in the future (including, +potentially, a VM-managed layout). + +Big picture +........... + +To make it easier to verify that all cases are covered, here's a scary-looking +big-picture decision tree. + +.. note:: + + The individual cases are easier to explain in isolation (see the + `reference implementation `_ for draft docs). + +* ``spec->basicsize > 0``: No change to the status quo. (The base + class layout is known.) + +* ``spec->basicsize == 0``: (Inheriting the basicsize) + + * ``base->tp_itemsize == 0``: The item size is set to ``spec->tp_itemsize``. + (No change to status quo.) + * ``base->tp_itemsize > 0``: (Extending a variable-size class) + + * ``spec->itemsize == 0``: The item size is inherited. + (No change to status quo.) + * ``spec->itemsize > 0``: The item size is set. (This is hard to use safely, + but it's CPython's current behavior.) + +* ``spec->basicsize < 0``: (Extending the basicsize) + + * ``base->tp_itemsize == 0``: (Extending a fixed-size class) + + * ``spec->itemsize == 0``: The item size is set to 0. + * ``spec->itemsize > 0``: Fail. (We'd need to add an ``ob_size``, which is + only possible for trivial types -- and the trivial layout must be known.) + + * ``base->tp_itemsize > 0``: (Extending a variable-size class) + + * ``spec->itemsize == 0``: (Inheriting the itemsize) + + * ``Py_TPFLAGS_ITEMS_AT_END`` used: itemsize is inherited. + * ``Py_TPFLAGS_ITEMS_AT_END`` not used: Fail. (Possible conflict.) + + * ``spec->itemsize > 0``: Fail. (Changing/extending the item size can't be + done safely.) + +Setting ``spec->itemsize < 0`` is always an error. +This PEP does not propose any mechanism to *extend* ``tp->itemsize`` +rather than just inherit it. + + +Relative member offsets +----------------------- + +One more piece of the puzzle is ``PyMemberDef.offset``. +Extensions that use a subclass-specific ``struct`` (``SubListState`` above) +will get a way to specify “relative” offsets (offsets based from this +``struct``) rather than “absolute” ones (based off the ``PyObject`` struct). + +One way to do it would be to automatically assume “relative” offsets +when creating a class using the new API. +However, this implicit assumption would be too surprising. + +To be more explicit, this PEP proposes a new flag for “relative” offsets. +At least initially, this flag will serve only as a check against misuse +(and a hint for reviewers). +It must be present if used with the new API, and must not be used otherwise. + + +Specification +============= + +In the code blocks below, only function headers are part of the specification. +Other code (the size/offset calculations) are details of the initial CPython +implementation, and subject to change. + +Relative ``basicsize`` +---------------------- + +The ``basicsize`` member of ``PyType_Spec`` will be allowed to be zero or +negative. +In that case, its absolute value will specify how much *extra* storage space +instances of the new class require, in addition to the basicsize of the +base class. +That is, the basicsize of the resulting class will be: + +.. code-block:: c + + type->tp_basicsize = _align(base->tp_basicsize) + _align(-spec->basicsize); + +where ``_align`` rounds up to a multiple of ``alignof(max_align_t)``. + +When ``spec->basicsize`` is zero, basicsize will be inherited +directly instead, i.e. set to ``base->tp_basicsize`` without aligning. +(This already works; explicit tests and documentation will be added.) + +On an instance, the memory area specific to a subclass -- that is, the +“extra space” that subclass reserves in addition its base -- will be available +through a new function, ``PyObject_GetTypeData``. +In CPython, this function will be defined as: + +.. code-block:: c + + void * + PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) { + return (char *)obj + _align(cls->tp_base->tp_basicsize); + } + +Another function will be added to retrieve the size of this memory area: + +.. code-block:: c + + Py_ssize_t + PyType_GetTypeDataSize(PyTypeObject *cls) { + return cls->tp_basicsize - _align(cls->tp_base->tp_basicsize); + } + +The result may be higher than requested by ``-basicsize``. It is safe to +use all of it (e.g. with ``memset``). + +The new ``*Get*`` functions come with an important caveat, which will be +pointed out in documentation: They may only be used for classes created using +negative ``PyType_Spec.basicsize``. For other classes, their behavior is +undefined. +(Note that this allows the above code to assume ``cls->tp_base`` is not +``NULL``.) + + +Inheriting ``itemsize`` +----------------------- + +When ``spec->itemsize`` is zero, ``tp_itemsize`` will be inherited +from the base. +(This already works; explicit tests and documentation will be added.) + +A new type flag, ``Py_TPFLAGS_ITEMS_AT_END``, will be added. +This flag can only be set on types with non-zero ``tp_itemsize``. +It indicates that the variable-sized portion of an instance +is stored at the end of the instance's memory. + +The default metatype (``PyType_Type``) will set this flag. + +A new function, ``PyObject_GetItemData``, will be added to access the +memory reserved for variable-sized content of types with the new flag. +In CPython it will be defined as: + +.. code-block:: c + + void * + PyObject_GetItemData(PyObject *obj) { + if (!PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_ITEMS_AT_END) { + + } + return (char *)obj + Py_TYPE(obj)->tp_basicsize; + } + +This function will initially *not* be added to the Limited API. + +Extending a class with positive ``base->itemsize`` using +negative ``spec->basicsize`` will fail unless ``Py_TPFLAGS_ITEMS_AT_END`` +is set, either on the base or in ``spec->flags``. +(See `Extending variable-size objects`_ for a full explanation.) + +Extending a class with positive ``spec->itemsize`` using negative +``spec->basicsize`` will fail. + + +Relative member offsets +----------------------- + +In types defined using negative ``PyType_Spec.basicsize``, the offsets of +members defined via ``Py_tp_members`` must be relative to the +extra subclass data, rather than the full ``PyObject`` struct. +This will be indicated by a new flag in ``PyMemberDef.flags``: +``Py_RELATIVE_OFFSET``. + +In the initial implementation, the new flag will be redundant. It only serves +to make the offset's changed meaning clear, and to help avoid mistakes. +It will be an error to *not* use ``Py_RELATIVE_OFFSET`` with negative +``basicsize``, and it will be an error to use it in any other context +(i.e. direct or indirect calls to ``PyDescr_NewMember``, ``PyMember_GetOne``, +``PyMember_SetOne``). + +CPython will adjust the offset and clear the ``Py_RELATIVE_OFFSET`` flag when +intitializing a type. +This means that: + +* the created type's ``tp_members`` will not match the input + definition's ``Py_tp_members`` slot, and +* any code that reads ``tp_members`` will not need to handle the flag. + + +List of new API +=============== + +The following new functions/values are proposed. + +These will be added to the Limited API/Stable ABI: + +* ``void * PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls)`` +* ``Py_ssize_t PyType_GetTypeDataSize(PyTypeObject *cls)`` +* ``Py_TPFLAGS_ITEMS_AT_END`` flag for ``PyTypeObject.tp_flags`` +* ``Py_RELATIVE_OFFSET`` flag for ``PyMemberDef.flags`` + +These will be added to the public C API only: + +* ``void *PyObject_GetItemData(PyObject *obj)`` + + +Backwards Compatibility +======================= + +No backwards compatibility concerns are known. + + +Assumptions +=========== + +The implementation assumes that an instance's memory +between ``type->tp_base->tp_basicsize`` and ``type->tp_basicsize`` offsets +“belongs” to ``type`` (except variable-length types). +This is not documented explicitly, but CPython up to version 3.11 relied on it +when adding ``__dict__`` to subclasses, so it should be safe. + + +Security Implications +===================== + +None known. + + +Endorsements +============ + +The author of ``pybind11`` originally requested solving the issue +(see point 2 in `this list `__), +and `has been verifying the implementation `__. + +Florian from the HPy project `said `__ +that the API looks good in general. +(See `below `_ for a possible solution to +performance concerns.) + + +How to Teach This +================= + +The initial implementation will include reference documentation +and a What's New entry, which should be enough for the target audience +-- authors of C extension libraries. + + +Reference Implementation +======================== + +A reference implementation is in the `extend-opaque branch `__ +in the ``encukou/cpython`` GitHub repo. + + +Possible Future Enhancements +============================ + +Alignment & Performance +----------------------- + +The proposed implementation may waste some space if instance structs +need smaller alignment than ``alignof(max_align_t)``. +Also, dealing with alignment makes the calculation slower than it could be +if we could rely on ``base->tp_basicsize`` being properly aligned for the +subtype. + +In other words, the proposed implementation focuses on safety and ease of use, +and trades space and time for it. +If it turns out that this is a problem, the implementation can be adjusted +without breaking the API: + +- The offset to the type-specific buffer can be stored, so + ``PyObject_GetTypeData`` effectively becomes + ``(char *)obj + cls->ht_typedataoffset``, possibly speeding things up at + the cost of an extra pointer in the class. +- Then, a new ``PyType_Slot`` can specify the desired alignment, to + reduce space requirements for instances. + +Other layouts for variable-size types +------------------------------------- + +A flag like ``Py_TPFLAGS_ITEMS_AT_END`` could be added to signal the +“tuple-like” layout described in `Extending variable-size objects`_, +and all mechanisms this PEP proposes could be adapted to support it. +Other layouts could be added as well. +However, it seems there'd be very little practical benefit, +so it's just a theoretical possibility. + + +Rejected Ideas +============== + +Instead of a negative ``spec->basicsize``, a new ``PyType_Spec`` flag could've +been added. The effect would be the same to any existing code accessing these +internals without up to date knowledge of the change as the meaning of the +field value is changing in this situation. + + +Footnotes +========= + +.. [#f1] This PEP does not make it “safe” to subclass NumPy arrays specifically. + NumPy publishes `an extensive list of caveats `__ + for subclassing its arrays from Python, and extending in C might need + a similar list. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0698.rst b/peps/pep-0698.rst new file mode 100644 index 000000000..9363c4493 --- /dev/null +++ b/peps/pep-0698.rst @@ -0,0 +1,475 @@ +PEP: 698 +Title: Override Decorator for Static Typing +Author: Steven Troxler , + Joshua Xu , + Shannon Zhu +Sponsor: Jelle Zijlstra +Discussions-To: https://discuss.python.org/t/pep-698-a-typing-override-decorator/20839 +Status: Accepted +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 05-Sep-2022 +Python-Version: 3.12 +Post-History: `20-May-2022 `__, + `17-Aug-2022 `__, + `11-Oct-2022 `__, + `07-Nov-2022 `__, +Resolution: https://discuss.python.org/t/pep-698-a-typing-override-decorator/20839/11 + +Abstract +======== + +This PEP proposes adding an ``@override`` decorator to the Python type system. +This will allow type checkers to prevent a class of bugs that occur when a base +class changes methods that are inherited by derived classes. + + +Motivation +========== + +A primary purpose of type checkers is to flag when refactors or changes break +pre-existing semantic structures in the code, so users can identify and make +fixes across their project without doing a manual audit of their code. + + +Safe Refactoring +---------------- + +Python’s type system does not provide a way to identify call sites that need to +be changed to stay consistent when an overridden function API changes. This +makes refactoring and transforming code more dangerous. + +Consider this simple inheritance structure: + +.. code-block:: python + + class Parent: + def foo(self, x: int) -> int: + return x + + class Child(Parent): + def foo(self, x: int) -> int: + return x + 1 + + def parent_callsite(parent: Parent) -> None: + parent.foo(1) + + def child_callsite(child: Child) -> None: + child.foo(1) + + +If the overridden method on the superclass is renamed or deleted, type checkers +will only alert us to update call sites that deal with the base type directly. +But the type checker can only see the new code, not the change we made, so it +has no way of knowing that we probably also needed to rename the same method on +child classes. + +A type checker will happily accept this code, even though we are likely +introducing bugs: + +.. code-block:: python + + class Parent: + # Rename this method + def new_foo(self, x: int) -> int: + return x + + class Child(Parent): + # This (unchanged) method used to override `foo` but is unrelated to `new_foo` + def foo(self, x: int) -> int: + return x + 1 + + def parent_callsite(parent: Parent) -> None: + # If we pass a Child instance we’ll now run Parent.new_foo - likely a bug + parent.new_foo(1) + + def child_callsite(child: Child) -> None: + # We probably wanted to invoke new_foo here. Instead, we forked the method + child.foo(1) + + +This code will type check, but there are two potential sources of bugs: + +- If we pass a ``Child`` instance to the ``parent_callsite`` function, it will + invoke the implementation in ``Parent.new_foo``. rather than ``Child.foo``. + This is probably a bug - we presumably would not have written ``Child.foo`` in + the first place if we didn’t need custom behavior. +- Our system was likely relying on ``Child.foo`` behaving in a similar way to + ``Parent.foo``. But unless we catch this early, we have now forked the + methods, and in future refactors it is likely no one will realize that major + changes to the behavior of ``new_foo`` likely require updating ``Child.foo`` as + well, which could lead to major bugs later. + +The incorrectly-refactored code is type-safe, but is probably not what we +intended and could cause our system to behave incorrectly. The bug can be +difficult to track down because our new code likely does execute without +throwing exceptions. Tests are less likely to catch the problem, and silent +errors can take longer to track down in production. + +We are aware of several production outages in multiple typed codebases caused by +such incorrect refactors. This is our primary motivation for adding an ``@override`` +decorator to the type system, which lets developers express the relationship +between ``Parent.foo`` and ``Child.foo`` so that type checkers can detect the problem. + + +Rationale +========= + + +Subclass Implementations Become More Explicit +--------------------------------------------- + +We believe that explicit overrides will make unfamiliar code easier to read than +implicit overrides. A developer reading the implementation of a subclass that +uses ``@override`` can immediately see which methods are overriding +functionality in some base class; without this decorator, the only way to +quickly find out is using a static analysis tool. + + +Precedent in Other Languages and Runtime Libraries +-------------------------------------------------- + +Static Override Checks in Other Languages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many popular programming languages support override checks. For example: + +- `C++ has `_ ``override``. +- `C# has `_ ``override``. +- `Hack has `_ ``<<__Override>>``. +- `Java has `_ ``@Override``. +- `Kotlin has `_ ``override``. +- `Scala has `_ ``override``. +- `Swift has `_ ``override``. +- `TypeScript has `_ ``override``. + +Runtime Override Checks in Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Today, there is an `Overrides library `_ +that provides decorators ``@overrides`` [sic] and ``@final`` and will enforce +them at runtime. + +:pep:`591` added a ``@final`` decorator with the same semantics as those in the +Overrides library. But the override component of the runtime library is not +supported statically at all, which has added some confusion around the +mix/matched support. + +Providing support for ``@override`` in static checks would add value because: + +- Bugs can be caught earlier, often in-editor. +- Static checks come with no performance overhead, unlike runtime checks. +- Bugs will be caught quickly even in rarely-used modules, whereas with runtime + checks these might go undetected for a time without automated tests of all + imports. + + +Disadvantages +------------- + +Using ``@override`` will make code more verbose. + + +Specification +============= + +When type checkers encounter a method decorated with ``@typing.override`` they +should treat it as a type error unless that method is overriding a compatible +method or attribute in some ancestor class. + + +.. code-block:: python + + from typing import override + + class Parent: + def foo(self) -> int: + return 1 + + def bar(self, x: str) -> str: + return x + + class Child(Parent): + @override + def foo(self) -> int: + return 2 + + @override + def baz() -> int: # Type check error: no matching signature in ancestor + return 1 + + +The ``@override`` decorator should be permitted anywhere a type checker +considers a method to be a valid override, which typically includes not only +normal methods but also ``@property``, ``@staticmethod``, and ``@classmethod``. + + +No New Rules for Override Compatibility +--------------------------------------- + +This PEP is exclusively concerned with the handling of the new ``@override`` decorator, +which specifies that the decorated method must override some attribute in +an ancestor class. This PEP does not propose any new rules regarding the type +signatures of such methods. + + +Strict Enforcement Per-Project +============================== + +We believe that ``@override`` is most useful if checkers also allow developers +to opt into a strict mode where methods that override a parent class are +required to use the decorator. Strict enforcement should be opt-in for backward +compatibility. + + +Motivation +---------- + +The primary reason for a strict mode that requires ``@override`` is that developers +can only trust that refactors are override-safe if they know that the ``@override`` +decorator is used throughout the project. + +There is another class of bug related to overrides that we can only catch using a strict mode. + +Consider the following code: + +.. code-block:: python + + class Parent: + pass + + class Child(Parent): + def foo() -> int: + return 2 + +Imagine we refactor it as follows: + + +.. code-block:: python + + class Parent + def foo() -> int: # This method is new + return 1 + + class Child(Parent): + def foo() -> int: # This is now an override! + return 2 + + def call_foo(parent: Parent) -> int: + return parent.foo() # This could invoke Child.foo, which may be surprising. + +The semantics of our code changed here, which could cause two problems: + +- If the author of the code change did not know that ``Child.foo`` already + existed (which is very possible in a large codebase), they might be surprised + to see that ``call_foo`` does not always invoke ``Parent.foo``. +- If the codebase authors tried to manually apply ``@override`` everywhere when + writing overrides in subclasses, they are likely to miss the fact that + ``Child.foo`` needs it here. + + +At first glance this kind of change may seem unlikely, but it can actually +happen often if one or more subclasses have functionality that developers later +realize belongs in the base class. + +With a strict mode, we will always alert developers when this occurs. + +Precedent +--------- + +Most of the typed, object-oriented programming languages we looked at have an +easy way to require explicit overrides throughout a project: + +- C#, Kotlin, Scala, and Swift always require explicit overrides +- TypeScript has a + `--no-implicit-override `_ + flag to force explicit overrides +- In Hack and Java the type checker always treats overrides as opt-in, but + widely-used linters can warn if explicit overrides are missing. + + +Backward Compatibility +====================== + +By default, the ``@override`` decorator will be opt-in. Codebases that do not +use it will type-check as before, without the additional type safety. + + +Runtime Behavior +================ + +Set ``__override__ = True`` when possible +----------------------------------------- + +At runtime, ``@typing.override`` will make a best-effort attempt to add an +attribute ``__override__`` with value ``True`` to its argument. By "best-effort" +we mean that we will try adding the attribute, but if that fails (for example +because the input is a descriptor type with fixed slots) we will silently +return the argument as-is. + +This is exactly what the ``@typing.final`` decorator does, and the motivation +is similar: it gives runtime libraries the ability to use ``@override``. As a +concrete example, a runtime library could check ``__override__`` in order +to automatically populate the ``__doc__`` attribute of child class methods +using the parent method docstring. + +Limitations of setting ``__override__`` +--------------------------------------- + +As described above, adding ``__override__`` may fail at runtime, in which +case we will simply return the argument as-is. + +In addition, even in cases where it does work, it may be difficult for users to +correctly work with multiple decorators, because successfully ensuring the +``__override__`` attribute is set on the final output requires understanding the +implementation of each decorator: + +- The ``@override`` decorator needs to execute *after* ordinary decorators + like ``@functools.lru_cache`` that use wrapper functions, since we want to + set ``__override__`` on the outermost wrapper. This means it needs to + go *above* all these other decorators. +- But ``@override`` needs to execute *before* many special descriptor-based + decorators like ``@property``, ``@staticmethod``, and ``@classmethod``. +- As discussed above, in some cases (for example a descriptor with fixed + slots or a descriptor that also wraps) it may be impossible to set the + ``__override__`` attribute at all. + +As a result, runtime support for setting ``__override__`` is best effort +only, and we do not expect type checkers to validate the ordering of +decorators. + + +Rejected Alternatives +===================== + + +Rely on Integrated Development Environments for safety +------------------------------------------------------ + +Modern Integrated Development Environments (IDEs) often provide the ability to +automatically update subclasses when renaming a method. But we view this as +insufficient for several reasons: + +- If a codebase is split into multiple projects, an IDE will not help and the + bug appears when upgrading dependencies. Type checkers are a fast way to catch + breaking changes in dependencies. +- Not all developers use such IDEs. And library maintainers, even if they do use + an IDE, should not need to assume pull request authors use the same IDE. We + prefer being able to detect problems in continuous integration without + assuming anything about developers’ choice of editor. + + + +Runtime enforcement +------------------- + +We considered having ``@typing.override`` enforce override safety at runtime, +similarly to how ``@overrides.overrides`` +`does today `_. + +We rejected this for four reasons: + +- For users of static type checking, it is not clear this brings any benefits. +- There would be at least some performance overhead, leading to projects + importing slower with runtime enforcement. We estimate the + ``@overrides.overrides`` implementation takes around 100 microseconds, which + is fast but could still add up to a second or more of extra initialization + time in million-plus line codebases, which is exactly where we think + ``@typing.override`` will be most useful. +- An implementation may have edge cases where it doesn’t work well (we heard + from a maintainer of one such closed-source library that this has been a + problem). We expect static enforcement to be simple and reliable. +- The implementation approaches we know of are not simple. The decorator + executes before the class is finished evaluating, so the options we know of + are either to inspect the bytecode of the caller (as ``@overrides.overrides`` + does) or to use a metaclass-based approach. Neither approach seems ideal. + + +Mark a base class to force explicit overrides on subclasses +----------------------------------------------------------- + +We considered including a class decorator ``@require_explicit_overrides``, which +would have provided a way for base classes to declare that all subclasses must +use the ``@override`` decorator on method overrides. The +`Overrides library `_ has a mixin class, +``EnforceExplicitOverrides``, which provides similar behavior in runtime checks. + +We decided against this because we expect owners of large codebases will benefit +most from ``@override``, and for these use cases having a strict mode where +explicit ``@override`` is required (see the Backward Compatibility section) +provides more benefits than a way to mark base classes. + +Moreover we believe that authors of projects who do not consider the extra type +safety to be worth the additional boilerplate of using ``@override`` should not +be forced to do so. Having an optional strict mode puts the decision in the +hands of project owners, whereas the use of ``@require_explicit_overrides`` in +libraries would force project owners to use ``@override`` even if they prefer +not to. + +Include the name of the ancestor class being overridden +------------------------------------------------------- + +We considered allowing the caller of ``@override`` to specify a specific +ancestor class where the overridden method should be defined: + +.. code-block:: python + + class Parent0: + def foo() -> int: + return 1 + + + class Parent1: + def bar() -> int: + return 1 + + + class Child(Parent0, Parent1): + @override(Parent0) # okay, Parent0 defines foo + def foo() -> int: + return 2 + + @override(Parent0) # type error, Parent0 does not define bar + def bar() -> int: + return 2 + + +This could be useful for code readability because it makes the override +structure more explicit for deep inheritance trees. It also might catch bugs by +prompting developers to check that the implementation of an override still makes +sense whenever a method being overridden moves from one base class to another. + +We decided against it because: + +- Supporting this would add complexity to the implementation of both + ``@override`` and type checker support for it, so there would need to + be considerable benefits. +- We believe that it would be rarely used and catch relatively few bugs. + + - The author of the + `Overrides package `_ has + `noted `__ + that early versions of his library included this capability but it was + rarely useful and seemed to have little benefit. After it was removed, the + ability was never requested by users. + + + +Reference Implementation +======================== + +Pyre: A proof of concept is implemented in Pyre: + +- The decorator + `@pyre_extensions.override `_ + can mark overrides +- Pyre can `type-check this decorator `_ + as specified in this PEP + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0699.rst b/peps/pep-0699.rst new file mode 100644 index 000000000..75bd3e30e --- /dev/null +++ b/peps/pep-0699.rst @@ -0,0 +1,114 @@ +PEP: 699 +Title: Remove private dict version field added in PEP 509 +Author: Ken Jin +Discussions-To: https://discuss.python.org/t/pep-699-remove-private-dict-version-field-added-in-pep-509/19724 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 03-Oct-2022 +Python-Version: 3.12 +Post-History: `05-Oct-2022 `__ +Replaces: 509 +Resolution: https://discuss.python.org/t/pep-699-remove-private-dict-version-field-added-in-pep-509/19724/13 + + + +Abstract +======== + +:pep:`509` introduced a private ``ma_version_tag`` field for dictionaries to +allow optimizations in CPython and extension libraries. This PEP proposes to +rescind :pep:`509` and declare the field an implementation detail, as it has +already been superseded by alternatives. This will further allow the field to +be reused for future optimization. + + +Motivation +========== + +:pep:`509` introduced the ``ma_version_tag`` field to dictionaries. This 64-bit +field stored a version counter that updates every time a dictionary was +modified. The original proposal was to use this version counter as a +guard for optimizations. + +Since CPython 3.11, this field has become unused by internal optimization +efforts. :pep:`659`-specialized instructions use other methods of verifying +that certain optimizations are safe. + +To enable further optimizations in CPython, this PEP proposes that the +``ma_version_tag`` field no longer conform to the :pep:`509` specification. +This will allow the CPython developers to store other optimization information, +such as dictionary write watchers. + + +Rationale +========= +This PEP does not specify what the field may be used for in the future. This is +intentional, as implementation details are subject to change, and the field +should be used only for internal consumption by CPython. + + +Specification +============= + +This specification rescinds that in :pep:`509`. The ``ma_version_tag`` field of +the Python :class:`dict` class is declared to be an internal implementation +detail and may be removed altogether, or may have a different representation. +C extensions should not rely on this field. + + +Backwards Compatibility +======================= + +Certain extensions use ``ma_version_tag`` for fast dictionary or globals +lookups. For example, +`Cython uses the field for fast dynamic module variable lookups `_. + +This PEP proposes to emit a compiler warning when accessing ``ma_version_tag``. +After two consecutive version releases with warnings, ``ma_version_tag`` +will be removed, in line with :pep:`387`. + +The biggest user the author could find for this field was Cython. +Discussions with a Cython maintainer indicated that +`removing support for it from Cython is trivial `_. + + +Security Implications +===================== + +:pep:`509` was concerned with integer overflow. However, this PEP does not +introduce any additional security concerns. + + +Rejected Ideas +============== + +A possible alternative is to preserve the field for backwards compatibility. +This PEP rejects that alternative as future optimizations will consume more +memory, and the field was always considered private and undocumented aside +from the PEP, with no backward compatibility guarantees. Dictionaries in Python +are commonly used, and any increase in their memory consumption will adversely +affect Python’s performance. + + +Reference Implementation +======================== + +A reference implementation can be found in +`python/cpython#101193 `_. + + +Special Thanks +============== + +Thanks to C.A.M. Gerlach for edits and wording changes to this document. +Thanks also to Mark Shannon and Kumar Aditya for providing possible +implementations. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + diff --git a/peps/pep-0700.rst b/peps/pep-0700.rst new file mode 100644 index 000000000..af1f64a11 --- /dev/null +++ b/peps/pep-0700.rst @@ -0,0 +1,180 @@ +PEP: 700 +Title: Additional Fields for the Simple API for Package Indexes +Author: Paul Moore +PEP-Delegate: Donald Stufft +Discussions-To: https://discuss.python.org/t/pep-700-additional-fields-for-the-simple-api-for-package-indexes/20177 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 21-Oct-2022 +Post-History: `21-Oct-2022 `__ +Resolution: https://discuss.python.org/t/pep-700-additional-fields-for-the-simple-api-for-package-indexes/20177/42 + + +Abstract +======== + +:pep:`691` defined a JSON form for the "Simple Repository API". This allowed +clients to more easily query the data that was previously only available in +HTML, as defined in :pep:`503`. + +This proposal adds three fields to the JSON form, which allow it to be used in +place of PyPI's `JSON API `__ +in a number of situations. + +- A field to allow retrieval of a list of all the published versions of a project. +- Fields containing the size and upload time for a project file. + +The new fields are all part of the data returned from the "project details" URL. + + +Rationale +========= + +With the introduction of the JSON form of the simple API in :pep:`691`, the +simple API offers functionality that is almost as complete as the PyPI JSON API. +This PEP adds a number of fields which were previously only available through +the JSON API, in order to allow more clients which were previously Warehouse +specific to support arbitrary standards-compliant indexes. + + +Specification +============= + +This specification defines version 1.1 of the simple repository API. For the +HTML version of the API, there is no change from version 1.0. For the JSON +version of the API, the following changes are made: + +- The ``api-version`` must specify version 1.1 or later. +- A new ``versions`` key is added at the top level. +- Two new "file information" keys, ``size`` and ``upload-time``, are added to + the ``files`` data. +- Keys (at any level) with a leading underscore are reserved as private for + index server use. No future standard will assign a meaning to any such key. + +The ``versions`` and ``size`` keys are mandatory. The ``upload-time`` key is +optional. + +Versions +-------- + +An additional key, ``versions`` MUST be present at the top level, in addition to +the keys ``name``, ``files`` and ``meta`` defined in :pep:`691`. This key MUST +contain a list of version strings specifying all of the project versions uploaded +for this project. The value is logically a set, and as such may not contain +duplicates, and the order of the values is not significant. + +All of the files listed in the ``files`` key MUST be associated with one of the +versions in the ``versions`` key. The ``versions`` key MAY contain versions with +no associated files (to represent versions with no files uploaded, if the server +has such a concept). + +Note that because servers may hold "legacy" data from before the adoption of +:pep:`440`, version strings currently cannot be required to be valid :pep:`440` +versions, and therefore cannot be assumed to be orderable using the :pep:`440` +rules. However, servers SHOULD use normalised :pep:`440` versions where +possible. + + +Additional file information +--------------------------- + +Two new keys are added to the ``files`` key. + +- ``size``: This field is mandatory. It MUST contain an integer which is the + file size in bytes. +- ``upload-time``: This field is optional. If present, it MUST contain a valid + ISO 8601 date/time string, in the format ``yyyy-mm-ddThh:mm:ss.ffffffZ``, + which represents the time the file was uploaded to the index. As indicated by + the ``Z`` suffix, the upload time MUST use the UTC timezone. The fractional + seconds part of the timestamp (the ``.ffffff`` part) is optional, and if + present may contain up to 6 digits of precision. If a server does not record + upload time information for a file, it MAY omit the ``upload-time`` key. + + +FAQ +=== + +Why not add this data to the HTML API as well? +---------------------------------------------- + +It would be possible to add the data to the HTML API, but the vast majority of +consumers for this data are likely to be currently getting it from the PyPI JSON +API, and so will already be expecting to parse JSON. Traditional consumers of +the HTML API have never needed this data previously. + +Does this imply that the HTML API is obsolete? +---------------------------------------------- + +No. The FAQ of :pep:`691` was clear that the HTML API is not being deprecated, +and this PEP does not change that position. However, clients wishing to access +the new data introduced by this PEP will need to use the JSON API to get it. And +indexes wanting to provide it will need to serve the JSON format. + +Is the simple API replacing the Warehouse JSON and XML-RPC APIs? +---------------------------------------------------------------- + +Where possible, clients should prefer the simple API over the JSON or XML-RPC +APIs, as the former is standardised and can be assumed to be available from any +index, whereas the latter are exclusive to the Warehouse project. + +However, while this PEP brings the simple API closer to being able to replace +the JSON API, there is no formal policy that the simple API will replicate all +of the functionality covered by the existing Warehouse APIs. Proposed additions +to the simple API will still be considered on their individual merits, and the +requirement that the API should be simple and fast for the primary use case of +locating files for a project will remain the overriding consideration. + +Why not allow other date formats? +--------------------------------- + +The ISO 8601 standard is complex, and there seems little value in requiring +clients to deal with that. The standard library ``datetime`` module provides +methods to parse ISO 8601 strings, but it is possible that users may want to +access index data *without* using Python (for example, piping the output of +``curl`` into ``jq``). Having a single, well-defined format makes this easier, +and doesn't have any significant disadvantages. + +What if file sizes are too big for a JSON number? +------------------------------------------------- + +The JSON standard does not specify how numbers are to be interpreted. Python can +read and write arbitrary-length integers in a JSON file, so this should not be +an issue for code written in Python. Non-Python implementations may need to take +care to handle large integers correctly, but this is not expected to be a +significant problem. + +Why not require PEP 440 versions? +--------------------------------- + +At the time this PEP was written, PyPI still contains (and serves) projects and +files with "legacy" versions. Requiring :pep:`440` versions would make it +impossible for PyPI to follow this specification while still serving the +existing content. + +Ideally, at some future time, the simple index API will be updated to require +:pep:`440` versions, at which time this specification should be updated to +reflect that. However, that change will need to be co-ordinated with existing +index providers including PyPI, to desupport and remove non-conforming projects +and/or files. + +Why not provide a "latest version" value? +----------------------------------------- + +For :pep:`440` versions, this is easy enough for the client to do (using the +``packaging`` library, ``latest = max(Version(s) for s in proj["versions"])``). +For non-standard versions, there is no well-defined ordering, and clients will +need to decide on what rule is appropriate for their needs. Requiring the server +to supply a latest version value takes the choice away from the client. + +Servers with an explicit concept of which version is the "latest", which cannot +be calculated from data available to the client, can provide a non-standard, +underscore-prefixed key to convey that information to the client if they wish. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0701.rst b/peps/pep-0701.rst new file mode 100644 index 000000000..f4b393d0b --- /dev/null +++ b/peps/pep-0701.rst @@ -0,0 +1,625 @@ +PEP: 701 +Title: Syntactic formalization of f-strings +Author: Pablo Galindo , + Batuhan Taskaya , + Lysandros Nikolaou , + Marta GĂłmez MacĂ­as +Discussions-To: https://discuss.python.org/t/pep-701-syntactic-formalization-of-f-strings/22046 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 15-Nov-2022 +Python-Version: 3.12 +Post-History: `19-Dec-2022 `__, +Resolution: https://discuss.python.org/t/pep-701-syntactic-formalization-of-f-strings/22046/119 + + +Abstract +======== + +This document proposes to lift some of the restrictions originally formulated in +:pep:`498` and to provide a formalized grammar for f-strings that can be +integrated into the parser directly. The proposed syntactic formalization of +f-strings will have some small side-effects on how f-strings are parsed and +interpreted, allowing for a considerable number of advantages for end users and +library developers, while also dramatically reducing the maintenance cost of +the code dedicated to parsing f-strings. + + +Motivation +========== + +When f-strings were originally introduced in :pep:`498`, the specification was +provided without providing a formal grammar for f-strings. Additionally, the +specification contains several restrictions that are imposed so the parsing of +f-strings could be implemented into CPython without modifying the existing +lexer. These limitations have been recognized previously and previous attempts +have been made to lift them in :pep:`536`, but `none of this work was ever implemented`_. +Some of these limitations (collected originally by :pep:`536`) are: + +#. It is impossible to use the quote character delimiting the f-string + within the expression portion:: + + >>> f'Magic wand: { bag['wand'] }' + ^ + SyntaxError: invalid syntax + +#. A previously considered way around it would lead to escape sequences + in executed code and is prohibited in f-strings:: + + >>> f'Magic wand { bag[\'wand\'] } string' + SyntaxError: f-string expression portion cannot include a backslash + +#. Comments are forbidden even in multi-line f-strings:: + + >>> f'''A complex trick: { + ... bag['bag'] # recursive bags! + ... }''' + SyntaxError: f-string expression part cannot include '#' + +#. Arbitrary nesting of expressions without expansion of escape sequences is + available in many other languages that employ a string interpolation + method that uses expressions instead of just variable names. Some examples: + + .. code-block:: text + + # Ruby + "#{ "#{1+2}" }" + + # JavaScript + `${`${1+2}`}` + + # Swift + "\("\(1+2)")" + + # C# + $"{$"{1+2}"}" + +These limitations serve no purpose from a language user perspective and +can be lifted by giving f-string literals a regular grammar without exceptions +and implementing it using dedicated parse code. + +The other issue that f-strings have is that the current implementation in +CPython relies on tokenising f-strings as ``STRING`` tokens and a post processing of +these tokens. This has the following problems: + +#. It adds a considerable maintenance cost to the CPython parser. This is because + the parsing code needs to be written by hand, which has historically led to a + considerable number of inconsistencies and bugs. Writing and maintaining parsing + code by hand in C has always been considered error prone and dangerous as it needs + to deal with a lot of manual memory management over the original lexer buffers. + +#. The f-string parsing code is not able to use the new improved error message mechanisms + that the new PEG parser, originally introduced in :pep:`617`, has allowed. The + improvements that these error messages brought has been greatly celebrated but + unfortunately f-strings cannot benefit from them because they are parsed in a + separate piece of the parsing machinery. This is especially unfortunate, since + there are several syntactical features of f-strings that can be confusing due + to the different implicit tokenization that happens inside the expression + part (for instance ``f"{y:=3}"`` is not an assignment expression). + +#. Other Python implementations have no way to know if they have implemented + f-strings correctly because contrary to other language features, they are not + part of the :ref:`official Python grammar `. + This is important because several prominent + alternative implementations are using CPython's PEG parser, `such as PyPy`_, + and/or are basing their grammars on the official PEG grammar. The + fact that f-strings use a separate parser prevents these alternative implementations + from leveraging the official grammar and benefiting from improvements in error messages derived + from the grammar. + + +A version of this proposal was originally `discussed on Python-Dev`_ and +`presented at the Python Language Summit 2022`_ where it was enthusiastically +received. + +Rationale +========= + +By building on top of the new Python PEG Parser (:pep:`617`), this PEP proposes +to redefine “f-strings”, especially emphasizing the clear separation of the +string component and the expression (or replacement, ``{...}``) component. :pep:`498` +summarizes the syntactical part of “f-strings” as the following: + + In Python source code, an f-string is a literal string, prefixed with ‘f’, which + contains expressions inside braces. The expressions are replaced with their values. + +However, :pep:`498` also contained a formal list of exclusions on what +can or cannot be contained inside the expression component (primarily due to the +limitations of the existing parser). By clearly establishing the formal grammar, we +now also have the ability to define the expression component of an f-string as truly "any +applicable Python expression" (in that particular context) without being bound +by the limitations imposed by the details of our implementation. + +The formalization effort and the premise above also has a significant benefit for +Python programmers due to its ability to simplify and eliminate the obscure +limitations. This reduces the mental burden and the cognitive complexity of +f-string literals (as well as the Python language in general). + +#. The expression component can include any string literal that a normal Python expression + can include. This opens up the possibility of nesting string literals (formatted or + not) inside the expression component of an f-string with the same quote type (and length):: + + >>> f"These are the things: {", ".join(things)}" + + >>> f"{source.removesuffix(".py")}.c: $(srcdir)/{source}" + + >>> f"{f"{f"infinite"}"}" + " " + f"{f"nesting!!!"}" + + This "feature" is not universally agreed to be desirable, and some users find this unreadable. + For a discussion on the different views on this, see the `considerations regarding quote reuse`_ section. + +#. Another issue that has felt unintuitive to most is the lack of support for backslashes + within the expression component of an f-string. One example that keeps coming up is including + a newline character in the expression part for joining containers. For example:: + + >>> a = ["hello", "world"] + >>> f"{'\n'.join(a)}" + File "", line 1 + f"{'\n'.join(a)}" + ^ + SyntaxError: f-string expression part cannot include a backslash + + A common work-around for this was to either assign the newline to an intermediate variable or + pre-create the whole string prior to creating the f-string:: + + >>> a = ["hello", "world"] + >>> joined = '\n'.join(a) + >>> f"{joined}" + 'hello\nworld' + + It only feels natural to allow backslashes in the expression part now that the new PEG parser + can easily support it. + + >>> a = ["hello", "world"] + >>> f"{'\n'.join(a)}" + 'hello\nworld' + +#. Before the changes proposed in this document, there was no explicit limit in + how f-strings can be nested, but the fact that string quotes cannot be reused + inside the expression component of f-strings made it impossible to nest + f-strings arbitrarily. In fact, this is the most nested-fstring that can be + written:: + + >>> f"""{f'''{f'{f"{1+1}"}'}'''}""" + '2' + + As this PEP allows placing **any** valid Python expression inside the + expression component of the f-strings, it is now possible to reuse quotes and + therefore is possible to nest f-strings arbitrarily:: + + >>> f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" + '2' + + Although this is just a consequence of allowing arbitrary expressions, the + authors of this PEP do not believe that this is a fundamental benefit and we + have decided that the language specification will not explicitly mandate that + this nesting can be arbitrary. This is because allowing arbitrarily-deep + nesting imposes a lot of extra complexity to the lexer implementation + (particularly as lexer/parser pipelines need to allow "untokenizing" to + support the 'f-string debugging expressions' and this is especially taxing when + arbitrary nesting is allowed). Implementations are therefore free to impose a + limit on the nesting depth if they need to. Note that this is not an uncommon + situation, as the CPython implementation already imposes several limits all + over the place, including a limit on the nesting depth of parentheses and + brackets, a limit on the nesting of the blocks, a limit in the number of + branches in ``if`` statements, a limit on the number of expressions in + star-unpacking, etc. + +Specification +============= + +The formal proposed PEG grammar specification for f-strings is (see :pep:`617` +for details on the syntax): + +.. code-block:: peg + + fstring + | FSTRING_START fstring_middle* FSTRING_END + fstring_middle + | fstring_replacement_field + | FSTRING_MIDDLE + fstring_replacement_field + | '{' (yield_expr | star_expressions) "="? [ "!" NAME ] [ ':' fstring_format_spec* ] '}' + fstring_format_spec: + | FSTRING_MIDDLE + | fstring_replacement_field + +The new tokens (``FSTRING_START``, ``FSTRING_MIDDLE``, ``FSTRING_END``) are defined +`later in this document `_. + +This PEP leaves up to the implementation the level of f-string nesting allowed +(f-strings within the expression parts of other f-strings) but **specifies a +lower bound of 5 levels of nesting**. This is to ensure that users can have a +reasonable expectation of being able to nest f-strings with "reasonable" depth. +This PEP implies that limiting nesting is **not part of the language +specification** but also the language specification **doesn't mandate arbitrary +nesting**. + +Similarly, this PEP leaves up to the implementation the level of expression nesting +in format specifiers but **specifies a lower bound of 2 levels of nesting**. This means +that the following should always be valid: + +.. code-block:: python + + f"{'':*^{1:{1}}}" + +but the following can be valid or not depending on the implementation: + +.. code-block:: python + + f"{'':*^{1:{1:{1}}}}" + +The new grammar will preserve the Abstract Syntax Tree (AST) of the current +implementation. This means that no semantic changes will be introduced by this +PEP on existing code that uses f-strings. + +Handling of f-string debug expressions +-------------------------------------- + +Since Python 3.8, f-strings can be used to debug expressions by using the +``=`` operator. For example:: + + >>> a = 1 + >>> f"{1+1=}" + '1+1=2' + +This semantics were not introduced formally in a PEP and they were implemented +in the current string parser as a special case in `bpo-36817 +`_ and documented in +`the f-string lexical analysis section +`_. + +This feature is not affected by the changes proposed in this PEP but is +important to specify that the formal handling of this feature requires the lexer +to be able to "untokenize" the expression part of the f-string. This is not a +problem for the current string parser as it can operate directly on the string +token contents. However, incorporating this feature into a given parser +implementation requires the lexer to keep track of the raw string contents of +the expression part of the f-string and make them available to the parser when +the parse tree is constructed for f-string nodes. A pure "untokenization" is not +enough because as specified currently, f-string debug expressions preserve whitespace in the expression, +including spaces after the ``{`` and the ``=`` characters. This means that the +raw string contents of the expression part of the f-string must be kept intact +and not just the associated tokens. + +How parser/lexer implementations deal with this problem is of course up to the +implementation. + +New tokens +---------- + +Three new tokens are introduced: ``FSTRING_START``, ``FSTRING_MIDDLE`` and +``FSTRING_END``. Different lexers may have different implementations that may be +more efficient than the ones proposed here given the context of the particular +implementation. However, the following definitions will be used as part of the +public APIs of CPython (such as the ``tokenize`` module) and are also provided +as a reference so that the reader can have a better understanding of the +proposed grammar changes and how the tokens are used: + +* ``FSTRING_START``: This token includes the f-string prefix (``f``/``F``/``fr``) and the opening quote(s). +* ``FSTRING_MIDDLE``: This token includes a portion of text inside the string that's not part of the + expression part and isn't an opening or closing brace. This can include the text between the opening quote + and the first expression brace (``{``), the text between two expression braces (``}`` and ``{``) and the text + between the last expression brace (``}``) and the closing quote. +* ``FSTRING_END``: This token includes the closing quote. + +These tokens are always string parts and they are semantically equivalent to the +``STRING`` token with the restrictions specified. These tokens must be produced by the lexer +when lexing f-strings. This means that **the tokenizer cannot produce a single token for f-strings anymore**. +How the lexer emits this token is **not specified** as this will heavily depend on every +implementation (even the Python version of the lexer in the standard library is implemented +differently to the one used by the PEG parser). + +As an example:: + + f'some words {a+b:.3f} more words {c+d=} final words' + +will be tokenized as:: + + FSTRING_START - "f'" + FSTRING_MIDDLE - 'some words ' + LBRACE - '{' + NAME - 'a' + PLUS - '+' + NAME - 'b' + OP - ':' + FSTRING_MIDDLE - '.3f' + RBRACE - '}' + FSTRING_MIDDLE - ' more words ' + LBRACE - '{' + NAME - 'c' + PLUS - '+' + NAME - 'd' + OP - '=' + RBRACE - '}' + FSTRING_MIDDLE - ' final words' + FSTRING_END - "'" + +while ``f"""some words"""`` will be tokenized simply as:: + + FSTRING_START - 'f"""' + FSTRING_MIDDLE - 'some words' + FSTRING_END - '"""' + +Changes to the tokenize module +------------------------------ + +The :mod:`tokenize` module will be adapted to emit these tokens as described in the previous section +when parsing f-strings so tools can take advantage of this new tokenization schema and avoid having +to implement their own f-string tokenizer and parser. + +How to produce these new tokens +------------------------------- + +One way existing lexers can be adapted to emit these tokens is to incorporate a +stack of "lexer modes" or to use a stack of different lexers. This is because +the lexer needs to switch from "regular Python lexing" to "f-string lexing" when +it encounters an f-string start token and as f-strings can be nested, the +context needs to be preserved until the f-string closes. Also, the "lexer mode" +inside an f-string expression part needs to behave as a "super-set" of the +regular Python lexer (as it needs to be able to switch back to f-string lexing +when it encounters the ``}`` terminator for the expression part as well as +handling f-string formatting and debug expressions). For reference, here is a +draft of the algorithm to modify a CPython-like tokenizer to emit these new +tokens: + +1. If the lexer detects that an f-string is starting (by detecting the letter + 'f/F' and one of the possible quotes) keep advancing until a valid quote is + detected (one of ``"``, ``"""``, ``'`` or ``'''``) and emit a + ``FSTRING_START`` token with the contents captured (the 'f/F' and the + starting quote). Push a new tokenizer mode to the tokenizer mode stack for + "F-string tokenization". Go to step 2. +2. Keep consuming tokens until a one of the following is encountered: + + * A closing quote equal to the opening quote. + * If in "format specifier mode" (see step 3), an opening brace (``{``) or a + closing brace (``}``). + * If not in "format specifier mode" (see step 3), an opening brace (``{``) or + a closing brace (``}``) that is not immediately followed by another opening/closing + brace. + + In all cases, if the character buffer is not empty, emit a ``FSTRING_MIDDLE`` + token with the contents captured so far but transform any double + opening/closing braces into single opening/closing braces. Now, proceed as + follows depending on the character encountered: + + * If a closing quote matching the opening quite is encountered go to step 4. + * If an opening bracket (not immediately followed by another opening bracket) + is encountered, go to step 3. + * If a closing bracket (not immediately followed by another closing bracket) + is encountered, emit a token for the closing bracket and go to step 2. +3. Push a new tokenizer mode to the tokenizer mode stack for "Regular Python + tokenization within f-string" and proceed to tokenize with it. This mode + tokenizes as the "Regular Python tokenization" until a ``:`` or a ``}`` + character is encountered with the same level of nesting as the opening + bracket token that was pushed when we enter the f-string part. Using this mode, + emit tokens until one of the stop points are reached. When this happens, emit + the corresponding token for the stopping character encountered and, pop the + current tokenizer mode from the tokenizer mode stack and go to step 2. If the + stopping point is a ``:`` character, enter step 2 in "format specifier" mode. +4. Emit a ``FSTRING_END`` token with the contents captured and pop the current + tokenizer mode (corresponding to "F-string tokenization") and go back to + "Regular Python mode". + +Of course, as mentioned before, it is not possible to provide a precise +specification of how this should be done for an arbitrary tokenizer as it will +depend on the specific implementation and nature of the lexer to be changed. + +Consequences of the new grammar +------------------------------- + +All restrictions mentioned in the PEP are lifted from f-string literals, as explained below: + +* Expression portions may now contain strings delimited with the same kind of + quote that is used to delimit the f-string literal. +* Backslashes may now appear within expressions just like anywhere else in + Python code. In case of strings nested within f-string literals, escape sequences are + expanded when the innermost string is evaluated. +* New lines are now allowed within expression brackets. This means that these are now allowed:: + + >>> x = 1 + >>> f"___{ + ... x + ... }___" + '___1___' + + >>> f"___{( + ... x + ... )}___" + '___1___' + +* Comments, using the ``#`` character, are allowed within the expression part of an f-string. + Note that comments require that the closing bracket (``}``) of the expression part to be present in + a different line as the one the comment is in or otherwise it will be ignored as part of the comment. + +.. _701-considerations-of-quote-reuse: + +Considerations regarding quote reuse +------------------------------------ + +One of the consequences of the grammar proposed here is that, as mentioned above, +f-string expressions can now contain strings delimited with the same kind of quote +that is used to delimit the external f-string literal. For example: + + >>> f" something { my_dict["key"] } something else " + +In the `discussion thread for this PEP `_, +several concerns have been raised regarding this aspect and we want to collect them here, +as these should be taken into consideration when accepting or rejecting this PEP. + +Some of these objections include: + +* Many people find quote reuse within the same string confusing and hard to read. This is because + allowing quote reuse will violate a current property of Python as it stands today: the fact that + strings are fully delimited by two consecutive pairs of the same kind of quote, which by itself is a very simple rule. + One of the reasons quote reuse may be harder for humans to parse, leading to less readable + code, is that the quote character is the same for both start and + end (as opposed to other delimiters). + +* Some users have raised concerns that quote reuse may break some lexer and syntax highlighting tools that rely + on simple mechanisms to detect strings and f-strings, such as regular expressions or simple delimiter + matching tools. Introducing quote reuse in f-strings will either make it trickier to keep these tools + working or will break the tools altogether (as, for instance, regular expressions cannot parse arbitrary nested + structures with delimiters). The IDLE editor, included in the standard library, is an example of a + tool which may need some work to correctly apply syntax highlighting to f-strings. + +Here are some of the arguments in favour: + +* Many languages that allow similar syntactic constructs (normally called "string interpolation") allow quote + reuse and arbitrary nesting. These languages include JavaScript, Ruby, C#, Bash, Swift and many others. + The fact that many languages allow quote reuse can be a compelling argument in favour of allowing it in Python. This + is because it will make the language more familiar to users coming from other languages. + +* As many other popular languages allow quote reuse in string interpolation constructs, this means that editors + that support syntax highlighting for these languages will already have the necessary tools to support syntax + highlighting for f-strings with quote reuse in Python. This means that although the files that handle syntax + highlighting for Python will need to be updated to support this new feature, is not expected to be impossible + or very hard to do. + +* One advantage of allowing quote reuse is that it composes cleanly with other syntax. Sometimes this is referred to + as "referential transparency". An example of this is that if we have ``f(x+1)``, assuming ``a`` is a brand new variable, it + should behave the same as ``a = x+1; f(a)``. And vice versa. So if we have:: + + def py2c(source): + prefix = source.removesuffix(".py") + return f"{prefix}.c" + + It should be expected that if we replace the variable ``prefix`` with its definition, the answer should be the same:: + + def py2c(source): + return f"{source.removesuffix(".py")}.c" + +* Code generators (like `ast.unparse `_ from standard library) in their + current form rely on complicated algorithms to ensure expressions within an f-string are properly suited for the context in + which they are being used. These non-trivial algorithms come with challenges such as finding an unused quote type (by tracking + the outer quotes), and generating string representations which would not include backslashes if possible. Allowing quote reuse + and backslashes would simplify the code generators which deal with f-strings considerably, as the regular Python expression logic + can be used inside and outside of f-strings without any special treatment. + +* Limiting quote reuse will considerably increase the complexity of the implementation of the proposed changes. This is because + it will force the parser to have the context that is parsing an expression part of an f-string with a given quote in order + to know if it needs to reject an expression that reuses the quote. Carrying this context around is not trivial in parsers that + can backtrack arbitrarily (such as the PEG parser). The issue becomes even more complex if we consider that f-strings can be + arbitrarily nested and therefore several quote types may need to be rejected. + + To gather feedback from the community, + `a poll `__ + has been initiated to get a sense of how the community feels about this aspect of the PEP. + +Backwards Compatibility +======================= + +This PEP does not introduce any backwards incompatible syntactic or semantic changes +to the Python language. However, the :mod:`tokenize` module (a quasi-public part of the standard +library) will need to be updated to support the new f-string tokens (to allow tool authors +to correctly tokenize f-strings). See `changes to the tokenize module`_ for more details regarding +how the public API of ``tokenize`` will be affected. + +How to Teach This +================= + +As the concept of f-strings is already ubiquitous in the Python community, there is +no fundamental need for users to learn anything new. However, as the formalized grammar +allows some new possibilities, it is important that the formal grammar is added to the +documentation and explained in detail, explicitly mentioning what constructs are possible +since this PEP is aiming to avoid confusion. + +It is also beneficial to provide users with a simple framework for understanding what can +be placed inside an f-string expression. In this case the authors think that this work will +make it even simpler to explain this aspect of the language, since it can be summarized as: + + You can place any valid Python expression inside an f-string expression. + +With the changes in this PEP, there is no need to clarify that string quotes are +limited to be different from the quotes of the enclosing string, because this is +now allowed: as an arbitrary Python string can contain any possible choice of +quotes, so can any f-string expression. Additionally there is no need to clarify +that certain things are not allowed in the expression part because of +implementation restrictions such as comments, new line characters or +backslashes. + +The only "surprising" difference is that as f-strings allow specifying a +format, expressions that allow a ``:`` character at the top level still need to be +enclosed in parenthesis. This is not new to this work, but it is important to +emphasize that this restriction is still in place. This allows for an easier +modification of the summary: + + You can place any valid Python expression inside + an f-string expression, and everything after a ``:`` character at the top level will + be identified as a format specification. + + +Reference Implementation +======================== + +A reference implementation can be found in the implementation_ fork. + +Rejected Ideas +============== + +#. Although we think the readability arguments that have been raised against + allowing quote reuse in f-string expressions are valid and very important, + we have decided to propose not rejecting quote reuse in f-strings at the parser + level. The reason is that one of the cornerstones of this PEP is to reduce the + complexity and maintenance of parsing f-strings in CPython and this will not + only work against that goal, but it may even make the implementation even more + complex than the current one. We believe that forbidding quote reuse should be + done in linters and code style tools and not in the parser, the same way other + confusing or hard-to-read constructs in the language are handled today. + +#. We have decided not to lift the restriction that some expression portions + need to wrap ``':'`` and ``'!'`` in parentheses at the top level, e.g.:: + + >>> f'Useless use of lambdas: { lambda x: x*2 }' + SyntaxError: unexpected EOF while parsing + + The reason is that this will introduce a considerable amount of + complexity for no real benefit. This is due to the fact that the ``:`` character + normally separates the f-string format specification. This format specification + is currently tokenized as a string. As the tokenizer MUST tokenize what's on the + right of the ``:`` as either a string or a stream of tokens, this won't allow the + parser to differentiate between the different semantics as that would require the + tokenizer to backtrack and produce a different set of tokens (this is, first try + as a stream of tokens, and if it fails, try as a string for a format specifier). + + As there is no fundamental advantage in being able to allow lambdas and similar + expressions at the top level, we have decided to keep the restriction that these must + be parenthesized if needed:: + + >>> f'Useless use of lambdas: { (lambda x: x*2) }' + +#. We have decided to disallow (for the time being) using escaped braces (``\{`` and ``\}``) + in addition to the ``{{`` and ``}}`` syntax. Although the authors of the PEP believe that + allowing escaped braces is a good idea, we have decided to not include it in this PEP, as it is not strictly + necessary for the formalization of f-strings proposed here, and it can be + added independently in a regular CPython issue. + +Open Issues +=========== + +None yet + + +Footnotes +========= + + +.. _official Python grammar: https://docs.python.org/3/reference/lexical_analysis.html#formatted-string-literals + +.. _none of this work was ever implemented: https://mail.python.org/archives/list/python-dev@python.org/thread/N43O4KNLZW4U7YZC4NVPCETZIVRDUVU2/#NM2A37THVIXXEYR4J5ZPTNLXGGUNFRLZ + +.. _such as PyPy: https://foss.heptapod.net/pypy/pypy/-/commit/fe120f89bf07e64a41de62b224e4a3d80e0fe0d4/pipelines?ref=branch%2Fpy3.9 + +.. _discussed on Python-Dev: https://mail.python.org/archives/list/python-dev@python.org/thread/54N3MOYVBDSJQZTU6MTCPLUPIFSDN5IS/#SAYU6SMP4KT7G7AQ6WVQYUDOSZPKHJMS + +.. _presented at the Python Language Summit 2022: https://pyfound.blogspot.com/2022/05/the-2022-python-language-summit-f.html + +.. _implementation: https://github.com/we-like-parsers/cpython/tree/fstring-grammar + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0702.rst b/peps/pep-0702.rst new file mode 100644 index 000000000..d81d42ad2 --- /dev/null +++ b/peps/pep-0702.rst @@ -0,0 +1,360 @@ +PEP: 702 +Title: Marking deprecations using the type system +Author: Jelle Zijlstra +Discussions-To: https://discuss.python.org/t/pep-702-marking-deprecations-using-the-type-system/23036 +Status: Draft +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 30-Dec-2022 +Python-Version: 3.13 +Post-History: `01-Jan-2023 `__, + `22-Jan-2023 `__ + + +Abstract +======== + +This PEP adds an ``@warnings.deprecated()`` decorator that marks a class or function +as deprecated, enabling static checkers to warn when it is used. By default, this +decorator will also raise a runtime ``DeprecationWarning``. + +Motivation +========== + +As software evolves, new functionality is added and old functionality becomes +obsolete. Library developers want to work towards removing obsolete code while +giving their users time to migrate to new APIs. Python provides a mechanism for +achieving these goals: the :exc:`DeprecationWarning` warning class, which is +used to show warnings when deprecated functionality is used. This mechanism is +widely used: as of the writing of this PEP, the CPython main branch contains +about 150 distinct code paths that raise :exc:`!DeprecationWarning`. Many +third-party libraries also use :exc:`!DeprecationWarning` to mark deprecations. +In the `top 5000 PyPI packages `__, +there are: + +- 1911 matches for the regex ``warnings\.warn.*\bDeprecationWarning\b``, + indicating use of :exc:`!DeprecationWarning` (not including cases where the + warning is split over multiple lines); +- 1661 matches for the regex ``^\s*@deprecated``, indicating use of some sort of + deprecation decorator. + +However, the current mechanism is often insufficient to ensure that users +of deprecated functionality update their code in time. For example, the +removal of various long-deprecated :mod:`unittest` features had to be +`reverted `__ +from Python 3.11 to give users more time to update their code. +Users may run their test suite with warnings disabled for practical reasons, +or deprecations may be triggered in code paths that are not covered by tests. + +Providing more ways for users to find out about deprecated functionality +can speed up the migration process. This PEP proposes to leverage static type +checkers to communicate deprecations to users. Such checkers have a thorough +semantic understanding of user code, enabling them to detect and report +deprecations that a single ``grep`` invocation could not find. In addition, many type +checkers integrate with IDEs, enabling users to see deprecation warnings +right in their editors. + +Rationale +========= + +At first glance, deprecations may not seem like a topic that type checkers should +touch. After all, type checkers are concerned with checking whether code will +work as is, not with potential future changes. However, the analysis that type +checkers perform on code to find type errors is very similar to the analysis +that would be needed to detect usage of many deprecations. Therefore, type +checkers are well placed to find and report deprecations. + +Other languages already have similar functionality: + +* GCC supports a ``deprecated`` `attribute `__ + on function declarations. This powers CPython's ``Py_DEPRECATED`` macro. +* GraphQL `supports `__ + marking fields as ``@deprecated``. +* Kotlin `supports `__ + a ``Deprecated`` annotation. +* Scala `supports `__ + an ``@deprecated`` annotation. +* Swift `supports `__ + using the ``@available`` attribute to mark APIs as deprecated. +* TypeScript `uses `__ + the ``@deprecated`` JSDoc tag to issue a hint marking use of + deprecated functionality. + +Several users have requested support for such a feature: + +* `typing-sig thread `__ +* `Pyright feature request `__ +* `mypy feature request `__ + +There are similar existing third-party tools: + +* `Deprecated `__ provides a decorator to + mark classes, functions, or methods as deprecated. Access to decorated objects + raises a runtime warning, but is not detected by type checkers. +* `flake8-deprecated `__ is a linter + plugin that warns about use of deprecated features. However, it is limited to + a short, hard-coded list of deprecations. + +Specification +============= + +A new decorator ``@deprecated()`` is added to the :mod:`warnings` module. This +decorator can be used on a class, function or method to mark it as deprecated. +This includes :class:`typing.TypedDict` and :class:`typing.NamedTuple` definitions. +With overloaded functions, the decorator may be applied to individual overloads, +indicating that the particular overload is deprecated. The decorator may also be +applied to the overload implementation function, indicating that the entire function +is deprecated. + +The decorator takes the following arguments: + +* A required positional-only argument representing the deprecation message. +* Two keyword-only arguments, ``category`` and ``stacklevel``, controlling + runtime behavior (see under "Runtime behavior" below). + +The positional-only argument is of type ``str`` and contains a message that should +be shown by the type checker when it encounters a usage of the decorated object. +The message must be a string literal. +The content of deprecation messages is up to the user, but it may include the version +in which the deprecated object is to be removed, and information about suggested +replacement APIs. + +Type checkers should produce a diagnostic whenever they encounter a usage of an +object marked as deprecated. For deprecated overloads, this includes all calls +that resolve to the deprecated overload. +For deprecated classes and functions, this includes: + +* References through module, class, or instance attributes (``module.deprecated_object``, + ``module.SomeClass.deprecated_method``, ``module.SomeClass().deprecated_method``) +* Any usage of deprecated objects in their defining module + (``x = deprecated_object()`` in ``module.py``) +* If ``import *`` is used, usage of deprecated objects from the + module (``from module import *; x = deprecated_object()``) +* ``from`` imports (``from module import deprecated_object``) +* Any syntax that indirectly triggers a call to the function. For example, + if the ``__add__`` method of a class ``C`` is deprecated, then + the code ``C() + C()`` should trigger a diagnostic. Similarly, if the + setter of a property is marked deprecated, attempts to set the property + should trigger a diagnostic. + +If a method is marked with the :func:`typing.override` decorator from :pep:`698` +and the base class method it overrides is deprecated, the type checker should +produce a diagnostic. + +There are additional scenarios where deprecations could come into play. +For example, an object may implement a :class:`typing.Protocol`, but one +of the methods required for protocol compliance is deprecated. +As scenarios such as this one appear complex and relatively unlikely to come up in practice, +this PEP does not mandate that type checkers detect them. + +Example +------- + +As an example, consider this library stub named ``library.pyi``: + +.. code-block:: python + + from warnings import deprecated + + @deprecated("Use Spam instead") + class Ham: ... + + @deprecated("It is pining for the fiords") + def norwegian_blue(x: int) -> int: ... + + @overload + @deprecated("Only str will be allowed") + def foo(x: int) -> str: ... + @overload + def foo(x: str) -> str: ... + + class Spam: + @deprecated("There is enough spam in the world") + def __add__(self, other: object) -> object: ... + + @property + @deprecated("All spam will be equally greasy") + def greasy(self) -> float: ... + + @property + def shape(self) -> str: ... + @shape.setter + @deprecated("Shapes are becoming immutable") + def shape(self, value: str) -> None: ... + +Here is how type checkers should handle usage of this library: + +.. code-block:: python + + from library import Ham # error: Use of deprecated class Ham. Use Spam instead. + + import library + + library.norwegian_blue(1) # error: Use of deprecated function norwegian_blue. It is pining for the fiords. + map(library.norwegian_blue, [1, 2, 3]) # error: Use of deprecated function norwegian_blue. It is pining for the fiords. + + library.foo(1) # error: Use of deprecated overload for foo. Only str will be allowed. + library.foo("x") # no error + + ham = Ham() # no error (already reported above) + + spam = library.Spam() + spam + 1 # error: Use of deprecated method Spam.__add__. There is enough spam in the world. + spam.greasy # error: Use of deprecated property Spam.greasy. All spam will be equally greasy. + spam.shape # no error + spam.shape = "cube" # error: Use of deprecated property setter Spam.shape. Shapes are becoming immutable. + +The exact wording of the diagnostics is up to the type checker and is not part +of the specification. + +Runtime behavior +---------------- + +In addition to the positional-only ``message`` argument, +the ``@deprecated`` decorator takes two keyword-only arguments: + +* ``category``: A warning class. Defaults to :exc:`DeprecationWarning`. If this + is set to ``None``, no warning is issued at runtime and the decorator returns + the original object, except for setting the ``__deprecated__`` attribute (see below). +* ``stacklevel``: The number of stack frames to skip when issuing the warning. + Defaults to 1, indicating that the warning should be issued at the site where the + deprecated object is called. Internally, the implementation will add the number of + stack frames it uses in wrapper code. + +If the decorated object is a class, the decorator wraps the ``__new__`` method +such that instantiating the class issues a warning. If the decorated object is a +callable, the decorator returns a new callable that wraps the original callable but +raises a warning when called. Otherwise, the decorator raises a ``TypeError`` +(unless ``category=None`` is passed). + +There are several scenarios where use of the decorated object cannot issue a warning, +including overloads, ``Protocol`` classes, and abstract methods. Type checkers may show a +warning if ``@deprecated`` is used without ``category=None`` in these cases. + +To accommodate runtime introspection, the decorator sets an attribute +``__deprecated__`` on the object it is passed, as well as on the wrapper +callables it generates for deprecated classes and functions. +The value of the attribute is the message passed to the decorator. +Decorating objecs that do not allow setting this attribute is not supported. + +If a ``Protocol`` with the ``@runtime_checkable`` decorator is marked as deprecated, +the ``__deprecated__`` attribute should not be considered a member of the protocol, +so its presence should not affect ``isinstance`` checks. + +For compatibility with :func:`typing.get_overloads`, the ``@deprecated`` +decorator should be placed after the ``@overload`` decorator. + +Type checker behavior +--------------------- + +This PEP does not specify exactly how type checkers should present deprecation +diagnostics to their users. However, some users (e.g., application developers +targeting only a specific version of Python) may not care about deprecations, +while others (e.g., library developers who want their library to remain +compatible with future versions of Python) would want to catch any use of +deprecated functionality in their CI pipeline. Therefore, it is recommended +that type checkers provide configuration options that cover both use cases. +As with any other type checker error, it is also possible to ignore deprecations +using ``# type: ignore`` comments. + +Deprecation policy +------------------ + +We propose that CPython's deprecation policy (:pep:`387`) is updated to require that new deprecations +use the functionality in this PEP to alert users +about the deprecation, if possible. Concretely, this means that new +deprecations should be accompanied by a change to the ``typeshed`` repo to +add the ``@deprecated`` decorator in the appropriate place. +This requirement does not apply to deprecations that cannot be expressed +using this PEP's functionality. + +Backwards compatibility +======================= + +Creating a new decorator poses no backwards compatibility concerns. +As with all new typing functionality, the ``@deprecated`` decorator +will be added to the ``typing_extensions`` module, enabling its use +in older versions of Python. + +How to teach this +================= + +For users who encounter deprecation warnings in their IDE or type +checker output, the messages they receive should be clear and self-explanatory. +Usage of the ``@deprecated`` decorator will be an advanced feature +mostly relevant to library authors. The decorator should be mentioned +in relevant documentation (e.g., :pep:`387` and the :exc:`DeprecationWarning` +documentation) as an additional way to mark deprecated functionality. + +Reference implementation +======================== + +A runtime implementation of the ``@deprecated`` decorator is +available in the `typing-extensions `_ +library since version 4.5.0. +The ``pyanalyze`` type checker has +`prototype support `__ +for emitting deprecation errors, as does +`Pyright `__. + +Rejected ideas +============== + +Deprecation of modules and attributes +------------------------------------- + +This PEP covers deprecations of classes, functions and overloads. This +allows type checkers to detect many but not all possible deprecations. +To evaluate whether additional functionality would be worthwhile, I +`examined `__ +all current deprecations in the CPython standard library. + +I found: + +* 74 deprecations of functions, methods and classes (supported by this PEP) +* 28 deprecations of whole modules (largely due to :pep:`594`) +* 9 deprecations of function parameters (supported by this PEP through + decorating overloads) +* 1 deprecation of a constant +* 38 deprecations that are not easily detectable in the type system (for + example, for calling :func:`asyncio.get_event_loop` without an active + event loop) + +Modules could be marked as deprecated by adding a ``__deprecated__`` +module-level constant. However, the need for this is limited, and it +is relatively easy to detect usage of deprecated modules simply by +grepping. Therefore, this PEP omits support for whole-module deprecations. +As a workaround, users could mark all module-level classes and functions +with ``@deprecated``. + +For deprecating module-level constants, object attributes, and function +parameters, a ``Deprecated[type, message]`` type modifier, similar to +``Annotated`` could be added. However, this would create a new place +in the type system where strings are just strings, not forward references, +complicating the implementation of type checkers. In addition, my data +show that this feature is not commonly needed. + +Features for deprecating more kinds of objects could be added in a future +PEP. + +Placing the decorator in the ``typing`` module +---------------------------------------------- + +An earlier version of this PEP proposed placing the ``@deprecated`` +decorator in the :mod:`typing` module. However, there was feedback +that it would be unexpected for a decorator in the :mod:`typing` module +to have runtime behavior. Therefore, the PEP now proposes adding the +decorator the :mod:`warnings` module instead. + +Acknowledgments +=============== + +A call with the typing-sig meetup group led to useful feedback on this +proposal. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0703.rst b/peps/pep-0703.rst new file mode 100644 index 000000000..61fd12d66 --- /dev/null +++ b/peps/pep-0703.rst @@ -0,0 +1,1915 @@ +PEP: 703 +Title: Making the Global Interpreter Lock Optional in CPython +Author: Sam Gross +Sponsor: Ɓukasz Langa +Discussions-To: https://discuss.python.org/t/22606 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 09-Jan-2023 +Python-Version: 3.13 +Post-History: `09-Jan-2023 `__, + `04-May-2023 `__ + + +Abstract +======== + +CPython's global interpreter lock ("GIL") prevents multiple threads +from executing Python code at the same time. The GIL is an obstacle +to using multi-core CPUs from Python efficiently. This PEP proposes +adding a build configuration (``--disable-gil``) to CPython to let it +run Python code without the global interpreter lock and with the +necessary changes needed to make the interpreter thread-safe. + +Motivation +========== + +The GIL is a major obstacle to concurrency. For scientific computing +tasks, this lack of concurrency is often a bigger issue than speed of +executing Python code, since most of the processor cycles are spent +in optimized CPU or GPU kernels. The GIL introduces a global +bottleneck that can prevent other threads from making progress if +they call any Python code. There are existing ways to enable +parallelism in CPython today, but those techniques come with +significant limitations (see `Alternatives`_). + +This section focuses on the GIL's impact on scientific computing, +particular AI/ML workloads because that is the area with which this +author has the most experience, but the GIL also affects other users +of Python. + + +The GIL Makes Many Types of Parallelism Difficult to Express +------------------------------------------------------------ + +Neural network-based AI models expose multiple opportunities for +parallelism. For example, individual operations may be parallelized +internally ("intra-operator"), multiple operations may be executed +simultaneously ("inter-operator"), and requests (spanning multiple +operations) may also be parallelized. Efficient execution requires +exploiting multiple types of parallelism [#yuemmwang2019]_. + +The GIL makes it difficult to express inter-operator parallelism, as +well as some forms of request parallelism, efficiently in Python. In +other programming languages, a system might use threads to run +different parts of a neural network on separate CPU cores, but this is +inefficient in Python due to the GIL. Similarly, latency-sensitive +inference workloads frequently use threads to parallelize across +requests, but face the same scaling bottlenecks in Python. + +The challenges the GIL poses to exploiting parallelism in Python +frequently come up in reinforcement learning. Heinrich Kuttler, +author of the NetHack Learning Environment and Member of Technical +Staff at Inflection AI, writes: + + Recent breakthroughs in reinforcement learning, such as on `Dota + 2`_, `StarCraft`_, and `NetHack`_ rely on running multiple + environments (simulated games) in parallel using asynchronous + actor-critic methods. Straightforward multithreaded implementations + in Python don't scale beyond more than a few parallel environments + due to GIL contention. Multiprocessing, with communication via + shared memory or UNIX sockets, adds much complexity and in effect + rules out interacting with CUDA from different workers, severely + restricting the design space. + +.. _Dota 2: https://openai.com/five/ +.. _StarCraft: https://www.deepmind.com/blog/alphastar-grandmaster-level-in-starcraft-ii-using-multi-agent-reinforcement-learning +.. _NetHack: https://ai.facebook.com/blog/nethack-learning-environment-to-advance-deep-reinforcement-learning/ + +Manuel Kroiss, software engineer at DeepMind on the reinforcement +learning team, describes how the bottlenecks posed by the GIL lead to +rewriting Python codebases in C++, making the code less accessible: + + We frequently battle issues with the Python GIL at DeepMind. In many + of our applications, we would like to run on the order of 50-100 + threads per process. However, we often see that even with fewer + than 10 threads the GIL becomes the bottleneck. To work around this + problem, we sometimes use subprocesses, but in many cases the + inter-process communication becomes too big of an overhead. To + deal with the GIL, we usually end up translating large parts of our + Python codebase into C++. This is undesirable because it makes the + code less accessible to researchers. + + +Projects that involve interfacing with multiple hardware devices face +similar challenges: efficient communication requires use of multiple +CPU cores. The `Dose-3D`_ project aims to improve cancer +radiotherapy with precise dose planning. It uses medical phantoms +(stand-ins for human tissue) together with custom hardware and a +server application written in Python. PaweƂ Jurgielewicz, lead +software architect for the data acquisition system on the Dose-3D +project, describes the scaling challenges posed by the GIL and how +using a fork of Python without the GIL simplified the project: + + In the Dose-3D project, the key challenge was to maintain a stable, + non-trivial concurrent communication link with hardware units while + utilizing a 1 Gbit/s UDP/IP connection to the maximum. Naturally, + we started with the multiprocessing package, but at some point, it + became clear that most CPU time was consumed by the data transfers + between the data processing stages, not by data processing itself. + The CPython multithreading implementation based on GIL was a dead + end too. When we found out about the "nogil" fork of Python it took + a single person less than half a working day to adjust the codebase + to use this fork and the results were astonishing. Now we can focus + on data acquisition system development rather than fine-tuning data + exchange algorithms. + +.. _Dose-3D: https://dose3d.fis.agh.edu.pl/en/projekt-dose-3d-z-programu-team-net-fnp-eng/ + + +Allen Goodman, author of `CellProfiler`_ and staff engineer at +Prescient Design and Genentech, describes how the GIL makes +biological methods research more difficult in Python: + + Issues with Python's global interpreter lock are a frequent source + of frustration throughout biological methods research. + + I wanted to better understand the current multithreading situation + so I reimplemented parts of `HMMER`_, a standard method for + multiple-sequence alignment. I chose this method because it + stresses both single-thread performance (scoring) and + multi-threaded performance (searching a database of sequences). The + GIL became the bottleneck when using only eight threads. This is a + method where the current popular implementations rely on 64 or + even 128 threads per process. I tried moving to subprocesses but + was blocked by the prohibitive IPC costs. HMMER is a relatively + elementary bioinformatics method and newer methods have far bigger + multi-threading demands. + + Method researchers are begging to use Python (myself included), + because of its ease of use, the Python ecosystem, and because "it's + what people know." Many biologists only know a little bit of + programming (and that's almost always Python). Until Python's + multithreading situation is addressed, C and C++ will remain the + lingua franca of the biological methods research community. + +.. _CellProfiler: https://cellprofiler.org/ +.. _HMMER: http://hmmer.org/ + + +The GIL Affects Python Library Usability +---------------------------------------- + +The GIL is a CPython implementation detail that limits multithreaded +parallelism, so it might seem unintuitive to think of it as a +usability issue. However, library authors frequently care a great +deal about performance and will design APIs that support working +around the GIL. These workaround frequently lead to APIs that are +more difficult to use. Consequently, users of these APIs may +experience the GIL as a *usability* issue and not just a performance +issue. + +For example, PyTorch exposes a multiprocessing-based API called +``DataLoader`` for building data input pipelines. It uses ``fork()`` +on Linux because it is generally faster and uses less memory +than ``spawn()``, but this leads to additional challenges for users: +creating a ``DataLoader`` after accessing a GPU can lead to confusing +CUDA errors. Accessing GPUs within a ``DataLoader`` worker quickly +leads to out-of-memory errors because processes do not share CUDA +contexts (unlike threads within a process). + +Olivier Grisel, scikit-learn developer and software engineer at Inria, +describes how having to work around the GIL in scikit-learn related +libraries leads to a more complex and confusing user experience: + + Over the years, scikit-learn developers have maintained ancillary + libraries such as ``joblib`` and ``loky`` to try to work around some + of the limitations of multiprocessing: extra memory usage partially + mitigated via semi-automated memory mapping of large data buffers, + slow worker startup by transparently reusing a pool of long + running workers, fork-safety problems of third-party native runtime + libraries such as GNU OpenMP by never using the fork-only + start-method, ability to perform parallel calls of interactively + defined functions in notebooks and REPLs in cross-platform manner + via cloudpickle. Despite our efforts, this multiprocessing-based + solution is still brittle, complex to maintain and confusing to + datascientists with limited understanding of system-level + constraints. Furthermore, there are still irreducible limitations + such as the overhead caused by the pickle-based + serialization/deserialization steps required for inter-process + communication. A lot of this extra work and complexity would not be + needed anymore if we could use threads without contention on + multicore hosts (sometimes with 64 physical cores or more) to run + data science pipelines that alternate between Python-level + operations and calls to native libraries. + +Ralf Gommers, co-director of Quansight Labs and NumPy and SciPy +maintainer, describes how the GIL affects the user experience of +NumPy and numeric Python libraries: + + A key problem in NumPy and the stack of packages built around it is + that NumPy is still (mostly) single-threaded --- and that has shaped + significant parts of the user experience and projects built around + it. NumPy does release the GIL in its inner loops (which do the + heavy lifting), but that is not nearly enough. NumPy doesn't offer + a solution to utilize all CPU cores of a single machine well, and + instead leaves that to Dask and other multiprocessing solutions. + Those aren't very efficient and are also more clumsy to use. That + clumsiness comes mainly in the extra abstractions and layers the + users need to concern themselves with when using, e.g., + ``dask.array`` which wraps ``numpy.ndarray``. It also shows up in + oversubscription issues that the user must explicitly be aware of + and manage via either environment variables or a third package, + ``threadpoolctl``. The main reason is that NumPy calls into BLAS + for linear algebra - and those calls it has no control over, they + do use all cores by default via either pthreads or OpenMP. + + Coordinating on APIs and design decisions to control parallelism is + still a major amount of work, and one of the harder challenges + across the PyData ecosystem. It would have looked a lot different + (better, easier) without a GIL. + + +GPU-Heavy Workloads Require Multi-Core Processing +------------------------------------------------- + +Many high-performance computing (HPC) and AI workloads make heavy use +of GPUs. These applications frequently require efficient multi-core +CPU execution even though the bulk of the computation runs on a GPU. + +Zachary DeVito, PyTorch core developer and researcher at FAIR +(Meta AI), describes how the GIL makes multithreaded scaling +inefficient even when the bulk of computation is performed outside of +Python: + + In PyTorch, Python is commonly used to orchestrate ~8 GPUs and ~64 + CPU threads, growing to 4k GPUs and 32k CPU threads for big models. + While the heavy lifting is done outside of Python, the speed of + GPUs makes even just the orchestration in Python not scalable. We + often end up with 72 processes in place of one because of the GIL. + Logging, debugging, and performance tuning are orders-of-magnitude + more difficult in this regime, continuously causing lower developer + productivity. + +The use of many processes (instead of threads) makes common tasks more +difficult. Zachary DeVito continues: + + On three separate occasions in the past couple of months + (reducing redundant compute in data loaders, writing model + checkpoints asynchronously, and parallelizing compiler + optimizations), I spent an order-of-magnitude more time figuring + out how to work around GIL limitations than actually solving the + particular problem. + +Even GPU-heavy workloads frequently have a CPU-intensive component. +For example, computer vision tasks typically require +multiple "pre-processing" steps in the data input pipeline, like +image decoding, cropping, and resizing. These tasks are commonly +performed on the CPU and may use Python libraries like `Pillow`_ +or `Pillow-SIMD`_. It is necessary to run the data input pipeline +on multiple CPU cores in order to keep the GPU "fed" with data. + +The increase in GPU performance compared to individual CPU cores makes +multi-core performance more important. It is progressively more +difficult to keep the GPUs fully occupied. To do so requires efficient +use of multiple CPU cores, especially on multi-GPU systems. For +example, NVIDIA's DGX-A100 has 8 GPUs and two 64-core CPUs in order to +keep the GPUs "fed" with data. + +.. _Pillow: https://pillow.readthedocs.io/en/stable/ +.. _Pillow-SIMD: https://github.com/uploadcare/pillow-simd + + +The GIL Makes Deploying Python AI Models Difficult +-------------------------------------------------- + +Python is widely used to develop neural network-based AI models. In +PyTorch, models are frequently deployed as part of multi-threaded, +mostly C++, environments. Python is often viewed skeptically +because the GIL can be a global bottleneck, preventing efficient +scaling even though the vast majority of the computations +occur "outside" of Python with the GIL released. The torchdeploy +paper [#torchdeploy]_ shows experimental evidence for these scaling +bottlenecks in multiple model architectures. + +PyTorch provides a number of mechanisms for deploying Python AI +models that avoid or work around the GIL, but they all come with +substantial limitations. For example, `TorchScript +`_ captures a +representation of the model that can be executed from C++ without any +Python dependencies, but it only supports a limited subset of Python +and often requires rewriting some of the model's code. The +`torch::deploy `_ API +allows multiple Python interpreters, each with its own GIL, in the +same process(similar to :pep:`684`). However, ``torch::deploy`` has +limited support for Python modules that use C-API extensions. + + +Motivation Summary +------------------ + +Python's global interpreter lock makes it difficult to use modern +multi-core CPUs efficiently for many scientific and numeric computing +applications. Heinrich Kuttler, Manuel Kroiss, and PaweƂ +Jurgielewicz found that multi-threaded implementations in Python did +not scale well for their tasks and that using multiple processes +was not a suitable alternative. + +The scaling bottlenecks are not solely in core numeric tasks. Both +Zachary DeVito and PaweƂ Jurgielewicz described challenges with +coordination and communication in Python. + +Olivier Grisel, Ralf Gommers, and Zachary DeVito described how current +workarounds for the GIL are "complex to maintain" and cause "lower +developer productivity." The GIL makes it more difficult to develop +and maintain scientific and numeric computing libraries as well +leading to library designs that are more difficult to use. + + + +Specification +============= + +Build Configuration Changes +--------------------------- + +The global interpreter lock will remain the default for CPython builds +and python.org downloads. A new build configuration flag, +``--disable-gil`` will be added to the configure script that will build +CPython with support for running without the global interpreter lock. + +When built with ``--disable-gil``, CPython will define the ``Py_NOGIL`` +macro in Python/patchlevel.h. The ABI tag will include the letter "t" +(for "threading"). + +The ``--disable-gil`` builds of CPython will still support optionally +running with the GIL enabled at runtime (see `PYTHONGIL Environment +Variable`_ and `Py_mod_gil Slot`_). + +Overview of CPython Changes +--------------------------- + +Removing the global interpreter lock requires substantial changes to +CPython internals, but relatively few changes to the public Python +and C APIs. This section describes the required changes to the +CPython implementation followed by the proposed API changes. + +The implementation changes can be grouped into the following four +categories: + +* Reference counting +* Memory management +* Container thread-safety +* Locking and atomic APIs + +Reference Counting +------------------ + +Removing the GIL requires changes to CPython's +reference counting implementation to make it thread-safe. +Furthermore, it needs to have low execution overhead and allow for +efficient scaling with multiple threads. This PEP proposes a +combination of three techniques to address these constraints. The +first is a switch from plain non-atomic reference counting to biased +reference counting, which is a thread-safe reference counting +technique with lower execution overhead than plain atomic reference +counting. The other two techniques are immortalization and a limited +form of deferred reference counting; they address some of the +multi-threaded scalability issues with reference counting by avoiding +some reference count modifications. + +Biased reference counting (BRC) is a technique first described in 2018 +by Jiho Choi, Thomas Shull, and Josep Torrellas [#brc]_. It is based on the +observation that most objects are only accessed by a single thread, +even in multi-threaded programs. Each object is associated with an +owning thread (the thread that created it). Reference counting +operations from the owning thread use non-atomic instructions to +modify a "local" reference count. Other threads use atomic +instructions to modify a "shared" reference count. This design avoids +many atomic read-modify-write operations that are expensive on +contemporary processors. + +The implementation of BRC proposed in this PEP largely matches the +original description of biased reference counting, but differs in +details like the size of reference counting fields and special bits in +those fields. BRC requires storing three pieces of information in each +object's header: the "local" reference count, the "shared" reference +count, and the identifier of the owning thread. The BRC paper packs +these three things into a single 64-bit field. This PEP proposes using +three separate fields in each object's header to avoid potential issues +due to reference count overflow. Additionally, the PEP supports a +faster deallocation path that avoids an atomic operation in the common +case. + +The proposed ``PyObject`` struct (also called ``struct _object``) is +below: + +.. code-block:: c + + struct _object { + _PyObject_HEAD_EXTRA + uintptr_t ob_tid; // owning thread id (4-8 bytes) + uint16_t __padding; // reserved for future use (2 bytes) + PyMutex ob_mutex; // per-object mutex (1 byte) + uint8_t ob_gc_bits; // GC fields (1 byte) + uint32_t ob_ref_local; // local reference count (4 bytes) + Py_ssize_t ob_ref_shared; // shared reference count and state bits (4-8 bytes) + PyTypeObject *ob_type; + }; + +The ``ob_tid``, ``ob_ref_local``, and ``ob_ref_shared`` are used by +the biased reference counting implementation. The ``ob_gc_bits`` field +is used store garbage collection flags that were previously stored in +``PyGC_Head`` (see `Garbage Collection (Cycle Collection)`_). The +``ob_mutex`` field provides a per-object lock in a single byte. + + + +Immortalization +''''''''''''''' + +Some objects, such as interned strings, small integers, statically +allocated PyTypeObjects, and the ``True``, ``False``, and ``None`` +objects stay alive for the lifetime of the program. These objects are +marked as immortal by setting the local reference count field +(``ob_ref_local``) to ``UINT32_MAX``. + +The ``Py_INCREF`` and ``Py_DECREF`` macros are no-ops for immortal +objects. This avoids contention on the reference count fields of +these objects when multiple threads access them concurrently. + +This proposed immortalization scheme is very similar to :pep:`683`, +adopted in Python 3.12, but with slightly different bit representation +in the reference count fields for immortal objects in order to work +with biased reference counting and deferred reference counting. See +also `Why Not Use PEP 683 Immortalization?`_. + +Biased Reference Counting +''''''''''''''''''''''''' + +Biased reference counting has a fast-path for objects "owned" by the +current thread and a slow-path for other objects. Ownership is +indicated by the ``ob_tid`` field. Determining the thread id requires +platform specific code [#tid]_. A value of ``0`` in ``ob_tid`` +indicates that the object is not owned by any thread. + +The ``ob_ref_local`` field stores the local reference count and two +flags. The two most significant bits are used to indicate the object +is immortal or uses deferred reference counting (see `Deferred +reference counting`_). + +The ``ob_ref_shared`` field stores the shared reference count. The +two *least* significant bits are used to store the reference +counting state. The shared reference count is therefore shifted left by +two. The ``ob_ref_shared`` field uses the least significant bits +because the shared reference count can be temporarily negative; increfs +and decrefs may not be balanced between threads. + +The possible reference counting states are listed below: + +* ``0b00`` - default +* ``0b01`` - weakrefs +* ``0b10`` - queued +* ``0b11`` - merged + +The states form a progression: during their lifecycle, objects may +transition to any numerically higher state. Objects can only be +deallocated from the "default" and "merged" states. Other states must +transition to the "merged" state before deallocation. Transitioning +states requires an atomic compare-and-swap on the ``ob_ref_shared`` +field. + +Default (``0b00``) +"""""""""""""""""" + +Objects are initially created in the default state. This is the only +state that allows for the quick deallocation code path. Otherwise, the +thread must merge the local and shared reference count fields, which +requires an atomic compare-and-swap. + +This quick deallocation code path would not be thread-safe with +concurrent dereferencing of weakrefs, so the first time a weak +reference is created, the object is transitioned to the "weakrefs" +state if it is currently in the "default" state. + +Similarly, the quick deallocation code path would not be thread-safe +with the lockless list and dictionary accesses (see `Optimistically +Avoiding Locking`_), so the first time a non-owning thread thread +attempts to retrieve an object in the "default" state it falls back to +the slower locking code path and transitions the object to +the "weakrefs" state. + + +Weakrefs (``0b01``) +""""""""""""""""""" + +Objects in weakref and higher states support dereferencing weakrefs +as well as the lockless list and dictionary access by non-owning +threads. They require transitioning to the merged state before +deallocation, which is more expensive than the quick deallocation code +path supported by the "default" state. + + +Queued (``0b10``) +"""""""""""""""""" + +The queued state indicates that the a non-owning thread has requested +that the reference count fields be merged. This can happen when the +shared reference count becomes negative (due to an imbalance between +increfs and decrefs between threads). The object is inserted into the +owning thread's queue of objects to be merged. The owning thread is +notified via the ``eval_breaker`` mechanism. In practice, this +operation is rare. Most objects are only accessed by a single thread +and those objects accessed by multiple threads rarely have negative +shared reference counts. + +If the owning thread has terminated, the acting thread immediately +merges the local and shared reference count fields and transitions to +the merged state. + + +Merged (``0b11``) +""""""""""""""""" + +The merged state indicates that the object is not owned by any thread. +The ``ob_tid`` field is zero in this state and ``ob_ref_local`` is not +used. Once the shared reference count reaches zero, the object can +be deallocated from the merged state. + + +Reference counting pseudo-code +"""""""""""""""""""""""""""""" + + +The proposed ``Py_INCREF`` and ``Py_DECREF`` operation should behave +as follows (using C-like pseudo-code): + +.. code-block:: c + + // low two bits of "ob_ref_shared" are used for flags + #define _Py_SHARED_SHIFT 2 + + void Py_INCREF(PyObject *op) + { + uint32_t new_local = op->ob_ref_local + 1; + if (new_local == 0) + return; // object is immortal + if (op->ob_tid == _Py_ThreadId()) + op->ob_ref_local = new_local; + else + atomic_add(&op->ob_ref_shared, 1 << _Py_SHARED_SHIFT); + } + + void Py_DECREF(PyObject *op) + { + if (op->ob_ref_local == _Py_IMMORTAL_REFCNT) { + return; // object is immortal + } + if (op->ob_tid == _Py_ThreadId()) { + op->ob_ref_local -= 1; + if (op->ob_ref_local == 0) { + _Py_MergeZeroRefcount(); // merge refcount + } + } + else { + _Py_DecRefShared(); // slow path + } + } + + void _Py_MergeZeroRefcount(PyObject *op) + { + if (op->ob_ref_shared == 0) { + // quick deallocation code path (common case) + op->ob_tid = 0; + _Py_Dealloc(op); + } + else { + // slower merging path not shown + } + } + +The reference implementation [#nogil312]_ contains implementations of +``_Py_MergeZeroRefcount`` and ``_Py_DecRefShared``. + +Note that the above is pseudocode: in practice, the implementation +should use "relaxed atomics" to access ``ob_tid`` and +``ob_ref_local`` to avoid undefined behavior in C and C++. + + +Deferred Reference Counting +''''''''''''''''''''''''''' + +A few types of objects, such as top-level functions, code objects, +modules, and methods, tend to be frequently accessed by many threads +concurrently. These objects don't necessarily live for the lifetime of +the program, so immortalization is not a good fit. This PEP proposes a +limited form of deferred reference counting to avoid contention on +these objects' reference count fields in multi-threaded programs. + +Typically, the interpreter modifies objects' reference counts as they +are pushed to and popped from the interpreter's stack. The +interpreter skips these reference counting operations for objects +that use deferred reference counting. Objects that support deferred +reference counting are marked by setting the two most significant +bits in the local reference count field to one. + +Because some reference counting operations are skipped, the reference +count fields no longer reflect the true number of references to these +objects. The true reference count is the sum of the reference count +fields plus any skipped references from each thread's interpreter +stack. The true reference count can only be safely computed when all +threads are paused during cyclic garbage collection. Consequently, +objects that use deferred reference counting can only be deallocated +during garbage collection cycles. + +Note that the objects that use deferred reference counting already +naturally form reference cycles in CPython, so they would typically be +deallocated by the garbage collector even without deferred reference +counting. For example, top-level functions and modules form a reference +cycle as do methods and type objects. + + +Garbage Collector Modifications for Deferred Reference Counting +''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +The tracing garbage collector finds and deallocates unreferenced +objects. Currently, the tracing garbage collector only finds +unreferenced objects that are part of a reference cycle. With +deferred reference counting, the tracing garbage collector will also +find and collect some unreferenced objects that may not be part of +any reference cycle, but whose collection has been delayed due to +deferred reference counting. This requires that all objects that +support deferred reference counting also have a corresponding type +object that supports tracing garbage collection (through the +``Py_TPFLAGS_HAVE_GC`` flag). Additionally, the garbage collector +will need to traverse each thread's stack to add references to the GC +reference count at the start of each collection. + +Reference Counting Type Objects +''''''''''''''''''''''''''''''' + +Type objects (``PyTypeObject``) use a mix of reference counting +techniques. Statically allocated type objects are immortalized because +the objects already live for the lifetime of the program. Heap type +objects use deferred reference counting in combination with per-thread +reference counting. Deferred reference counting is not sufficient to +address the multi-threaded scaling bottlenecks with heap types because +most references to heap types are from object instances, not references +on the interpreter stack. + +To address this, heap type reference counts are partially stored in a +distributed manner in per-thread arrays. Every thread stores an +array of local reference counts for each heap type object. Heap type +objects are assigned a unique number that determines its position in +the local reference count arrays. A heap type's true reference count +is the sum of its entries in the per-thread arrays, plus the reference +count on the ``PyTypeObject``, plus any deferred references in the +interpreter stack. + +Threads may grow their own type reference count arrays as needed when +incrementing or decrementing the local reference count of a type +object. + +Use of the per-thread reference count arrays is limited to a few +places: + +* ``PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)``: + Increments the current thread's local reference count for ``type``, + if it is a heap type. +* ``subtype_dealloc(PyObject *self)``: Decrements the current thread's + local reference count for ``self->ob_type``, if the type is a heap + type. +* ``gcmodule.c``: Adds each thread's local reference counts to the + ``gc_refs`` count for the corresponding heap type object. + +Additionally, when a thread terminates, it adds any non-zero local +reference counts to each type object's own reference count field. + + +Memory Management +----------------- + +CPython currently uses an internal allocator, pymalloc, which is +optimized for small object allocation. The pymalloc implementation is +not thread-safe without the GIL. This PEP proposes replacing pymalloc +with mimalloc, a general-purpose thread-safe allocator with good +performance, including for small allocations. + +Using mimalloc, with some modifications, also addresses two other +issues related to removing the GIL. First, traversing the internal +mimalloc structures allows the garbage collector to find all Python +objects without maintaining a linked list. This is described in more +detail in the garbage collection section. Second, mimalloc heaps and +allocations based on size class enable collections like dict to +generally avoid acquiring locks during read-only operations. This is +described in more detail in the collection thread-safety section. + +CPython already requires that objects that support garbage collection +use the GC allocator APIs (typically indirectly by calling +``PyType_GenericAlloc``). This PEP would add additional requirements +to the use of the Python allocator APIs. First, Python objects must +be allocated through object allocation APIs, such as +``PyType_GenericAlloc``, ``PyObject_Malloc``, or other Python APIs +that wrap those calls. Python objects should not be allocated through +other APIs, such as raw calls to C's malloc or the C++ new operator. +Additionally, ``PyObject_Malloc`` should be used only for allocating +Python objects; it should not be used for allocating buffers, +storages, or other data structures that are not PyObjects. + +This PEP also imposes restrictions on the pluggable allocator API +(``PyMem_SetAllocator``). When compiling without the GIL, allocators +set using this API must eventually delegate the allocation to the +corresponding underlying allocator, such as ``PyObject_Malloc``, for +Python object allocations. This allows for allocators that "wrap" +underlying allocators, such as Python's tracemalloc and debug +allocator, but not for wholly replacing the allocator. + + +CPython Free Lists +'''''''''''''''''' + +CPython makes use of free lists to speed up the allocation of small, +frequently allocated objects like tuples and numbers. These free +lists are moved to ``PyThreadState`` from per-interpreter state. + + + +Garbage Collection (Cycle Collection) +------------------------------------- + +The CPython garbage collector requires the following changes to work +with this proposal: + +* Use of "stop-the-world" to provide thread-safety guarantees that + were previously provided by the GIL. +* Elimination of generational garbage collection in favor of + non-generational collector. +* Integration with deferred reference counting and biased reference + counting. + +Additionally, the above changes enable removing the +``_gc_prev`` and ``_gc_next`` fields from GC objects. The GC bits +that stored the tracked, finalized, and unreachable states are moved +to the ``ob_gc_bits`` field in the PyObject header. + +Stop-the-World +'''''''''''''' + +The CPython cycle garbage collector currently relies on the global +interpreter lock to prevent other threads from accessing Python +objects while the collector finds cycles. The GIL is never released +during the cycle-finding routine, so the collector can rely on +stable (i.e., unchanging) reference counts and references for the +duration of that routine. However, following cycle detection, the GIL +may be temporarily released while calling objects' finalizers and +clear (``tp_clear``) functions, allowing other threads to run in an +interleaved fashion. + +When running without the GIL, the implementation needs a way to ensure +that reference counts remain stable during cycle detection. Threads +running Python code must be paused to ensure that references and +reference counts remain stable. Once the cycles are identified, other +threads are resumed. + +The current CPython cyclic garbage collector involves two +cycle-detection passes during each garbage collection cycle. +Consequently, this requires two stop-the-world pauses when running the +garbage collector without the GIL. The first cycle-detection pass +identifies cyclic trash. The second pass runs after finalizers to +identify which objects still remain unreachable. Note that other +threads are resumed before finalizers and ``tp_clear`` functions are +called to avoid introducing potential deadlocks that are not present in +the current CPython behavior. + +Thread States +''''''''''''' + +To support pausing threads for garbage collection, the PyThreadState +gets a new "status" field. Like the other fields in PyThreadState, +the status field is not part of the public CPython API. The status +field may be in one of three states: + +* ``ATTACHED`` +* ``DETACHED`` +* ``GC`` + +The ``ATTACHED`` and ``DETACHED`` states correspond closely to +acquiring and releasing the global interpreter lock. When compiling +without the GIL, functions that previously acquired the GIL instead +transition the thread state to ``ATTACHED``, and functions that +previously released the GIL transition the thread state +to ``DETACHED``. Just as threads previously needed to acquire the +GIL before accessing or modifying Python objects, they now must be in +the ``ATTACHED`` state before accessing or modifying Python +objects. Since the same public C-API functions "attach" the thread as +previously acquired the GIL (e.g., ``PyEval_RestoreThread``), the +requirements for thread initialization in extensions remain the same. +The substantial difference is that multiple threads can be in the +attached state simultaneously, while previously only one thread could +acquire the GIL at a time. + +During stop-the-world pauses, the thread performing garbage collection +needs to ensure that no other thread is accessing or modifying Python +objects. All other threads must be in the "GC" state. The garbage +collection thread can transition other threads from the ``DETACHED`` +state to the GC state using an atomic compare-and-swap operation on +the status field. Threads in the ``ATTACHED`` state are requested to +pause themselves and set their status to "GC", using the +existing "eval breaker" mechanism. At the end of the stop-the-world +pause, all threads in the "GC" state are set to ``DETACHED`` and +woken up if they are paused. Threads that were previously attached +(i.e., executing Python bytecode) can re-attach (set their thread +states to ``ATTACHED``) and resume executing Python code. Threads +that were previously ``DETACHED`` ignore the notification. + +Generations +''''''''''' + +The existing Python garbage collector uses three generations. When +compiling without the GIL, the garbage collector will only use a single +generation (i.e., it will be non-generational). The primary reason for +this change is to reduce the impact of the stop-the-world pauses in +multithreaded applications. Frequent stop-the-world pauses for +collecting the young generation would have more of an impact on +multi-threaded applications than less frequent collections. + + +Integration With Deferred and Biased Reference Counting +''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +To find unreferenced objects, the cyclic garbage collector computes +the difference between the number of incoming references and the +object's reference count. This difference is called ``gc_refs`` and +is stored in the ``_gc_prev`` field. If ``gc_refs`` is greater than +zero, then the object is guaranteed to be alive (i.e., not cyclic +trash). If ``gc_refs`` is zero, then the object is only alive if it +is transitively referenced by another live object. When computing +this difference, the collector should traverse each thread's stack, +and for every deferred reference, increment the ``gc_refs`` for the +referred object. Since generator objects also have stacks with +deferred references, the same procedure is applied to each +generator's stack. + +Python unit tests commonly use ``gc.collect()`` to ensure that any +unreferenced objects are destructed and their finalizers run. Since +biased reference counting can delay the destruction of some objects +that are referenced by multiple threads, it's convenient to ensure +that those objects are destructed during garbage collection, even +though they may not be part of any reference cycles. While other +threads are paused, the garbage collector thread should merge the +reference counts for any queued objects, but not call any destructors +even if the combined reference count is zero. (Calling destructors +while other threads are paused risks introducing deadlocks.) Once +other threads are resumed, the GC thread should call ``_Py_Dealloc`` +on those objects with a zero merged reference count. + +Container Thread-Safety +----------------------- + +In CPython, the global interpreter lock protects against corruption of +internal interpreter states when multiple threads concurrently access +or modify Python objects. For example, if multiple threads +concurrently modify the same list, the GIL ensures that the length of +the list (``ob_size``) accurately matches the number of elements, and +that the reference counts of each element accurately reflect the +number of references to those elements. Without the GIL --- and +absent other changes --- concurrent modifications would corrupt those +fields and likely lead to program crashes. + +The GIL does not necessarily ensure that operations are atomic or +remain correct when multiple operations occur concurrently. For +example, ``list.extend(iterable)`` may not appear atomic if the +iterable has an iterator implemented in Python (or releases the GIL +internally). Similarly, ``list.remove(x)`` can remove the wrong +object if it overlaps with another operation that modifies the list, +depending on the implementation of the equality operator. Still, the +GIL ensures that some operations are effectively atomic. For example, +the constructor ``list(set)`` atomically copies the items of the set +to a new list, and some code relies on that copy being atomic +(i.e., having a snapshot of the items in the set). This PEP preserves +that property. + +This PEP proposes using per-object locks to provide many of the same +protections that the GIL provides. For example, every list, +dictionary, and set will have an associated lightweight lock. All +operations that modify the object must hold the object's lock. Most +operations that read from the object should acquire the object's lock +as well; the few read operations that can proceed without holding a +lock are described below. + +Per-object locks with critical sections provide weaker protections +than the GIL. Because the GIL doesn't necessarily ensure that +concurrent operations are atomic or correct, the per-object locking +scheme also cannot ensure that concurrent operations are atomic or +correct. Instead, per-object locking aims for similar protections as +the GIL, but with mutual exclusion limited to individual objects. + +Most operations on an instance of a container type require locking +that object. For example: + +* ``list.append``, ``list.insert``, ``list.repeat``, + ``PyList_SetItem`` +* ``dict.__setitem__``, ``PyDict_SetItem`` +* ``list.clear``, ``dict.clear`` +* ``list.__repr__``, ``dict.__repr__``, etc. +* ``list.extend(iterable)`` +* ``setiter_iternext`` + +Some operations operate directly on two container objects, with +knowledge about both containers' internal structure. For example, +there are internal specializations of ``list.extend(iterable)`` for +specific iterable types, like ``set``. These operations need to lock +both container objects because they access the internals of both +objects simultaneously. Note that the generic implementation of +``list.extend`` only needs to lock one object (the list) because the +other object is accessed indirectly through the thread-safe iterator +API. Operations that lock two containers are: + +* ``list.extend(list)``, ``list.extend(set)``, ``list.extend + (dictitems)``, and other specializations where the implementation + is specialized for argument type. +* ``list.concat(list)`` +* ``list.__eq__(list)``, ``dict.__eq__(dict)`` + +Some simple operations can be implemented directly with atomic +accesses and do not need locks because they only access a single +field. These operations include: + +* ``len(list)`` i.e., ``list_length(PyListObject *a)`` +* ``len(dict)`` +* ``len(set)`` + +A select few operations optimistically avoid locking to improve +performance. These require special implementations and cooperation +from the memory allocator: + +* ``list[idx]`` (``list_subscript``) +* ``dict[key]`` (``dict_subscript``) +* ``listiter_next``, ``dictiter_iternextkey/value/item`` +* ``list.contains`` + +Borrowed References +''''''''''''''''''' + +Per-object locking provides many of the important protections that the +GIL provides, but there are a few cases where it's not sufficient. +For example, code that relies on upgrading a borrowed reference to +an "owned" reference may be unsafe in certain circumstances: + +.. code-block:: c + + PyObject *item = PyList_GetItem(list, idx); + Py_INCREF(item); + +The GIL ensures that no other thread can modify the list in between +the access and the ``Py_INCREF`` call. Without the GIL -- even with +per-object locking -- another thread might modify the list leading to +``item`` being freed between the access and the ``Py_INCREF`` call. + +The problematic borrowed reference APIs are supplemented with +functions that return "new references" but are otherwise +equivalent: + +* ``PyList_FetchItem(list, idx)`` for ``PyList_GetItem`` +* ``PyDict_FetchItem(dict, key)`` for ``PyDict_GetItem`` +* ``PyWeakref_FetchObject`` for ``PyWeakref_GetObject`` + +Note that some APIs that return borrowed references, such as +``PyTuple_GetItem``, are not problematic because tuples are +immutable. Similarly, not all uses of the above APIs are problematic. +For example, ``PyDict_GetItem`` is often used for parsing keyword +argument dictionaries in function calls; those keyword argument +dictionaries are effectively private (not accessible by other +threads). + +Python Critical Sections +'''''''''''''''''''''''' + +Straightforward per-object locking could introduce deadlocks that were +not present when running with the GIL. Threads may hold locks for +multiple objects simultaneously because Python operations can nest. +Operations on objects can invoke operations on other objects, +acquiring multiple per-object locks. If threads try to acquire the +same locks in different orders, they will deadlock. + +This PEP proposes a scheme called "Python critical sections" to +implicitly release per-object locks to avoid deadlocks. To +understand the scheme, we first introduce a general approach to avoid +deadlocks, and then propose a refinement of that approach with better +performance. + +One way to avoid deadlocks is to allow threads to hold only the lock +(or locks) for a single operation at a time (typically a single lock, +but some operations involve two locks as described above). When a +thread begins a nested operation it should suspend the locks for any +outer operation: before beginning the nested operation, the locks for +the outer operation are released and when the nested operation +completes, the locks for the outer operation are reacquired. + +Additionally, the locks for any active operation should be suspended +around potentially blocking operations, such as I/O (i.e., operations +that would have released the GIL). This is because the interaction +between locks and blocking operations can lead to deadlocks in the +same way as the interaction between multiple locks. + +To improve performance, this PEP proposes a variation of the above +scheme that still avoids deadlocks. Instead of immediately +suspending locks any time a nested operation begins, locks are only +suspended if the thread would block (i.e., would have released the +GIL). This reduces the number of lock acquisitions and releases for +nested operations, while avoiding deadlocks. + +The proposed API for Python critical sections are the following four +macros. These are intended to be public (usable by C-API extensions), +but not part of the limited API: + +- ``Py_BEGIN_CRITICAL_SECTION(PyObject *op);``: + Begins a critical section by acquiring the mutex for the referenced + object. If the object is already locked, then locks for any + outstanding critical sections are released before this thread waits + for referenced object to be unlocked. + +- ``Py_END_CRITICAL_SECTION;``: + Ends the most recent operation, unlocking the mutex. The next + most recent previous critical section (if any) is resumed if it is + currently suspended. + +- ``Py_BEGIN_CRITICAL_SECTION2(PyObject *a, PyObject *b);``: + Begins a critical section by acquiring the mutexes for two objects. + To ensure consistent lock ordering, the order of acquisition is + determined by memory address (i.e., the mutex with lower memory + address is acquired first). If either mutex is already locked, then + locks for any outstanding critical sections are released before this + thread waits for the referenced objects to be unlocked. + +- ``Py_END_CRITICAL_SECTION2;``: + Behaves the same as ``Py_END_CRITICAL_SECTION`` but unlocks two + objects. + +Additionally, when a thread transitions from the ``ATTACHED`` state to +the ``DETACHED`` state, it should suspend any active critical +sections. When transitioning from ``DETACHED`` to ``ATTACHED``, the +most recent suspended critical section, if any, should be resumed. + +Note that operations that lock two containers simultaneously need to use +the ``Py_BEGIN_CRITICAL_SECTION2`` macro. It is not sufficient to nest +two calls to ``Py_BEGIN_CRITICAL_SECTION`` because the inner critical +section may release the locks from the outer critical section. + +Optimistically Avoiding Locking +''''''''''''''''''''''''''''''' + +A few operations on ``dict`` and ``list`` optimistically avoid +acquiring the per-object locks. They have a fast path operation that +does not acquire locks, but may fall back to a slower operation that +acquires the dictionary's or list's lock when another thread is +concurrently modifying that container. + +The operations with an optimistic fast path are: + +* ``PyDict_FetchItem/GetItem`` and ``dict.__getitem__`` +* ``PyList_FetchItem/GetItem`` and ``list.__getitem__`` + +Additionally, iterators for ``dict`` and ``list`` use the above +functions so they also optimistically avoid locking when returning +the next item. + +There are two motivations for avoiding lock acquisitions in these +functions. The primary reason is that it is necessary for scalable +multi-threaded performance even for simple applications. Dictionaries +hold top-level functions in modules and methods for classes. These +dictionaries are inherently highly shared by many threads in +multi-threaded programs. Contention on these locks in multi-threaded +programs for loading methods and functions would inhibit efficient +scaling in many basic programs. + +The secondary motivation for avoiding locking is to reduce overhead +and improve single-threaded performance. Although lock acquisition +has low overhead compared to most operations, accessing individual +elements of lists and dictionaries are fast operations (so the +locking overhead is comparatively larger) and frequent (so the +overhead has more impact). + +This section describes the challenges with implementing dictionary and +list accesses without locking followed by a description of this PEP's +changes to the Python interpreter required to address those +challenges. + +The main challenge is that retrieving an item from a list or +dictionary and incrementing the reference count of that item is not +an atomic operation. In between the time the item is retrieved and +the reference count is incremented, another thread may modify the +list or dictionary, possibly freeing the memory for the previously +retrieved item. + +A partial attempt at addressing this issue would be to convert the +reference count increment to a conditional increment, only +incrementing the reference count if it's not zero. This change is +not sufficient because when a Python object's reference count reaches +zero, the object's destructor is called and the memory storing the +object may be re-used for other data structures or returned to the +operating system. Instead, this PEP proposes a technique to ensure +that the reference count fields remain valid for the duration of the +access, so that the conditional reference count increment is safe. +This technique requires cooperation from the memory allocator +(mimalloc) as well as changes to the list and dictionary objects. The +proposed technique is similar to read-copy update (RCU) [#rcu]_, a +synchronization mechanism widely used in the Linux kernel. + +The current implementation of ``list_item`` (the C function +implementing ``list.__getitem__``) is the following: + +.. code-block:: c + + Py_INCREF(a->ob_item[i]); + return a->ob_item[i]; + +The proposed implementation uses the conditional increment +(``_Py_TRY_INCREF``) and has additional checks: + +.. code-block:: c + + PyObject **ob_item = atomic_load(&a->ob_item); + PyObject *item = atomic_load(&ob_item[i]); + if (!item || !_Py_TRY_INCREF(item)) goto retry; + if (item != atomic_load(&ob_item[i])) { + Py_DECREF(item); + goto retry; + } + if (ob_item != atomic_load(&a->ob_item)) { + Py_DECREF(item); + goto retry; + } + return item; + + +The "retry" subroutine implements the locked fallback path when +concurrent modifications to the list cause the above fast, +non-locking path to fail: + +.. code-block:: c + + retry: + PyObject *item; + Py_BEGIN_CRITICAL_SECTION(a->ob_mutex); + item = a->ob_item[i]; + Py_INCREF(item); + Py_END_CRITICAL_SECTION(a->ob_mutex); + return item; + +The modifications to the ``dict`` implementation are similar, because +the relevant parts of both list and dictionary retrieval involve +loading an item/value from an array at a known index. + +The additional checks following the conditional increment are +necessary because the scheme allows immediate re-use of memory, +including the memory that previously held a ``PyObject`` structure or +``list`` or ``dict`` array. Without these extra checks, the function +might return a Python object that was never in the list, if the +memory occupied by the Python object previously held a different +``PyObject`` whose memory previously stored an item in the list. + + +Mimalloc Changes for Optimistic ``list`` and ``dict`` Access +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +The implementation requires additional constraints to the memory +allocator, including some changes to the mimalloc code. Some +background on mimalloc's implementation is helpful to understand the +required changes. Individual allocations from mimalloc are +called "blocks." Mimalloc "pages" contain consecutive blocks that +are all the same size. A mimalloc "page" is similar to +a "superblock" in other allocators; it is NOT an operating system +page. A mimalloc "heap" contains pages of various size classes; each +page belongs to a single heap. If none of the blocks of a page are +allocated, then mimalloc may re-use the page for a different size +class or different heap (i.e., it might reinitialize the page). + +The list and dictionary access scheme works by partially restricting +re-use of mimalloc pages so that reference count fields remains valid +for the duration of the access. The restricted re-use of mimalloc +pages is enforced by having separate heaps for Python objects +[#heaps]_. This ensures that even if an item is freed during access +and the memory reused for a new object, the new object's reference +count field is placed at the same location in memory. The reference +count field remains valid (or zero) across allocations. + +Python objects that support ``Py_TPFLAGS_MANAGED_DICT`` have their +dictionary and weak reference fields preceding the ``PyObject`` +header, so their reference count fields are at a different offset from +the start of their allocations. They are stored in a separate mimalloc +heap. Additionally, non-GC objects are stored in their own heap so +that the GC only has to look at GC objects. There are therefore three +mimalloc heaps for Python objects, one for non-GC objects, one for GC +objects with managed dictionaries, and one for GC objects without +managed dictionaries. + + +Mimalloc Page Reuse +''''''''''''''''''' + +It is beneficial to keep the restrictions on mimalloc page reuse to a +short period of time to avoid increasing overall memory usage. +Precisely limiting the restrictions to list and dictionary accesses +would minimize memory usage, but would require expensive +synchronizations. At the other extreme, keeping the restrictions +until the next GC cycle would avoid introducing any extra +synchronizations, but would potentially increase memory usage. + +This PEP proposes a system that lies between those two extremes based +on FreeBSD's "GUS" [#gus]_. It uses a combination of global and +per-thread counters (or "sequence numbers") to coordinate the +determination of when it is safe to reuse an empty mimalloc page for +a different heap or for a different size class, or to return it to +the operating system: + +* There is a global write sequence number that monotonically + increases. +* When a mimalloc page is empty, it's tagged with the current write + sequence number. The thread may also atomically increment the + global write sequence number. +* Each thread has a local read sequence number that records the most + recent write sequence number it has observed. +* Threads may observe the write sequence number whenever they are not + in a list or dictionary access. The reference implementation does + this in mimalloc's slow-path allocation function. This is called + regularly enough to be useful, but not so frequently as to + introduce significant overhead. +* There is a global read sequence number that stores the minimum of + all active threads' read sequence numbers. A thread may update the + global read sequence number by scanning each threads' local read + sequence number. The reference implementation does this before + allocating a fresh mimalloc page if there are restricted pages + that could possibly be reused. +* An empty mimalloc page may be reused for a different heap or size + class when the global read sequence number is larger than the + page's tag number. + +The condition that the global read sequence number is larger than the +page's tag is sufficient because it ensures that any thread that had +a concurrent optimistic list or dictionary access is finished with +that access. In other words, there are no threads accessing the +empty blocks in the freed page, so the page can be used for any other +purpose or even returned to the operating system. + +Optimistic ``dict`` and ``list`` Access Summary +''''''''''''''''''''''''''''''''''''''''''''''' + +This PEP proposes a technique for thread-safe list and dictionary +accesses that typically avoids acquiring locks. This reduces +execution overhead and avoids some multi-threaded scaling bottlenecks +in common operations, like calling functions and methods. The scheme +works by placing temporary restrictions on mimalloc page reuse to +ensure that objects' reference count fields remain valid after +objects are freed so that conditional reference count increment +operations are safe. The restrictions are placed on mimalloc pages +instead of on individual objects to improve opportunities for memory +reuse. The restrictions are lifted as soon as the system can +determine that there are no outstanding accesses involving the empty +mimalloc page. To determine this, the system uses a combination of +lightweight per-thread sequence counters and also tags pages when +they are empty. Once each thread's local counter is larger than the +page's tag, it can be reused for any purpose or returned to the +operating system. The restrictions are also lifted whenever the +cyclic garbage collector runs because the stop-the-world pause +ensures that threads do not have any outstanding references to empty +mimalloc pages. + + +Specializing Interpreter +------------------------ + +The specializing interpreter requires some changes to be thread-safe +when running without the GIL: + +* Concurrent specializations are prevented by using a mutex. This + prevents multiple threads writing to the same inline cache. +* In multi-threaded programs running without the GIL, each bytecode is + only specialized once. This prevents a thread from reading a + partially written inline cache. +* Locking also ensures that cached values of ``tp_version_tag`` and + ``keys_version`` are consistent with the cached descriptors and other + values. +* Modifications to inline counters use "relaxed atomics". In other + words, some counter decrements may be missed or overwritten, but that + does not affect correctness. + + +``Py_mod_gil`` Slot +------------------- + +In ``--disable-gil`` builds, when loading an extension, CPython will +check for a new :pep:`489`-style ``Py_mod_gil`` slot. If the slot is +set to ``Py_mod_gil_not_used``, then extension loading proceeds as +normal. If the slot is not set, the interpreter pauses all threads and +enables the GIL before continuing. Additionally, the interpreter will +issue a visible warning naming the extension, that the GIL was enabled +(and why) and the steps the user can take to override it. + + +``PYTHONGIL`` Environment Variable +---------------------------------- + +In ``--disable-gil`` builds, the user can also override the behavior at +runtime by setting the ``PYTHONGIL`` environment variable. Setting +``PYTHONGIL=0``, forces the GIL to be disabled, overriding the module +slot logic. Setting ``PYTHONGIL=1``, forces the GIL to be enabled. + +The ``PYTHONGIL=0`` override is important because extensions that are +not thread-safe can still be useful in multi-threaded applications. For +example, one may want to use the extension from only a single thread or +guard access by locks. For context, there are already some extensions +that are not thread-safe even with the GIL, and users already have to +take these sorts of steps. + +The ``PYTHONGIL=1`` override is sometimes useful for debugging. + + +Rationale +========= + +Non-Generational Garbage Collection +----------------------------------- + +This PEP proposes switching from a generational cyclic garbage +collector to a non-generational collector (when CPython is built +without the GIL). That is equivalent to only having one generation +(the "old" generation). There are two reasons for this proposed +change. + +Cyclic garbage collection, even for just the young generation, +requires pausing other threads in the program. The author is +concerned that frequent collections of the young generation would +inhibit efficient scaling in multi-threaded programs. This is a +concern for young generations (but not the old generation) because +the young generations are collected after a fixed number of +allocations, while the collections for the older generation are +scheduled in proportion to the number of live objects in the heap. +Additionally, it is difficult to efficiently keep track of objects in +each generation without the GIL. For example, CPython currently uses +a linked list of objects in each generation. If CPython were to keep +that design, those lists would need to be made thread-safe, and it's +not clear how to do that efficiently. + +Generational garbage collection is used to good effect in many other +language runtimes. For example, many of the Java HotSpot garbage +collector implementations use multiple generations [#hotspotgc]_. In +these runtimes, a young generation is frequently a throughput win: +since a large percentage of the young generation is typically "dead," +the GC is able to reclaim a large amount memory relative to the +amount of work performed. For example, several Java benchmarks show +over 90% of "young" objects are typically collected [#decapo]_ +[#exploitingmemoryjava]_. This is commonly referred to as the "weak +generational hypothesis;" the observation is that most objects die +young. This pattern is reversed in CPython due to the use of +reference counting. Although most objects still die young, they are +collected when their reference counts reach zero. Objects that +survive to a garbage collection cycle are most likely to remain +alive [#cpythongc]_. This difference means that generational +collection is much less effective in CPython than in many other +language runtimes [#golangc]_. + + +Optimistic Avoiding Locking in ``dict`` and ``list`` Accesses +------------------------------------------------------------- + +This proposal relies on a scheme that mostly avoids acquiring locks +when accessing individual elements in lists and dictionaries. Note +that this is not "lock free" in the sense of "lock-free" +and "wait-free" algorithms that guarantee forward progress. It +simply avoids acquiring locks (mutexes) in the common case to improve +parallelism and reduce overhead. + +A much simpler alternative would be to use reader-writer locks to +protect dictionary and list accesses. Reader-writer locks allow +concurrent reads, but not updates, which might seem ideal for list +and dictionaries. The problem is that reader-writer locks have +substantial overhead and poor scalability, particularly when the +critical sections are small, as they are for single-element +dictionary and list accesses [#perfbook]_. The poor reader +scalability stems from the fact that readers must all update the same +data structure, such as the number of readers in +``pthread_rwlocks``. + +The technique described in this PEP is related to RCU +("read-copy-update") [#rcu]_ and, to a lesser extent, hazard +pointers, two well-known schemes for optimizing concurrent, +read-mostly data structures. RCU is widely used in the Linux kernel +to protect shared data structures in a scalable manner. Both the +technique in this PEP and RCU work by deferring reclamation while +readers may be accessing the concurrent data structure. RCU is most +commonly used to protect individual objects (like hash tables or +linked lists), while this PEP proposes a scheme to protect larger +blocks of memory (mimalloc "pages") [#typesafe_rcu]_. + +The need for this scheme is largely due to the use of reference +counting in CPython. If CPython only relied on a tracing garbage +collector, then this scheme would probably not be necessary because +tracing garbage collectors already defer reclamation in the required +manner. This would not "solve" scaling issues, but would shift many +of the challenges to the garbage collector implementation. + + +Backwards Compatibility +======================= + +This PEP poses a number of backwards compatibility issues when +building CPython with the ``--disable-gil`` flag, but those issues do +not occur when using the default build configuration. Nearly all the +backwards compatibility concerns involve the C-API: + +* CPython builds without the GIL will not be ABI compatible with the + standard CPython build or with the stable ABI due to changes to the + Python object header needed to support biased reference counting. + C-API extensions will need to be rebuilt specifically for this + version. +* C-API extensions that rely on the GIL to protect global state or + object state in C code will need additional explicit locking to + remain thread-safe when run without the GIL. +* C-API extensions that use borrowed references in ways that are not + safe without the GIL will need to use the equivalent new APIs that + return non-borrowed references. Note that only some uses of + borrowed references are a concern; only references to objects that + might be freed by other threads pose an issue. +* Custom memory allocators (``PyMem_SetAllocator``) are required to + delegate the actual allocation to the previously set allocator. For + example, the Python debug allocator and tracing allocators will + continue to work because they delegate the allocation to the + underlying allocator. On the other hand, wholesale replacing of the + allocator (e.g., with jemalloc or tcmalloc) will not work + correctly. +* Python objects must be allocated through the standard APIs, such as + ``PyType_GenericNew`` or ``PyObject_Malloc``. Non-Python objects + must **not** be allocated through those APIs. For example, it is + currently acceptable to allocate buffers(non-Python objects) + through ``PyObject_Malloc``; that will no longer be allowed and + buffers should instead be allocated through ``PyMem_Malloc``, + ``PyMem_RawMalloc``, or ``malloc``. + +There are fewer potential backwards compatibility issues for Python +code: + +* Destructors and weak reference callbacks for code objects and + top-level function objects are delayed until the next cyclic + garbage collection due to the use of deferred reference counting. +* Destructors for some objects accessed by multiple threads may be + delayed slightly due to biased reference counting. This is rare: + most objects, even those accessed by multiple threads, are + destroyed immediately as soon as their reference counts are zero. + Two places in the Python standard library tests required + ``gc.collect()`` calls to continue to pass. + + +Distribution +============ + +This PEP poses new challenges for distributing Python. At least for +some time, there will be two versions of Python requiring separately +compiled C-API extensions. It may take some time for C-API extension +authors to build ``--disable-gil`` compatible packages and upload +them to PyPI. Additionally, some authors may be hesitant to support +the ``--disable-gil`` mode until it has wide adoption, but adoption +will likely depend on the availability of Python's rich set of +extensions. + +To mitigate this, the author will work with Anaconda to distribute +a ``--disable-gil`` version of Python together with compatible +packages from conda channels. This centralizes the challenges of +building extensions, and the author believes this will enable more +people to use Python without the GIL sooner than they would otherwise +be able to. + + +Performance +=========== + +The changes to make CPython thread-safe without the GIL increase +execution overhead for ``--disable-gil`` builds. The performance +impact is different for programs that use only a single thread compared +to programs that use multiple threads, so the table below reports +execution overhead separately for these types of programs separately. + + +.. list-table:: Execution Overhead on pyperformance 1.0.6 + :header-rows: 1 + :widths: auto + + * - + - Intel Skylake + - AMD Zen 3 + * - One thread + - 6% + - 5% + * - Multiple threads + - 8% + - 7% + +The baseline used to measure overhead is ``018be4c`` from `PR 19474`_, +which implements immortal objects for Python 3.12. The largest +contribution to execution overhead is biased reference counting +followed by per-object locking. For thread-safety reasons, an +application running with multiple threads will only specialize a given +bytecode once; this is why the overhead for programs that use multiple +threads is larger compared to programs that only use one thread. +However, with the GIL disabled, programs that use multiple threads +should also be able to more effectively use multiple CPU cores. + +Note that this PEP would not affect the performance of the default +(non ``--disable-gil``) builds of CPython. + +.. _PR 19474: https://github.com/python/cpython/pull/19474 + + +Build Bots +========== + +The stable build bots will also include ``--disable-gil`` builds. + + +How to Teach This +================= + +As part of implementing the ``--disable-gil`` mode, the author will +write a "HOWTO" guide [#howto]_ for making packages compatible when +running Python without the GIL. + + +Reference Implementation +======================== + +There are two GitHub repositories implementing versions of CPython +without the GIL: + +* https://github.com/colesbury/nogil-3.12 +* https://github.com/colesbury/nogil + +The ``nogil-3.12`` is based on Python 3.12.0a4. It is useful for +evaluating single-threaded execution overhead and as a reference +implementation for this PEP. It is less useful for evaluating C-API +extension compatibility because many extensions are not currently +compatible with Python 3.12. Due to limited time for the 3.12 port, +the ``nogil-3.12`` implementation does not skip all deferred reference +counts. As a temporary work around, the implementation immortalizes +objects that use deferred reference counting in programs that spawn +multiple threads. + + +The ``nogil`` repository is based on Python 3.9.10. It is useful for +evaluating multi-threading scaling in real world applications and +extension compatibility. It is more stable and well tested than the +``nogil-3.12`` repository. + +Alternatives +============ + +Python currently supports a number of ways to enable parallelism, but +the existing techniques come with significant limitations. + +Multiprocessing +--------------- + +The multiprocessing library allows Python programs to start and +communicate with Python subprocesses. This allows for parallelism +because each subprocess has its own Python interpreter (i.e., there's +one GIL per process). Multiprocessing has a few substantial +limitations. Communication between processes is limited: objects +generally need to be serialized or copied to shared memory. This +introduces overhead (due to serialization) and complicates building +APIs on top of multiprocessing. Starting a subprocess is also more +expensive than starting a thread, especially with the "spawn" +implementation. Starting a thread takes ~100 ”s, while spawning a +subprocess takes ~50 ms (50,000 ”s) due to Python re-initialization. + +Finally, many C and C++ libraries support access from multiple +threads but do not support access or use across multiple processes. + +Releasing the GIL in C-API Extensions +------------------------------------- + +C-API extensions can release the GIL around long running functions. +This allows for some degree of parallelism, since multiple threads +can run concurrently when the GIL is released, but the overhead of +acquiring and releasing the GIL typically prevents this from scaling +efficiently beyond a few threads. Many scientific computing +libraries release the GIL in computational heavy functions, and the +CPython standard library releases the GIL around blocking I/O. + +Internal Parallelization +------------------------ + +Functions implemented in C may use multiple threads internally. For +example, Intel's NumPy distribution, PyTorch, and TensorFlow all use +this technique to internally parallelize individual operations. This +works well when the basic operations are large enough to be +parallelized efficiently, but not when there are many small +operations or when the operations depend on some Python code. Calling +into Python from C requires acquiring the GIL -- even short snippets +of Python code can inhibit scaling. + + +Related Work +============= + + +Per-Interpreter GIL +------------------- + +The recently accepted :pep:`684` proposes a per-interpreter GIL to +address multi-core parallelism. This would allow parallelism between +interpreters in the same process, but places substantial restrictions +on sharing Python data between interpreters. Both this PEP +and :pep:`684` address the multi-core parallelism, but with different +tradeoffs and techniques. It is feasible to implement both PEPs in +CPython at the same time. + + +Gilectomy +--------- + +Gilectomy [#gilectomy]_ was a project by Larry Hastings to remove the +GIL in CPython. Like the design proposed by this PEP, the Gilectomy +supported multiple threads running in parallel within the same +interpreter (i.e., "free-threading") and made use of fine-grained +locking. The reference implementation in this PEP improves on +single-threaded performance and scalability compared to the +Gilectomy. + + +PyParallel +---------- + +PyParallel [#pyparallel]_ was a proof-of-concept fork of Python 3.3 by +Trent Nelson that supported multiple threads running simultaneously +in a single Python process. The fork introduced the concept +of "parallel threads" -- threads that can run simultaneously while +the main Python thread is suspended. Parallel threads had read-only +access to objects created by the main thread. Objects created within +parallel threads lived for the lifetime of the creating thread. For +HTTP servers, this might correspond to the lifetime of a request. + + + +python-safethread +----------------- + +The python-safethread [#pythonsafethread]_ project was a patch to +Python 3.0 by Adam Olsen to remove the GIL. Some aspects of the +project are similar to the design proposed by this PEP. Both use +fine-grained locking and optimize reference counting for cases +where the object is created and accessed by the same thread. + + +Greg Stein's Free-Threading Patch +--------------------------------- + +In 1996, Greg Stein published a patch against Python 1.4 that removed +the GIL [#gsteinpatch]_. The patch used atomic reference counting on +Windows and a global reference count lock on Linux. List and +dictionary accesses were protected by mutexes. Parts of the patch +were adopted in CPython. In particular, the patch introduced a +PyThreadState structure and correct per-thread exception handling. + + +Dave Beazley revisited the patch in a 2011 blog post [#dabeaz]_. + + +Jython and IronPython +--------------------- + +Some alternative Python implementations like Jython [#jython]_ and +IronPython [#ironpython]_ do not have a global interpreter lock. +However, they do not support CPython extensions. (The implementations +can interface with code written in Java or C#). + + +PyPy-STM +-------- + +The pypy-stm [#pypystm]_ interpreter is a variant of PyPy that uses +software transactional memory. The authors report single-threaded +performance overhead in the 20%-50% range compared to PyPy. It is +not compatible with CPython extensions. + + + +Rejected Ideas +============== + +Why Not Use a Concurrent Garbage Collector? +------------------------------------------- + +Many recent garbage collectors are mostly concurrent -- they avoid long +stop-the-world pauses by allowing the garbage collector to run +concurrently with the application. So why not use a concurrent +collector? + +Concurrent collection requires write barriers (or read barriers). The +author is not aware of a way to add write barriers to CPython without +substantially breaking the C-API. + + +Why Not Deprecate ``PyDict_GetItem`` in Favor of ``PyDict_FetchItem``? +---------------------------------------------------------------------- + +This PEP proposes a new API ``PyDict_FetchItem`` which behaves like +``PyDict_GetItem``, but returns a new reference instead of a borrowed +reference. As described in `Borrowed References`_, some uses of +borrowed references that were safe when running with the GIL are +unsafe when running without the GIL and need to be replaced by +functions like ``PyDict_FetchItem`` that return new references. + +This PEP does *not* propose deprecating ``PyDict_GetItem`` and similar +functions that return borrowed references for a few reasons: + +* Many of the uses of borrowed references are safe, even when running + without the GIL. For example, C API functions often use + ``PyDict_GetItem`` to retrieve items from the keyword + argument dictionary. These calls are safe because the keyword + argument dictionary is only visible to a single thread. +* I tried this approach early on and found that wholesale replacing of + ``PyDict_GetItem`` with ``PyDict_FetchItem`` frequently introduced + new reference counting bugs. In my opinion, the risk of + introducing new reference counting bugs generally outweighs the + risks of missing a ``PyDict_GetItem`` call that is unsafe without + the GIL. + + +Why Not Use PEP 683 Immortalization? +------------------------------------ + +Like :pep:`683`, this PEP proposes an immortalization scheme for +Python objects, but the PEPs use different bit representations to +mark immortal objects. The schemes cannot be identical because this +PEP depends on biased reference counting, which has two reference +count fields instead of one. + + +Open Issues +=========== + +Improved Specialization +----------------------- + +The Python 3.11 release introduced quickening and specialization as part +of the faster CPython project, substantially improving performance. +Specialization replaces slow bytecode instructions with faster +variants [#pep659]_. To maintain thread-safety, applications that use +multiple threads (and run without the GIL) will only specialize each +bytecode once, which can lower performance on some programs. It is +possible to support specializing multiple times, but that requires more +investigation and is not part of this PEP. + + +Python Build Modes +------------------ + +This PEP introduces a new build mode (``--disable-gil``) that is not +ABI compatible with the standard build mode. The additional build +mode adds complexity for both Python core developers and extension +developers. The author believes a worthwhile goal is to combine +these build modes and have the global interpreter lock controlled at +runtime, possibly disabled by default. The path to this goal remains +an open issue, but a possible path might look like the following: + +#. In 2024, CPython 3.13 is released with support for a + ``--disable-gil`` build time flag. There are two ABIs for + CPython, one with the GIL and one without. Extension authors + target both ABIs. +#. After 2--3 releases, (i.e., in 2026--2027), CPython is released + with with the GIL controlled by a runtime environment variable or + flag. The GIL is enabled by default. There is only a single ABI. +#. After another 2--3 release (i.e., 2028--2030), CPython switches to + the GIL being disabled by default. The GIL can still be enabled + at runtime via an environment variable or command line flag. + +This PEP covers the first step, with the remaining steps left as open +issues. In this scenario, there would be a two to three year period +where extension authors would target an extra CPython build per +supported CPU architecture and OS. + +Integration +----------- + +The reference implementation changes approximately 15,000 lines of code +in CPython and includes mimalloc, which is also approximately 15,000 +lines of code. Most changes are not performance sensitive and can be +included in both ``--disable-gil`` and the default builds. Some +macros, like ``Py_BEGIN_CRITICAL_SECTION`` will be no-ops in the +default build. Thee author does not expect a huge number of ``#ifdef`` +statements to support the ``--disable-gil`` builds. + + +Mitigations for Single-Threaded Performance +------------------------------------------- + +The changes proposed in the PEP will increase execution overhead for +``--disable-gil`` builds compared to Python builds with the GIL. In +other words, it will have slower single-threaded performance. There +are some possible optimizations to reduce execution overhead, +especially for ``--disable-gil`` builds that only use a single +thread. These may be worthwhile if a longer term goal is to have a +single build mode, but the choice of optimizations and their +trade-offs remain an open issue. + + +References +========== + +.. [#yuemmwang2019] "Exploiting Parallelism Opportunities with Deep Learning Frameworks." + Yu Emma Wang, Carole-Jean Wu, Xiaodong Wang, Kim Hazelwood, David Brooks. 2019. + https://arxiv.org/abs/1908.04705. + +.. [#torchdeploy] "Using Python for Model Inference in Deep Learning." + Zachary DeVito, Jason Ansel, Will Constable, Michael Suo, Ailing Zhang, Kim Hazelwood. 2021. + https://arxiv.org/abs/2104.00254. See Figure 5. + +.. [#brc] "Biased reference counting: minimizing atomic operations in garbage collection". + Jiho Choi, Thomas Shull, and Josep Torrellas. PACT 2018. + https://dl.acm.org/doi/abs/10.1145/3243176.3243195. + +.. [#pep683] :pep:`683` -- Immortal Objects, Using a Fixed Refcount. + +.. [#tid] https://github.com/colesbury/nogil/blob/f7e45d6bfbbd48c8d5cf851c116b73b85add9fc6/Include/object.h#L428-L455. + +.. [#rcu] "What is RCU, Fundamentally?" + Paul E. McKenney, Jonathan Walpole. 2017. + https://lwn.net/Articles/262464/ + +.. [#heaps] There are two heaps for Python objects because PyObjects + that support cyclic garbage collection have extra fields preceding + the PyObject struct. + +.. [#gus] "Global Unbounded Sequences (GUS)" + https://github.com/freebsd/freebsd-src/blob/9408f36627b74a472dc82f7a43320235c0c9055a/sys/kern/subr_smr.c#L44. + See also https://people.kernel.org/joelfernandes/gus-vs-rcu. + +.. [#perfbook] "Is Parallel Programming Hard, And, If So, What Can You Do About It?" + Paul E. McKenney. 2022. + https://mirrors.edge.kernel.org/pub/linux/kernel/people/paulmck/perfbook/perfbook.html. + +.. [#typesafe_rcu] ``SLAB_TYPESAFE_BY_RCU`` is an example in which RCU + protects blocks of memory and not any individual object. See + https://www.kernel.org/doc/html/latest/RCU/whatisRCU.html#analogy-with-reference-counting. + +.. [#hotspotgc] "HotSpot Virtual Machine Garbage Collection Tuning Guide." + https://docs.oracle.com/en/java/javase/12/gctuning/hotspot-virtual-machine-garbage-collection-tuning-guide.pdf. + Most of the hotspot garbage collectors are generational, with the + notable exception of ZGC, although there is ongoing work to make + that generational. + +.. [#decapo] `The DaCapo Benchmarks: Java Benchmarking Development and + Analysis + `_. + See column "Nursery Survival" in Table 4. + +.. [#exploitingmemoryjava] "Exploiting memory usage patterns to improve garbage collections in Java." + https://dl.acm.org/doi/abs/10.1145/1852761.1852768. + +.. [#cpythongc] "most things usually turn out to be reachable" + https://github.com/python/cpython/blob/cd6655a8589e99ae4088b3bed4a692a19ed48779/Modules/gcmodule.c#L1106. + +.. [#golangc] The Go team observed something similar in Go, but due to + escape analysis and pass-by-value instead of reference + counting. Recent versions of Go use a non-generational garbage + collector. https://go.dev/blog/ismmkeynote. + +.. [#nogil] https://github.com/colesbury/nogil. + +.. [#nogil312] https://github.com/colesbury/nogil-3.12. + +.. [#howto] Python HOWTOs. + https://docs.python.org/3/howto/index.html. + +.. [#pep659] :pep:`659` -- Specializing Adaptive Interpreter. + +.. [#gilectomy] Gilectomy. + Larry Hastings. 2016. + https://github.com/larryhastings/gilectomy/tree/gilectomy. + +.. [#pyparallel] PyParallel. + Trent Nelson. 2016. + http://pyparallel.org/. + +.. [#pythonsafethread] python-safethread. + Adam Olsen. 2008. + https://launchpad.net/python-safethread + +.. [#gsteinpatch] https://www.python.org/ftp/python/contrib-09-Dec-1999/System/threading.tar.gz. + +.. [#dabeaz] An Inside Look at the GIL Removal Patch of Lore. + David Beazley. 2011. + https://dabeaz.blogspot.com/2011/08/inside-look-at-gil-removal-patch-of.html. + +.. [#jython] Jython. + https://www.jython.org/ + +.. [#ironpython] IronPython. + https://ironpython.net/ + +.. [#pypystm] PyPy: Software Transactional Memory. + https://doc.pypy.org/en/latest/stm.html + + + +Acknowledgments +=============== + +Thanks to Hugh Leather, Ɓukasz Langa, and Eric Snow for providing +feedback on drafts of this PEP. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0704.rst b/peps/pep-0704.rst new file mode 100644 index 000000000..18cb4f342 --- /dev/null +++ b/peps/pep-0704.rst @@ -0,0 +1,180 @@ +PEP: 704 +Title: Require virtual environments by default for package installers +Author: Pradyun Gedam +Sponsor: Brett Cannon +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/22846 +Status: Withdrawn +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 16-Jan-2023 +Post-History: `16-Jan-2023 `__ + + +Abstract +======== + +This PEP recommends that package installers like ``pip`` require a virtual environment by default on Python 3.13+. + +PEP Withdrawal +============== + +During discussion of this PEP, it became clear that changes to pip's UX are not controlled by PEPs as proposed. It also became clear that a significant number of users rely on being able to mix managed-by-pip dependencies with managed-by-some-other-tool dependencies (most prominently, when using Conda). + +Further, a significant subset of the benefits of the proposed change are achievable via :pep:`668` (accepted and implemented at the time of writing). It provides redistributors of the Python interpreter control over whether users should be required to use a virtual environment, what the messaging presented to the user is and how the rollout of that change should happen for their users. + +Since enforcement of virtual environments with ``pip`` was the primary focus of this PEP, it felt more appropriate to withdraw this PEP than to refocus it on a different topic. + +A future PEP to resolve the virtual environment naming convention question/issues would still be appropriate, but it's worth starting any such effort as a fresh PEP focused on the benefits of such a convention, rather than on the enforcement of it. + +Motivation +========== + +Python virtual environments are an essential part of the development workflow for Python. However, they require extra effort since they are an opt-in feature, and requires users to either: + +- take explicit steps to activate/deactivate a virtual environment +- use :samp:`{}/{}/{}` to run files + +For new users, things will seemingly work correctly when virtual environments are not used -— until they don't. Further, activating a virtual environment uses slightly different syntax and mechanisms on different platforms. This complicates the introduction to virtual environments, since now information and context about how/why they are useful needs to explained to justify adding additional steps to the workflow. + +It also creates a scope for mistakes, since users need to remember to activate the virtual environment before running an installer like ``pip`` or configure those installers to error out. On certain Linux distributions, forgetting to do so can result in the installer modifying files that are owned by the operating system (which is partially mitigated by :pep:`668` for distributions that opt-in to marking their environments accordingly). + + +Rationale +========= + +Changing the default behaviour of installers like ``pip`` to require a virtual environment to be active would: + +- make it easier for new users to get started with Python (since there's a consistent experience and virtual environments are understood as a thing you're required to use) +- reduce the scope for accidental installation issues for all users by default (by explicitly flagging when you're not using a virtual environment). + +Setting up a convention of placing the virtual environment in-tree in a directory named ``.venv`` removes a decision point for common workflows and creates a clear convention within the ecosystem. + +Specification +============= + +Requiring a virtual environment by default +------------------------------------------ + +When a user runs an installer without an active virtual environment, the installer SHOULD print an error message and exit with a non-zero exit code. + +The error message SHOULD inform the user that a virtual environment is required, SHOULD provide shell-specific instructions on how to create and activate a virtual environment named ``.venv``, and SHOULD provide a link to a documentation page that explains how to create and activate a virtual environment. + +See `Implementation Notes`_ for more details. + +Opting out of virtual environments +---------------------------------- + +The installer SHOULD also provide a explicit opt-in to disable this requirement, allowing an end user to use it outside of a virtual environment. If the installer does not provide this functionality, it SHOULD mention this in the error message and documentation page. + +Consistent timeline for the change +---------------------------------- + +Installers MAY choose to implement this default behaviour on any Python versions, but SHOULD implement it on Python 3.13 or newer. + + +Backwards Compatibility +======================= + +This PEP is backwards incompatible with workflows where users are using installers outside of virtual environments. Such users will be prompted with an error message and will need to either: + +- explicitly opt-in to running the installer outside of a virtual environment, or +- create and use a virtual environment + +Users who are already using virtual environments will not be affected by this change. + +Workflow tools (which manage virtual environments for the user, under the hood) should be unaffected, since they should already be using a virtual environment for running the installer. + + +Security Implications +===================== + +This PEP does not introduce any new security implications. + + +How to Teach This +================= + +This PEP requires that new users create and use a virtual environment to get started with using Python packages. This is, however, a best practice, as `demonstrated `__ by the section on "basics of how to install Python packages" in the Python Packaging User Guide, which explains how/what virtual environments are before discussing using ``pip``. + + +Reference Implementation +======================== + +There is no reference implementation for this PEP. However, the proposed behaviour is largely already implemented in ``pip`` and can be activated by setting the ``PIP_REQUIRE_VENV`` environment variable to ``1``. (Leaving it unset results in the proposed opt-in behaviour of not requiring a virtual environment for installation.) + + +Implementation Notes +==================== + +Detecting an active virtual environment +--------------------------------------- + +:pep:`As discussed in PEP 668 <668#backwards-compatibility>`, the logic for robustly detecting a virtual environment is something like:: + + def is_virtual_environment(): + return sys.base_prefix != sys.prefix or hasattr(sys, "real_prefix") + +Documentation on using a virtual environment +-------------------------------------------- + +Package installers are expected to provide a link to a documentation page in the error message. + +Ideally, such a documentation page would explain what virtual environments are, why they are required, and how to create and activate a virtual environment using ``venv``. It should include instructions for the most common shells and platforms. + +Such a documentation page should be made available in the `Python Packaging User Guide `__ to reduce duplicated effort across installers for covering this topic. + +Rejected Ideas +============== + +Do not specify a name for the virtual environment directory +----------------------------------------------------------- + +Using a consistent name for the virtual environment directory is important for a few reasons: + +1. It makes it easier for users to find the virtual environment directory, and to activate it. +2. It removes a decision point for new users, since they do not need to decide on a name for the virtual environment directory. +3. It creates a clear convention within the ecosystem, which makes it easier for users to find documentation. +4. It ensures consistency across different tools, so that differences in the error messages do not confuse users. + +Use a different name for the virtual environment directory +---------------------------------------------------------- + +Functionally, the directory name does not matter much as long as there is a single consistent suggestion. + +The name ``.venv`` was picked since it: + +1. does not conflict with any valid Python import name +2. does not conflict ``venv`` module in the standard library +3. has pre-existing usage in the Python community +4. has support for auto-detection in common text editors +5. can be typed without modifier keys on common keyboard layouts + +Do not couple tooling behaviour with a Python version +----------------------------------------------------- + +This PEP creates a coupling between the behaviour of installers and the Python version. + +This is already a rollout mechanism being used for behaviour changes in the installation tooling. For example, ``pip`` on Python 3.11 will use ``importlib.metadata`` instead of ``pkg_resources`` for parsing/fetching package metadata, and ``sysconfig`` instead of ``distutils.sysconfig`` for getting the paths to unpack wheels into. + +The difference with those cases is that they're supposed to be largely transparent to end users. This PEP is proposing a behaviour change that is not transparent to end users, and requires them to take action. + +The primary benefit of this is that it allows for redistributors to adapt their tooling in time for the new Python version and provides a clear and consistent point for change across the ecosystem. It also puts a clear deadline on when the default behaviour will consistently require a virtual environment by default (once Python 3.12 goes end-of-life). + +The primary issue with this approach is that it enforces a behaviour change on users when they upgrade to a new Python version, which can hamper the adoption of a new Python version. However, this is a migration/upgrade for existing users and it is a common expectation that *some* changes will be needed for migration/upgrades. + +The author of this PEP believes that the benefits of applying this consistently throughout the ecosystem with a deadline outweigh the drawbacks of enforcing a best-practice on users when they upgrade. + + +Open Issues +=========== + +None. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0705.rst b/peps/pep-0705.rst new file mode 100644 index 000000000..f62a8305c --- /dev/null +++ b/peps/pep-0705.rst @@ -0,0 +1,295 @@ +PEP: 705 +Title: TypedMapping: Type Hints for Mappings with a Fixed Set of Keys +Author: Alice Purcell +Sponsor: Pablo Galindo +Discussions-To: https://discuss.python.org/t/pep-705-typedmapping/24827 +Status: Draft +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 07-Nov-2022 +Python-Version: 3.12 +Post-History: `30-Sep-2022 `__, + `02-Nov-2022 `__, + `14-Mar-2023 `__, + + +Abstract +======== + +:pep:`589` defines the structural type :class:`~typing.TypedDict` for dictionaries with a fixed set of keys. +As ``TypedDict`` is a mutable type, it is difficult to correctly annotate methods which accept read-only parameters in a way that doesn't prevent valid inputs. +This PEP proposes a type constructor ``typing.TypedMapping`` to support this use case. + +Motivation +========== + +Representing structured data using (potentially nested) dictionaries with string keys is a common pattern in Python programs. :pep:`589` allows these values to be type checked when the exact type is known up-front, but it is hard to write read-only code that accepts more specific variants: for instance, where fields may be subtypes or restrict a union of possible types. This is an especially common issue when writing APIs for services, which may support a wide range of input structures, and typically do not need to modify their input. + +For illustration, we will try to add type hints to a function ``movie_string``:: + + def movie_string(movie: Movie) -> str: + if movie.get("year") is None: + return movie["name"] + else: + return f'{movie["name"]} ({movie["year"]})' + +We could define this ``Movie`` type using a ``TypedDict``:: + + from typing import NotRequired, TypedDict + + class Movie(TypedDict): + name: str + year: NotRequired[int | None] + +But suppose we have another type where year is required:: + + class MovieRecord(TypedDict): + name: str + year: int + +Attempting to pass a ``MovieRecord`` into ``movie_string`` results in the error (using mypy): + +.. code-block:: text + + Argument 1 to "movie_string" has incompatible type "MovieRecord"; expected "Movie" + +This particular use case should be type-safe, but the type checker correctly stops the +user from passing a ``MovieRecord`` into a ``Movie`` parameter in the general case, because +the ``Movie`` class has mutator methods that could potentially allow the function to break +the type constraints in ``MovieRecord`` (e.g. with ``movie["year"] = None`` or ``del movie["year"]``). +The problem disappears if we don't have mutator methods in ``Movie``. This could be achieved by defining an immutable interface using a :pep:`544` :class:`~typing.Protocol`:: + + from typing import Literal, Protocol, overload + + class Movie(Protocol): + @overload + def get(self, key: Literal["name"]) -> str: ... + + @overload + def get(self, key: Literal["year"]) -> int | None: ... + + @overload + def __getitem__(self, key: Literal["name"]) -> str: ... + + @overload + def __getitem__(self, key: Literal["year"]) -> int | None: ... + +This is very repetitive, easy to get wrong, and is still missing important method definitions like ``__contains__()`` and ``keys()``. + +Rationale +========= + +The proposed ``TypedMapping`` type allows a straightforward way of defining these types that should be familiar to existing users of ``TypedDict`` and support the cases exemplified above:: + + from typing import NotRequired, TypedMapping + + class Movie(TypedMapping): + name: str + year: NotRequired[int | None] + +In addition to those benefits, by flagging arguments of a function as ``TypedMapping``, it makes explicit not just to typecheckers but also to users that the function is not going to modify its inputs, which is usually a desirable property of a function interface. +Finally, this allows bringing the benefits of ``TypedDict`` to other mapping types that are unrelated to ``dict``. + +Specification +============= + +A ``TypedMapping`` type defines a protocol with the same methods as :class:`~collections.abc.Mapping`, but with value types determined per-key as with ``TypedDict``. + +Notable similarities to ``TypedDict``: + +* A ``TypedMapping`` protocol can be declared using class-based or alternative syntax. +* Keys must be strings. +* By default, all specified keys must be present in a ``TypedMapping`` instance. It is possible to override this by specifying totality, or by using ``NotRequired`` from :pep:`655`. +* Methods are not allowed in the declaration (though they may be inherited). + +Notable differences from ``TypedDict``: + +* The runtime type of a ``TypedMapping`` object is not constrained to be a ``dict``. +* No mutator methods (``__setitem__``, ``__delitem__``, ``update``, etc.) will be generated. +* The ``|`` operator is not supported. +* A class definition defines a ``TypedMapping`` protocol if and only if ``TypedMapping`` appears directly in its class bases. +* Subclasses can narrow value types, in the same manner as other protocols. + +As with :pep:`589`, this PEP provides a sketch of how a type checker is expected to support type checking operations involving ``TypedMapping`` and ``TypedDict`` objects, but details are left to implementors. In particular, type compatibility should be based on structural compatibility. + + +Multiple inheritance and TypedDict +---------------------------------- + +A type that inherits from a ``TypedMapping`` protocol and from ``TypedDict`` (either directly or indirectly): + +* is the structural intersection of its parents, or invalid if no such intersection exists +* instances must be a dict subclass +* adds mutator methods only for fields it explicitly (re)declares + +For example:: + + class Movie(TypedMapping): + name: str + year: int | None + + class MovieRecord(Movie, TypedDict): + year: int + + movie: MovieRecord = { "name": "Blade Runner", + "year": 1982 } + + movie["year"] = 1985 # Fine; mutator methods added in definition + movie["name"] = "Terminator" # Type check error; "name" mutator not declared + +Inheriting, directly or indirectly, from both ``TypedDict`` and ``Protocol`` will continue to fail at runtime, and should continue to be rejected by type checkers. + + +Multiple inheritance and Protocol +--------------------------------- + +* A type that inherits from a ``TypedMapping`` protocol and from a ``Protocol`` protocol must satisfy the protocols defined by both, but is not itself a protocol unless it inherits directly from ``TypedMapping`` or ``Protocol``. +* A type that inherits from a ``TypedMapping`` protocol and from ``Protocol`` itself is configured as a ``Protocol``. Methods and properties may be defined; keys may not:: + + class A(Movie, Protocol): + # Declare a mutable property called 'year' + # This does not affect the dictionary key 'year' + year: str + +* A type that inherits from a ``Protocol`` protocol and from ``TypedMapping`` itself is configured as a ``TypedMapping``. Keys may be defined; methods and properties may not:: + + class B(A, TypedMapping): + # Declare a key 'year' + # This does not affect the property 'year' + year: int + + +Type consistency rules +---------------------- + +Informally speaking, *type consistency* is a generalization of the is-subtype-of relation to support the ``Any`` type. It is defined more formally in :pep:`483`. This section introduces the new, non-trivial rules needed to support type consistency for ``TypedMapping`` types. + +First, any ``TypedMapping`` type is consistent with ``Mapping[str, object]``. +Second, a ``TypedMapping`` or ``TypedDict`` type ``A`` is consistent with ``TypedMapping`` ``B`` if ``A`` is structurally compatible with ``B``. This is true if and only if both of these conditions are satisfied: + +* For each key in ``A``, ``B`` has the corresponding key and the corresponding value type in ``B`` is consistent with the value type in ``A``. + +* For each required key in ``A``, the corresponding key is required in ``B``. + +Discussion: + +* Value types behave covariantly, since ``TypedMapping`` objects have no mutator methods. This is similar to container types such as ``Mapping``, and different from relationships between two ``TypedDict`` types. Example:: + + class A(TypedMapping): + x: int | None + + class B(TypedDict): + x: int + + def f(a: A) -> None: + print(a['x'] or 0) + + b: B = {'x': 0} + f(b) # Accepted by type checker + +* A ``TypedDict`` or ``TypedMapping`` type with a required key is consistent with a ``TypedMapping`` type where the same key is a non-required key, again unlike relationships between two ``TypedDict`` types. Example:: + + class A(TypedMapping, total=False): + x: int + + class B(TypedDict): + x: int + + def f(a: A) -> None: + print(a.get('x', 0)) + + b: B = {'x': 0} + f(b) # Accepted by type checker + +* A ``TypedMapping`` type ``A`` with no key ``'x'`` is not consistent with a ``TypedMapping`` type with a non-required key ``'x'``, since at runtime the key ``'x'`` could be present and have an incompatible type (which may not be visible through ``A`` due to structural subtyping). This is the same as for ``TypedDict`` types. Example:: + + class A(TypedMapping, total=False): + x: int + y: int + + class B(TypedMapping, total=False): + x: int + + class C(TypedMapping, total=False): + x: int + y: str + + def f(a: A) -> None: + print(a.get('y') + 1) + + def g(b: B) -> None: + f(b) # Type check error: 'B' incompatible with 'A' + + c: C = {'x': 0, 'y': 'foo'} + g(c) # Runtime error: str + int + +* A ``TypedMapping`` with all ``int`` values is not consistent with ``Mapping[str, int]``, since there may be additional non-``int`` values not visible through the type, due to structural subtyping. This mirrors ``TypedDict``. Example:: + + class A(TypedMapping): + x: int + + class B(TypedMapping): + x: int + y: str + + def sum_values(m: Mapping[str, int]) -> int: + return sum(m.values()) + + def f(a: A) -> None: + sum_values(a) # Type check error: 'A' incompatible with Mapping[str, int] + + b: B = {'x': 0, 'y': 'foo'} + f(b) # Runtime error: int + str + + +Backwards Compatibility +======================= + +This PEP changes the rules for how ``TypedDict`` behaves (allowing subclasses to +inherit from ``TypedMapping`` protocols in a way that changes the resulting +overloads), so code that inspects ``TypedDict`` types will have to change. This +is expected to mainly affect type-checkers. + +The ``TypedMapping`` type will be added to the ``typing_extensions`` module, +enabling its use in older versions of Python. + + +Security Implications +===================== + +There are no known security consequences arising from this PEP. + + +How to Teach This +================= + +Class documentation should be added to the :mod:`typing` module's documentation, using +that for :class:`~collections.abc.Mapping`, :class:`~typing.Protocol` and +:class:`~typing.TypedDict` as examples. Suggested introductory sentence: "Base class +for read-only mapping protocol classes." + +This PEP could be added to the others listed in the :mod:`typing` module's documentation. + + +Reference Implementation +======================== + +No reference implementation exists yet. + + +Rejected Alternatives +===================== + +Several variations were considered and discarded: + +* A ``readonly`` parameter to ``TypedDict``, behaving much like ``TypedMapping`` but with the additional constraint that instances must be dictionaries at runtime. This was discarded as less flexible due to the extra constraint; additionally, the new type nicely mirrors the existing ``Mapping``/``Dict`` types. +* Inheriting from a ``TypedMapping`` subclass and ``TypedDict`` resulting in mutator methods being added for all fields, not just those actively (re)declared in the class body. Discarded as less flexible, and not matching how inheritance works in other cases for ``TypedDict`` (e.g. total=False and total=True do not affect fields not specified in the class body). +* A generic type that removes mutator methods from its parameter, e.g. ``Readonly[MovieRecord]``. This would naturally want to be defined for a wider set of types than just ``TypedDict`` subclasses, and also raises questions about whether and how it applies to nested types. We decided to keep the scope of this PEP narrower. +* Declaring methods directly on a ``TypedMapping`` class. Methods are a kind of property, but declarations on a ``TypedMapping`` class are defining keys, so mixing the two is potentially confusing. Banning methods also makes it very easy to decide whether a ``TypedDict`` subclass can mix in a protocol or not (yes if it's just ``TypedMapping`` superclasses, no if there's a ``Protocol``). + + +Copyright +========= +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0706.rst b/peps/pep-0706.rst new file mode 100644 index 000000000..5585a809b --- /dev/null +++ b/peps/pep-0706.rst @@ -0,0 +1,693 @@ +PEP: 706 +Title: Filter for tarfile.extractall +Author: Petr Viktorin +Discussions-To: https://discuss.python.org/t/23903 +Status: Final +Type: Standards Track +Content-Type: text/x-rst +Created: 09-Feb-2023 +Python-Version: 3.12 +Post-History: `25-Jan-2023 `__, + `15-Feb-2023 `__, +Resolution: https://discuss.python.org/t/23903/10 + +.. canonical-doc:: :ref:`tarfile documentation ` + + +Abstract +======== + +The extraction methods in :external+py3.11:mod:`tarfile` gain a ``filter`` argument, +which allows rejecting files or modifying metadata as the archive is extracted. +Three built-in named filters are provided, aimed at limiting features that +might be surprising or dangerous. +These can be used as-is, or serve as a base for custom filters. + +After a deprecation period, a strict (but safer) filter will become the default. + + +Motivation +========== + +The ``tar`` format is used for several use cases, many of which have different +needs. For example: + +- A backup of a UNIX workstation should faithfully preserve all kinds of + details like file permissions, symlinks to system configuration, and various + kinds of special files. +- When unpacking a data bundle, it’s much more important that the unpacking + will not have unintended consequences – like exposing a password file by + symlinking it to a public place. + +To support all its use cases, the ``tar`` format has many features. +In many cases, it's best to ignore or disallow some of them when extracting +an archive. + +Python allows extracting ``tar`` archives using +:external+py3.11:meth:`tarfile.TarFile.extractall`, whose docs warn to +*never extract archives from untrusted sources without prior inspection*. +However, it’s not clear what kind of inspection should be done. +Indeed, it’s quite tricky to do such an inspection correctly. +As a result, many people don’t bother, or do the check incorrectly, resulting in +security issues such as `CVE-2007-4559`_. + +Since :external+py3.11:mod:`tarfile` was first written, it's become more +accepted that warnings in documentation are not enough. +Whenever possible, an unsafe operation should be *explicitly requested*; +potentially dangerous operations should *look* dangerous. +However, ``TarFile.extractall`` looks benign in a code review. + +Tarfile extraction is also exposed via :external+py3.11:func:`shutil.unpack_archive`, +which allows the user to not care about the kind of archive they're +dealing with. +The API is very inviting for extracting archives without prior inspection, +even though the docs again warn against it. + +It has been argued that Python is not wrong -- it behaves exactly as +documented -- but that's beside the point. +Let's improve the situation rather than assign/avoid blame. +Python and its docs are the best place to improve things. + + +Rationale +========= + +How do we improve things? +Unfortunately, we will need to change the defaults, which implies +breaking backwards compatibility. :external+py3.11:meth:`TarFile.extractall ` +is what people reach for when they need to extract a tarball. +Its default behaviour needs to change. + +What would be the best behaviour? That depends on the use case. +So, we'll add several general “policies” to control extraction. +They are based on *use cases*, and ideally they should have straightforward +security implications: + +- Current behavior: trusting the archive. Suitable e.g. as a building block + for libraries that do the check themselves, or extracting an archive you just + made yourself. +- Unpacking a UNIX archive: roughly following GNU ``tar``, e.g. stripping + leading ``/`` from filenames. +- Unpacking a general data archive: the :external+py3.11:func:`shutil.unpack_archive` + use case, + where it's not important to preserve details specific to ``tar`` or + Unix-like filesystems. + +After a deprecation period, the last option -- the most limited +but most secure one -- will become the default. + +Even with better general defaults, users should still verify the archives +they extract, and perhaps modify some of the metadata. +Superficially, the following looks like a reasonable way to do this today: + +* Call :external+py3.11:meth:`TarFile.getmembers ` +* Verify or modify each member's :external+py3.11:class:`~tarfile.TarInfo` +* Pass the result to ``extractall``'s ``members`` + +However, there are some issues with this approach: + +- It's possible to modify ``TarInfo`` objects, but the changes to them + affect all subsequent operations on the same ``TarFile`` object. + This behavior is fine for most uses, but despite that, it would be very + surprising if ``TarFile.extractall`` did this by default. +- Calling ``getmembers`` can be expensive and it + `requires a seekable archive `__. +- When verifying members in advance, it may be necessary to track how each + member would have changed the filesystem, e.g. how symlinks are being set up. + This is hard. We can't expect users to do it. + +To solve these issues we'll: + +- Provide a supported way to “clone” and modify ``TarInfo`` objects. + A ``replace`` method, similar to :external+py3.11:func:`dataclasses.replace` + or :external+py3.11:meth:`namedtuple._replace ` + should do the trick. +- Provide a “filter” hook in ``extractall``'s loop that can modify or discard + members before they are processed. +- Require that this hook is called just before extracting each member, + so it can scan the *current* state of the disk. This will greatly simplify + the implementation of policies (both in stdlib and user code), + at the cost of not being able to do a precise “dry run”. + +The hook API will be very similar to the existing ``filter`` argument +for :external+py3.11:meth:`TarFile.add `. +We'll also name it ``filter``. +(In some cases “policy” would be a more fitting name, +but the API can be used for more than security policies.) + +The built-in policies/filters described above will be implemented using the +public filter API, so they can be used as building blocks or examples. + + +Setting a precedent +------------------- + +If and when other libraries for archive extraction, such as :external+py3.11:mod:`zipfile`, +gain similar functionality, they should mimic this API as much as it's +reasonable. + +To enable this for simple cases, the built-in filters will have string names; +e.g. users can pass ``filter='data'`` instead of a specific function that deals +with :external+py3.11:class:`~tarfile.TarInfo` objects. + +The :external+py3.11:func:`shutil.unpack_archive` function will get a +``filter`` argument, which it will pass to ``extractall``. + +Adding function-based API that would work across archive formats is +out of scope of this PEP. + + +Full disclosure & redistributor info +------------------------------------ + +The PEP author works for Red Hat, a redistributor of Python with different +security needs and support periods than CPython in general. +Such redistributors may want to carry vendor patches to: + +* Allow configuring the defaults system-wide, and +* Change the default as soon as possible, even in older Python versions. + +The proposal makes this easy to do, and it allows users to query +the settings. + + +Specification +============= + +Modifying and forgetting member metadata +---------------------------------------- + +The :external+py3.11:class:`~tarfile.TarInfo` class will gain a new method, +``replace()``, which will work similarly to ``dataclasses.replace``. +It will return a copy of the ``TarInfo`` object with attributes +replaced as specified by keyword-only arguments: + +* ``name`` +* ``mtime`` +* ``mode`` +* ``linkname`` +* ``uid`` +* ``gid`` +* ``uname`` +* ``gname`` + +Any of these, except ``name`` and ``linkname``, will be allowed to be set +to ``None``. +When ``extract`` or ``extractall`` encounters such a ``None``, it will not +set that piece of metadata. +(If ``uname`` or ``gname`` is ``None``, it will fall back to ``uid`` or ``gid`` +as if the name wasn't found.) +When ``addfile`` or ``tobuf`` encounters such a ``None``, it will raise a +``ValueError``. +When ``list`` encounters such a ``None``, it will print a placeholder string. + +The documentation will mention why the method is there: +``TarInfo`` objects retrieved from :external+py3.11:meth:`TarFile.getmembers ` +are “live”; modifying them directly will affect subsequent unrelated +operations. + + +Filters +------- + +:external+py3.11:meth:`TarFile.extract ` and +:external+py3.11:meth:`TarFile.extractall ` methods +will grow a ``filter`` keyword-only parameter, +which takes a callable that can be called as:: + + filter(/, member: TarInfo, path: str) -> TarInfo|None + +where ``member`` is the member to be extracted, and ``path`` is the path to +where the archive is extracted (i.e., it'll be the same for every member). + +When used it will be called on each member as it is extracted, +and extraction will work with the result. +If it returns ``None``, the member will be skipped. + +The function can also raise an exception. +This can, depending on ``TarFile.errorlevel``, +abort the extraction or cause the member to be skipped. + +.. note:: + + If extraction is aborted, the archive may be left partially + extracted. It is the user’s responsibility to clean up. + +We will also provide a set of defaults for common use cases. +In addition to a function, the ``filter`` argument can be one +of the following strings: + +* ``'fully_trusted'``: Current behavior: honor the metadata as is. + Should be used if the user trusts the archive completely, or implements their + own complex verification. +* ``'tar'``: Roughly follow defaults of the GNU ``tar`` command + (when run as a normal user): + + * Strip leading ``'/'`` and ``os.sep`` from filenames + * Refuse to extract files with absolute paths (after the ``/`` stripping + above, e.g. ``C:/foo`` on Windows). + * Refuse to extract files whose absolute path (after following symlinks) + would end up outside the destination. + (Note that GNU ``tar`` instead delays creating some links.) + * Clear high mode bits (setuid, setgid, sticky) and group/other write bits + (:external+py3.11:data:`S_IWGRP|S_IWOTH `). + (This is an approximation of GNU ``tar``'s default, which limits the mode + by the current ``umask`` setting.) + +* ``'data'``: Extract a "data" archive, disallowing common attack vectors + but limiting functionality. + In particular, many features specific to UNIX-style filesystems (or + equivalently, to the ``tar`` archive format) are ignored, making this a good + filter for cross-platform archives. + In addition to ``tar``: + + * Refuse to extract links (hard or soft) that link to absolute paths. + * Refuse to extract links (hard or soft) which end up linking to a path + outside of the destination. + (On systems that don't support links, ``tarfile`` will, in most cases, + fall back to creating regular files. + This proposal doesn't change that behaviour.) + * Refuse to extract device files (including pipes). + * For regular files and hard links: + + * Set the owner read and write permissions (:external+py3.11:data:`S_IRUSR|S_IWUSR `). + * Remove the group & other *executable* permission (:external+py3.11:data:`S_IXGRP|S_IXOTH `) + if the owner doesn't have it (:external+py3.11:data:`~stat.S_IXUSR`). + + * For other files (directories), ignore mode entirely (set it to ``None``). + * Ignore user and group info (set ``uid``, ``gid``, ``uname``, ``gname`` + to ``None``). + +Any other string will cause a ``ValueError``. + +The corresponding filter functions will be available as +``tarfile.fully_trusted_filter()``, ``tarfile.tar_filter()``, etc., so +they can be easily used in custom policies. + +Note that these filters never return ``None``. +Skipping members this way is a feature for user-defined filters. + +Defaults and their configuration +-------------------------------- + +:external+py3.11:class:`~tarfile.TarFile` will gain a new attribute, +``extraction_filter``, to allow configuring the default filter. +By default it will be ``None``, but users can set it to a callable +that will be used if the ``filter`` argument is missing or ``None``. + +.. note:: + + String names won't be accepted here. That would encourage code like + ``my_tarfile.extraction_filter = 'data'``. + On Python versions without this feature, this would do nothing, + silently ignoring a security-related request. + +If both the argument and attribute are ``None``: + +* In Python 3.12-3.13, a ``DeprecationWarning`` will be emitted and + extraction will use the ``'fully_trusted'`` filter. +* In Python 3.14+, it will use the ``'data'`` filter. + +Applications and system integrators may wish to change ``extraction_filter`` +of the ``TarFile`` class itself to set a global default. +When using a function, they will generally want to wrap it in ``staticmethod()`` +to prevent injection of a ``self`` argument. + +Subclasses of ``TarFile`` can also override ``extraction_filter``. + + +FilterError +----------- + +A new exception, ``FilterError``, will be added to the :external+py3.11:mod:`tarfile` +module. +It'll have several new subclasses, one for each of the refusal reasons above. +``FilterError``'s ``member`` attribute will contain the relevant ``TarInfo``. + +In the lists above, “refusing" to extract a file means that a ``FilterError`` +will be raised. +As with other extraction errors, if the ``TarFile.errorlevel`` +is 1 or more, this will abort the extraction; with ``errorlevel=0`` the error +will be logged and the member will be ignored, but extraction will continue. +Note that ``extractall()`` may leave the archive partially extracted; +it is the user's responsibility to clean up. + + +Errorlevel, and fatal/non-fatal errors +-------------------------------------- + +Currently, :external+py3.11:class:`~tarfile.TarFile` has an *errorlevel* +argument/attribute, which specifies how errors are handled: + +- With ``errorlevel=0``, documentation says that “all errors are ignored + when using :external+py3.11:meth:`~tarfile.TarFile.extract` and + :external+py3.11:meth:`~tarfile.TarFile.extractall`”. + The code only ignores *non-fatal* and *fatal* errors (see below), + so, for example, you still get ``TypeError`` if you pass ``None`` as the + destination path. +- With ``errorlevel=1`` (the default), all *non-fatal* errors are ignored. + (They may be logged to ``sys.stderr`` by setting the *debug* + argument/attribute.) + Which errors are *non-fatal* is not defined in documentation, but code treats + ``ExtractionError`` as such. Specifically, it's these issues: + + - “unable to resolve link inside archive” (raised on systems that do not + support symlinks) + - “fifo/special devices not supported by system” (not used for failures if + the system supports these, e.g. for a ``PermissionError``) + - “could not change owner/mode/modification time” + + Note that, for example, *file name too long* or *out of disk space* don't + qualify. + The *non-fatal* errors are not very likely to appear on a Unix-like system. +- With ``errorlevel=2``, all errors are raised, including *fatal* ones. + Which errors are *fatal* is, again, not defined; in practice it's + ``OSError``. + +A filter refusing to extract a member does not fit neatly into the +*fatal*/*non-fatal* categories. + +- This PEP does not change existing behavior. (Ideas for improvements are + welcome in `Discourse topic 25970 `_.) +- When a filter refuses to extract a member, the error should not pass + silently by default. + +To satisfy this, ``FilterError`` will be considered a *fatal* error, that is, +it'll be ignored only with ``errorlevel=0``. + +Users that want to ignore ``FilterError`` but not other *fatal* errors should +create a custom filter function, and call another filter in a ``try`` block. + + +Hints for further verification +------------------------------ + +Even with the proposed changes, :external+py3.11:mod:`tarfile` will not be +suited for extracting untrusted files without prior inspection. +Among other issues, the proposed policies don't prevent denial-of-service +attacks. +Users should do additional checks. + +New docs will tell users to consider: + +* extracting to a new empty directory, +* using external (e.g. OS-level) limits on disk, memory and CPU usage, +* checking filenames against an allow-list of characters (to filter out control + characters, confusables, etc.), +* checking that filenames have expected extensions (discouraging files that + execute when you “click on them”, or extension-less files like Windows + special device names), +* limiting the number of extracted files, total size of extracted data, + and size of individual files, +* checking for files that would be shadowed on case-insensitive filesystems. + +Also, the docs will note that: + +* tar files commonly contain multiple versions of the same file: later ones are + expected to overwrite earlier ones on extraction, +* ``tarfile`` does not protect against issues with “live” data, e.g. an attacker + tinkering with the destination directory while extracting (or adding) is + going on (see the `GNU tar manual `__ + for more info). + +This list is not comprehensive, but the documentation is a good place to +collect such general tips. +It can be moved into a separate document if grows too long or if it needs to +be consolidated with :external+py3.11:mod:`zipfile` or :external+py3.11:mod:`shutil` +(which is out of scope for this proposal). + + +.. _706-offset: + +TarInfo identity, and ``offset`` +-------------------------------- + +With filters that use ``replace()``, the ``TarInfo`` objects handled +by the extraction machinery will not necessarily be the same objects +as those present in ``members``. +This may affect ``TarInfo`` subclasses that override methods like +``makelink`` and rely on object identity. + +Such code can switch to comparing ``offset``, the position of the member +header inside the file. + +Note that both the overridable methods and ``offset`` are only +documented in source comments. + + +tarfile CLI +----------- + +The CLI (``python -m tarfile``) will gain a ``--filter`` option +that will take the name of one of the provided default filters. +It won't be possible to specify a custom filter function. + +If ``--filter`` is not given, the CLI will use the default filter +(``'fully_trusted'`` with a deprecation warning now, and ``'data'`` from +Python 3.14 on). + +There will be no short option. (``-f`` would be confusingly similar to +the filename option of GNU ``tar``.) + + +Other archive libraries +----------------------- + +If and when other archive libraries, such as :external+py3.11:mod:`zipfile`, +grow similar functionality, their extraction functions should use a ``filter`` +argument that takes, at least, the strings ``'fully_trusted'`` (which should +disable any security precautions) and ``'data'`` (which should avoid features +that might surprise users). + +Standardizing a function-based filter API is out of scope of this PEP. + + +Shutil +------ + +:external+py3.11:func:`shutil.unpack_archive` will gain a ``filter`` argument. +If it's given, it will be passed to the underlying extraction function. +Passing it for a ``zip`` archive will fail for now (until :external+py3.11:mod:`zipfile` +gains a ``filter`` argument, if it ever does). + +If ``filter`` is not specified (or left as ``None``), it won't be passed +on, so extracting a tarball will use the default filter +(``'fully_trusted'`` with a deprecation warning now, and ``'data'`` from +Python 3.14 on). + + +Complex filters +--------------- + +Note that some user-defined filters need, for example, +to count extracted members of do post-processing. +This requires a more complex API than a ``filter`` callable. +However, that complex API need not be exposed to ``tarfile``. +For example, with a hypothetical ``StatefulFilter`` users would write:: + + with StatefulFilter() as filter_func: + my_tar.extract(path, filter=filter_func) + +A simple ``StatefulFilter`` example will be added to the docs. + +.. note:: + + The need for stateful filters is a reason against allowing + registration of custom filter names in addition to ``'fully_trusted'``, + ``'tar'`` and ``'data'``. + With such a mechanism, API for (at least) set-up and tear-down would need + to be set in stone. + + +Backwards Compatibility +======================= + +The default behavior of :external+py3.11:meth:`TarFile.extract ` +and :external+py3.11:meth:`TarFile.extractall ` +will change, after raising ``DeprecationWarning`` for 2 releases +(shortest deprecation period allowed in Python's +:pep:`backwards compatibility policy <387>`). + +Additionally, code that relies on :external+py3.11:class:`tarfile.TarInfo` +object identity may break, see :ref:`706-offset`. + + +Backporting & Forward Compatibility +=================================== + +This feature may be backported to older versions of Python. + +In CPython, we don't add warnings to patch releases, so the default +filter should be changed to ``'fully_trusted'`` in backports. + +Other than that, *all* of the changes to ``tarfile`` should be backported, so +``hasattr(tarfile, 'data_filter')`` becomes a reliable check for all +of the new functionality. + +Note that CPython's usual policy is to avoid adding new APIs in security +backports. +This feature does not make sense without a new API +(``TarFile.extraction_filter`` and the ``filter`` argument), +so we'll make an exception. +(See `Discourse comment 23149/16 `__ +for details.) + +Here are examples of code that takes into account that ``tarfile`` may or may +not have the proposed feature. + +When copying these snippets, note that setting ``extraction_filter`` +will affect subsequent operations. + +* Fully trusted archive:: + + my_tarfile.extraction_filter = (lambda member, path: member) + my_tarfile.extractall() + +* Use the ``'data'`` filter if available, but revert to Python 3.11 behavior + (``'fully_trusted'``) if this feature is not available:: + + my_tarfile.extraction_filter = getattr(tarfile, 'data_filter', + (lambda member, path: member)) + my_tarfile.extractall() + + (This is an unsafe operation, so it should be spelled out explicitly, + ideally with a comment.) + +* Use the ``'data'`` filter; *fail* if it is not available:: + + my_tarfile.extractall(filter=tarfile.data_filter) + + or:: + + my_tarfile.extraction_filter = tarfile.data_filter + my_tarfile.extractall() + +* Use the ``'data'`` filter; *warn* if it is not available:: + + if hasattr(tarfile, 'data_filter'): + my_tarfile.extractall(filter='data') + else: + # remove this when no longer needed + warn_the_user('Extracting may be unsafe; consider updating Python') + my_tarfile.extractall() + + +Security Implications +===================== + +This proposal improves security, at the expense of backwards compatibility. +In particular, it will help users avoid `CVE-2007-4559`_. + + +How to Teach This +================= + +The API, usage notes and tips for further verification will be added to +the documentation. +These should be usable for users who are familiar with archives in general, but +not with the specifics of UNIX filesystems nor the related security issues. + + +Reference Implementation +======================== + +See `pull request #102953 `_ on GitHub. + + +Rejected Ideas +============== + +SafeTarFile +----------- + +An initial idea from Lars GustĂ€bel was to provide a separate class that +implements security checks (see `gh-65308`_). +There are two major issues with this approach: + +* The name is misleading. General archive operations can never be made “safe” + from all kinds of unwanted behavior, without impacting legitimate use cases. +* It does not solve the problem of unsafe defaults. + +However, many of the ideas behind SafeTarFile were reused in this PEP. + +Add absolute_path option to tarfile +----------------------------------- + +Issue `gh-73974`_ asks for adding an ``absolute_path`` option to extraction +methods. This would be a minimal change to formally resolve `CVE-2007-4559`_. +It doesn't go far enough to protect the unaware, nor to empower the diligent +and curious. + +Other names for the ``'tar'`` filter +------------------------------------ + +The ``'tar'`` filter exposes features specific to UNIX-like filesystems, +so it could be named ``'unix'``. +Or ``'unix-like'``, ``'nix'``, ``'*nix'``, ``'posix'``? + +Feature-wise, *tar format* and *UNIX-like filesystem* are essentially +equivalent, so ``tar`` is a good name. + + +Possible Further Work +===================== + +Adding filters to zipfile and shutil.unpack_archive +--------------------------------------------------- + +For consistency, :external+py3.11:mod:`zipfile` and +:external+py3.11:func:`shutil.unpack_archive` could gain support +for a ``filter`` argument. +However, this would require research that this PEP's author can't promise +for Python 3.12. + +Filters for ``zipfile`` would probably not help security. +Zip is used primarily for cross-platform data bundles, and correspondingly, +:external+py3.11:meth:`ZipFile.extract `'s defaults +are already similar to what a ``'data'`` filter would do. +A ``'fully_trusted'`` filter, which would *newly allow* absolute paths and +``..`` path components, might not be useful for much except +a unified ``unpack_archive`` API. + +Filters should be useful for use cases other than security, but those +would usually need custom filter functions, and those would need API that works +with both :external+py3.11:class:`~tarfile.TarInfo` and +:external+py3.11:class:`~zipfile.ZipInfo`. +That is *definitely* out of scope of this PEP. + +If only this PEP is implemented and nothing changes for ``zipfile``, +the effect for callers of ``unpack_archive`` is that the default +for *tar* files is changing from ``'fully_trusted'`` to +the more appropriate ``'data'``. +In the interim period, Python 3.12-3.13 will emit ``DeprecationWarning``. +That's annoying, but there are several ways to handle it: e.g. add a +``filter`` argument conditionally, set ``TarFile.extraction_filter`` +globally, or ignore/suppress the warning until Python 3.14. + +Also, since many calls to ``unpack_archive`` are likely to be unsafe, +there's hope that the ``DeprecationWarning`` will often turn out to be +a helpful hint to review affected code. + + +Thanks +====== + +This proposal is based on prior work and discussions by many people, +in particular Lars GustĂ€bel, Gregory P. Smith, Larry Hastings, Joachim Wagner, +Jan Matejek, Jakub Wilk, Daniel Garcia, LumĂ­r Balhar, Miro Hrončok, +and many others. + +References +========== + +.. _CVE-2007-4559: https://nvd.nist.gov/vuln/detail/CVE-2007-4559 + +.. _gh-65308: https://github.com/python/cpython/issues/65308 + +.. _gh-73974: https://github.com/python/cpython/issues/73974 + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0707.rst b/peps/pep-0707.rst new file mode 100644 index 000000000..eb1ad75b0 --- /dev/null +++ b/peps/pep-0707.rst @@ -0,0 +1,402 @@ +PEP: 707 +Title: A simplified signature for __exit__ and __aexit__ +Author: Irit Katriel +Discussions-To: https://discuss.python.org/t/24402 +Status: Rejected +Type: Standards Track +Content-Type: text/x-rst +Created: 18-Feb-2023 +Python-Version: 3.12 +Post-History: `02-Mar-2023 `__, +Resolution: https://discuss.python.org/t/pep-707-a-simplified-signature-for-exit-and-aexit/24402/46 + +Rejection Notice +================ + +`Per the SC `__: + + We discussed the PEP and have decided to reject it. Our thinking was the + magic and risk of potential breakage didn’t warrant the benefits. We are + totally supportive, though, of exploring a potential context manager v2 + API or ``__leave__``. + +Abstract +======== + +This PEP proposes to make the interpreter accept context managers whose +:meth:`~py3.11:object.__exit__` / :meth:`~py3.11:object.__aexit__` method +takes only a single exception instance, +while continuing to also support the current ``(typ, exc, tb)`` signature +for backwards compatibility. + +This proposal is part of an ongoing effort to remove the redundancy of +the 3-item exception representation from the language, a relic of earlier +Python versions which now confuses language users while adding complexity +and overhead to the interpreter. + +The proposed implementation uses introspection, which is tailored to the +requirements of this use case. The solution ensures the safety of the new +feature by supporting it only in non-ambiguous cases. In particular, any +signature that *could* accept three arguments is assumed to expect them. + +Because reliable introspection of callables is not currently possible in +Python, the solution proposed here is limited in that only the common types +of single-arg callables will be identified as such, while some of the more +esoteric ones will continue to be called with three arguments. This +imperfect solution was chosen among several imperfect alternatives in the +spirit of practicality. It is my hope that the discussion about this PEP +will explore the other options and lead us to the best way forward, which +may well be to remain with our imperfect status quo. + + +Motivation +========== + +In the past, an exception was represented in many parts of Python by a +tuple of three elements: the type of the exception, its value, and its +traceback. While there were good reasons for this design at the time, +they no longer hold because the type and traceback can now be reliably +deduced from the exception instance. Over the last few years we saw +several efforts to simplify the representation of exceptions. + +Since 3.10 in `CPython PR #70577 `_, +the :mod:`py3.11:traceback` module's functions accept either a 3-tuple +as described above, or just an exception instance as a single argument. + +Internally, the interpreter no longer represents exceptions as a triplet. +This was `removed for the handled exception in 3.11 +`_ and +`for the raised exception in 3.12 +`_. As a consequence, +several APIs that expose the triplet can now be replaced by +simpler alternatives: + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - + - Legacy API + - Alternative + * - Get handled exception (Python) + - :func:`py3.12:sys.exc_info` + - :func:`py3.12:sys.exception` + * - Get handled exception (C) + - :external+py3.12:c:func:`PyErr_GetExcInfo` + - :external+py3.12:c:func:`PyErr_GetHandledException` + * - Set handled exception (C) + - :external+py3.12:c:func:`PyErr_SetExcInfo` + - :external+py3.12:c:func:`PyErr_SetHandledException` + * - Get raised exception (C) + - :external+py3.12:c:func:`PyErr_Fetch` + - :external+py3.12:c:func:`PyErr_GetRaisedException` + * - Set raised exception (C) + - :external+py3.12:c:func:`PyErr_Restore` + - :external+py3.12:c:func:`PyErr_SetRaisedException` + * - Construct an exception instance from the 3-tuple (C) + - :external+py3.12:c:func:`PyErr_NormalizeException` + - N/A + + +The current proposal is a step in this process, and considers the way +forward for one more case in which the 3-tuple representation has +leaked to the language. The motivation for all this work is twofold. + +Simplify the implementation of the language +------------------------------------------- + +The simplification gained by reducing the interpreter's internal +representation of the handled exception to a single object was significant. +Previously, the interpreter needed to push onto/pop +from the stack three items whenever it did anything with exceptions. +This increased stack depth (adding pressure on caches and registers) and +complicated some of the bytecodes. Reducing this to one item +`removed about 100 lines of code `_ +from ``ceval.c`` (the interpreter's eval loop implementation), and it was later +followed by the removal of the ``POP_EXCEPT_AND_RERAISE`` opcode which has +become simple enough to be `replaced by generic stack manipulation instructions +`_. Micro-benchmarks showed +`a speedup of about 10% for catching and raising an exception, as well as +for creating generators +`_. +To summarize, removing this redundancy in Python's internals simplified the +interpreter and made it faster. + +The performance of invoking ``__exit__``/``__aexit__`` when leaving +a context manager can be also improved by replacing a multi-arg function +call with a single-arg one. Micro-benchmarks showed that entering and exiting +a context manager with single-arg ``__exit__`` is about 13% faster. + +Simplify the language itself +---------------------------- + +One of the reasons for the popularity of Python is its simplicity. The +:func:`py3.11:sys.exc_info` triplet is cryptic for new learners, +and the redundancy in it is confusing for those who do understand it. + +It will take multiple releases to get to a point where we can think of +deprecating ``sys.exc_info()``. However, we can relatively quickly reach a +stage where new learners do not need to know about it, or about the 3-tuple +representation, at least until they are maintaining legacy code. + +Rationale +========= + +The only reason to object today to the removal of the last remaining +appearances of the 3-tuple from the language is the concerns about +disruption that such changes can bring. The goal of this PEP is to propose +a safe, gradual and minimally disruptive way to make this change in the +case of ``__exit__``, and with this to initiate a discussion of our options +for evolving its method signature. + +In the case of the :mod:`py3.11:traceback` module's API, evolving the +functions to have a hybrid signature is relatively straightforward and +safe. The functions take one positional and two optional arguments, and +interpret them according to their types. This is safe when sentinels +are used for default values. The signatures of callbacks, which are +defined by the user's program, are harder to evolve. + +The safest option is to make the user explicitly indicate which signature +the callback is expecting, by marking it with an additional attribute or +giving it a different name. For example, we could make the interpreter +look for a ``__leave__`` method on the context manager, and call it with +a single arg if it exists (otherwise, it looks for ``__exit__`` and +continues as it does now). The introspection-based alternative proposed +here intends to make it more convenient for users to write new code, +because they can just use the single-arg version and remain unaware of +the legacy API. However, if the limitations of introspection are found +to be too severe, we should consider an explicit option. Having both +``__exit__`` and ``__leave__`` around for 5-10 years with similar +functionality is not ideal, but it is an option. + +Let us now examine the limitations of the current proposal. It identifies +2-arg python functions and ``METH_O`` C functions as having a single-arg +signature, and assumes that anything else is expecting 3 args. Obviously +it is possible to create false negatives for this heuristic (single-arg +callables that it will not identify). Context managers written in this +way won't work, they will continue to fail as they do now when their +``__exit__`` function will be called with three arguments. + +I believe that it will not be a problem in practice. First, all working +code will continue to work, so this is a limitation on new code rather +than a problem impacting existing code. Second, exotic callable types are +rarely used for ``__exit__`` and if one is needed, it can always be wrapped +by a plain vanilla method that delegates to the callable. For example, we +can write this:: + + class C: + __enter__ = lambda self: self + __exit__ = ExoticCallable() + +as follows:: + + class CM: + __enter__ = lambda self: self + _exit = ExoticCallable() + __exit__ = lambda self, exc: CM._exit(exc) + +While discussing the real-world impact of the problem in this PEP, it is +worth noting that most ``__exit__`` functions don't do anything with their +arguments. Typically, a context manager is implemented to ensure that some +cleanup actions take place upon exit. It is rarely appropriate for the +``__exit__`` function to handle exceptions raised within the context, and +they are typically allowed to propagate out of ``__exit__`` to the calling +function. This means that most ``__exit__`` functions do not access their +arguments at all, and we should take this into account when trying to +assess the impact of different solutions on Python's userbase. + + +Specification +============= + +A context manager's ``__exit__``/``__aexit__`` method can have a single-arg +signature, in which case it is invoked by the interpreter with the argument +equal to an exception instance or ``None``: + +.. code-block:: + + >>> class C: + ... def __enter__(self): + ... return self + ... def __exit__(self, exc): + ... print(f'__exit__ called with: {exc!r}') + ... + >>> with C(): + ... pass + ... + __exit__ called with: None + >>> with C(): + ... 1/0 + ... + __exit__ called with: ZeroDivisionError('division by zero') + Traceback (most recent call last): + File "", line 2, in + ZeroDivisionError: division by zero + +If ``__exit__``/``__aexit__`` has any other signature, it is invoked with +the 3-tuple ``(typ, exc, tb)`` as happens now: + +.. code-block:: + + >>> class C: + ... def __enter__(self): + ... return self + ... def __exit__(self, *exc): + ... print(f'__exit__ called with: {exc!r}') + ... + >>> with C(): + ... pass + ... + __exit__ called with: (None, None, None) + >>> with C(): + ... 1/0 + ... + __exit__ called with: (, ZeroDivisionError('division by zero'), ) + Traceback (most recent call last): + File "", line 2, in + ZeroDivisionError: division by zero + + +These ``__exit__`` methods will also be called with a 3-tuple: + +.. code-block:: + + def __exit__(self, typ, *exc): + pass + + def __exit__(self, typ, exc, tb): + pass + +A reference implementation is provided in +`CPython PR #101995 `_. + +When the interpreter reaches the end of the scope of a context manager, +and it is about to call the relevant ``__exit__`` or ``__aexit__`` function, +it instrospects this function to determine whether it is the single-arg +or the legacy 3-arg version. In the draft PR, this introspection is performed +by the ``is_legacy___exit__`` function: + +.. code-block:: c + + static int is_legacy___exit__(PyObject *exit_func) { + if (PyMethod_Check(exit_func)) { + PyObject *func = PyMethod_GET_FUNCTION(exit_func); + if (PyFunction_Check(func)) { + PyCodeObject *code = (PyCodeObject*)PyFunction_GetCode(func); + if (code->co_argcount == 2 && !(code->co_flags & CO_VARARGS)) { + /* Python method that expects self + one more arg */ + return false; + } + } + } + else if (PyCFunction_Check(exit_func)) { + if (PyCFunction_GET_FLAGS(exit_func) == METH_O) { + /* C function declared as single-arg */ + return false; + } + } + return true; + } + +It is important to note that this is not a generic introspection function, but +rather one which is specifically designed for our use case. We know that +``exit_func`` is an attribute of the context manager class (taken from the +type of the object that provided ``__enter__``), and it is typically a function. +Furthermore, for this to be useful we need to identify enough single-arg forms, +but not necessarily all of them. What is critical for backwards compatibility is +that we will never misidentify a legacy ``exit_func`` as a single-arg one. So, +for example, ``__exit__(self, *args)`` and ``__exit__(self, exc_type, *args)`` +both have the legacy form, even though they *could* be invoked with one arg. + +In summary, an ``exit_func`` will be invoke with a single arg if: + +* It is a ``PyMethod`` with ``argcount`` ``2`` (to count ``self``) and no vararg, or +* it is a ``PyCFunction`` with the ``METH_O`` flag. + +Note that any performance cost of the introspection can be mitigated via +:pep:`specialization <659>`, so it won't be a problem if we need to make it more +sophisticated than this for some reason. + + +Backwards Compatibility +======================= + +All context managers that previously worked will continue to work in the +same way because the interpreter will call them with three args whenever +they can accept three args. There may be context managers that previously +did not work because their ``exit_func`` expected one argument, so the call +to ``__exit__`` would have caused a ``TypeError`` exception to be raised, +and now the call would succeed. This could theoretically change the +behaviour of existing code, but it is unlikely to be a problem in practice. + +The backwards compatibility concerns will show up in some cases when libraries +try to migrate their context managers from the multi-arg to the single-arg +signature. If ``__exit__`` or ``__aexit__`` is called by any code other than +the interpreter's eval loop, the introspection does not automatically happen. +For example, this will occur where a context manager is subclassed and its +``__exit__`` method is called directly from the derived ``__exit__``. Such +context managers will need to migrate to the single-arg version with their +users, and may choose to offer a parallel API rather than breaking the +existing one. Alternatively, a superclass can stay with the signature +``__exit__(self, *args)``, and support both one and three args. Since +most context managers do not use the value of the arguments to ``__exit__``, +and simply allow the exception to propagate onward, this is likely to be the +common approach. + + +Security Implications +===================== + +I am not aware of any. + +How to Teach This +================= + +The language tutorial will present the single-arg version, and the documentation +for context managers will include a section on the legacy signatures of +``__exit__`` and ``__aexit__``. + + +Reference Implementation +======================== + +`CPython PR #101995 `_ +implements the proposal of this PEP. + + +Rejected Ideas +============== + +Support ``__leave__(self, exc)`` +---------------------------------- + +It was considered to support a method by a new name, such as ``__leave__``, +with the new signature. This basically makes the programmer explicitly declare +which signature they are intending to use, and avoid the need for introspection. + +Different variations of this idea include different amounts of magic that can +help automate the equivalence between ``__leave__`` and ``__exit__``. For example, +`Mark Shannon suggested `_ +that the type constructor would add a default implementation for each of ``__exit__`` +and ``__leave__`` whenever one of them is defined on a class. This default +implementation acts as a trampoline that calls the user's function. This would +make inheritance work seamlessly, as well as the migration from ``__exit__`` to +``__leave__`` for particular classes. The interpreter would just need to call +``__leave__``, and that would call ``__exit__`` whenever necessary. + +While this suggestion has several advantages over the current proposal, it has +two drawbacks. The first is that it adds a new dunder name to the data model, +and we would end up with two dunders that mean the same thing, and only slightly +differ in their signatures. The second is that it would require the migration of +every ``__exit__`` to ``__leave__``, while with introspection it would not be +necessary to change the many ``__exit__(*arg)`` methods that do not access their +args. While it is not as simple as a grep for ``__exit__``, it is possible to write +an AST visitor that detects ``__exit__`` methods that can accept multiple arguments, +and which do access them. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0708.rst b/peps/pep-0708.rst new file mode 100644 index 000000000..a9ba4eebc --- /dev/null +++ b/peps/pep-0708.rst @@ -0,0 +1,930 @@ +PEP: 708 +Title: Extending the Repository API to Mitigate Dependency Confusion Attacks +Author: Donald Stufft +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/24179 +Status: Provisional +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 20-Feb-2023 +Post-History: `01-Feb-2023 `__, + `23-Feb-2023 `__ +Resolution: https://discuss.python.org/t/24179/72 + + +Provisional Acceptance +====================== + +This PEP has been **provisionally accepted**, +with the following required conditions before the PEP is made Final: + +1. An implementation of the PEP in PyPI (Warehouse) + including any necessary UI elements + to allow project owners to set the tracking data. +2. An implementation of the PEP in at least one repository other than PyPI, + as you can’t really test merging indexes without at least two indexes. +3. An implementation of the PEP in pip, + which supports the intended semantics and can be used to demonstrate + that the expected security benefits are achieved. + This implementation will need to be "off by default" initially, + which means that users will have to opt in to testing it. + Ideally, we should collect explicit positive reports from users + (both project owners and project users) + who have successfully tried out the new feature, + rather than just assuming that "no news is good news". + + +Abstract +======== + +Dependency confusion attacks, in which a malicious package is installed instead +of the one the user expected, are an `increasingly common supply chain threat +`__. +Most such attacks against Python dependencies, including the +`recent PyTorch incident `_, +occur with multiple package repositories, where a dependency expected to come +from one repository (e.g. a custom index) is installed from another (e.g. PyPI). + +To help address this problem, this PEP proposes extending the +:ref:`Simple Repository API ` +to allow repository operators to indicate that a project found on their +repository "tracks" a project on different repositories, and allows projects to +extend their namespaces across multiple repositories. + +These features will allow installers to determine when a project being made +available from a particular mix of repositories is expected and should be +allowed, and when it is not and should halt the install with an error to protect +the user. + + +Motivation +=========== + +There is a long-standing class of attacks that are called "dependency confusion" +attacks, which roughly boil down to an individual user expected to get package +``A``, but instead they got ``B``. In Python, this almost always happens due to +the configuration of multiple repositories (possibly including the default of +PyPI), where they expected package ``A`` to come from repository ``X``, but +someone is able to publish package ``B`` to repository ``Y`` under the same +name. + +Dependency Confusion attacks have long been possible, but they've recently +gained press with +`public examples of cases where these attacks were successfully executed `__. + +A specific example of this is the recent case where the PyTorch project had an +internal package named ``torchtriton`` which was only ever intended to be +installed from their repositories located at ``https://download.pytorch.org/``, +but that repository was designed to be used in conjunction with PyPI, and +the name of ``torchtriton`` was not claimed on PyPI, which allowed the attacker +to use that name and publish a malicious version. + +There are a number of ways to mitigate against these attacks today, but they all +require that the end user go out of their way to protect themselves, rather than +being protected by default. This means that for the vast bulk of users, they are +likely to remain vulnerable, even if they are ultimately aware of these types of +attacks. + +Ultimately the underlying cause of these attacks come from the fact that there +is no globally unique namespace that all Python package names come from. +Instead, each repository is its own distinct namespace, and when given an +"abstract" name such as ``spam`` to install, an installer has to implicitly turn +that into a "concrete" name such as ``pypi.org:spam`` or ``example.com:spam``. +Currently the standard behavior in Python installation tools is to implicitly +flatten these multiple namespaces into one that contains the files from all +namespaces. + +This assumption that collapsing the namespaces is what was expected means that +when packages with the same name in different repositories +are authored by different parties (such as in the ``torchtriton`` case) +dependency confusion attacks become possible. + +This is made particularly tricky in that there is no "right" answer; there are +valid use cases both for wanting two repositories merged into one namespace +*and* for wanting two repositories to be treated as distinct namespaces. This +means that an installer needs some mechanism by which to determine when it +should merge the namespaces of multiple repositories and when it should not, +rather than a blanket always merge or never merge rule. + +This functionality could be pushed directly to the end user, since ultimately +the end user is the person whose expectations of what gets installed from what +repository actually matters. However, by extending the repository specification +to allow a repository to indicate when it is safe, we can enable individual +projects and repositories to "work by default", even when their +project naturally spans multiple distinct namespaces, while maintaining the +ability for an installer to be secure by default. + +On its own, this PEP does not solve dependency confusion attacks, but what it +does do is provide enough information so that installers can prevent them +without causing too much collateral damage to otherwise valid and safe use +cases. + + +Rationale +========= + +There are two broad use cases for merging names across repositories that this +PEP seeks to enable. + +The first use case is when one repository is not defining its own names, but +rather is extending names defined in other repositories. This commonly happens +in cases where a project is being mirrored from one repository to another (see +`Bandersnatch `__) or when a repository +is providing supplementary artifacts for a specific platform (see +`Piwheels `__). + +In this case neither the repositories nor the projects that are being extended +may have any knowledge that they are being extended or by whom, so this cannot +rely on any information that isn't present in the "extending" repository itself. + +The second use case is when the project wants to publish to one "main" +repository, but then have additional repositories that provide binaries for +additional platforms, GPUs, CPUs, etc. Currently wheel tags are not sufficiently +able to express these types of binary compatibility, so projects that wish to +rely on them are forced to set up multiple repositories and have their users +manually configure them to get the correct binaries for their platform, GPU, +CPU, etc. + +This use case is similar to the first, but the important difference that makes +it a distinct use case on it's own is who is providing the information and what +their level of trust is. + +When a user configures a specific repository (or relies on the default) there +is no ambiguity as to what repository they mean. A repository is identified by +an URL, and through the domain system, URLs are globally unique identifiers. +This lack of ambiguity means that an installer can assume that the repository +operator is trustworthy and can trust metadata that they provide without needing +to validate it. + +On the flip side, given an installer finds a name in multiple repositories it is +ambiguous which of them the installer should trust. This ambiguity means that an +installer cannot assume that the project owner on either repository is +trustworthy and needs to validate that they are indeed the same project and that +one isn't a dependency confusion attack. + +Without some way for the installer to validate the metadata between multiple +repositories, projects would be forced into becoming repository operators to +safely support this use case. That wouldn't be a particularly wrong choice to +make; however, there is a danger that if we don't provide a way for repositories +to let project owners express this relationship safely, they will be +incentivized to let them use the repository operator's metadata instead which +would reintroduce the original insecurity. + + +Specification +============= + +This specification defines the changes in version 1.2 of the simple repository +API, adding new two new metadata items: Repository "Tracks" and "Alternate +Locations". + + +Repository "Tracks" Metadata +---------------------------- + +To enable one repository to host a project that is intended to "extend" a +project that is hosted at other repositories, this PEP allows the extending +repository to declare that a particular project "tracks" a project at another +repository or repositories by adding the URLs of the project and repositories +that it is extending. + +This is exposed in JSON as the key ``meta.tracks`` and in HTML as a meta element +named ``pypi:tracks`` on the project specific URLs, (``$root/$project/``). + + +There are a few key properties that **MUST** be preserved when using this +metadata: + +- It **MUST** be under the control of the repository operators themselves, not + any individual publisher using that repository. + + - "Repository Operator" can also include anyone who managed the overall + namespace for a particular repository, which may be the case in situations + like hosted repository services where one entity operates the software but + another owns/manages the entire namespace of that repository. + +- All URLs **MUST** represent the same "project" as the project in the extending + repository. + + - This does not mean that they need to serve the same files. It is valid for + them to include binaries built on different platforms, copies with local + patches being applied, etc. This is purposefully left vague as it's + ultimately up to the expectations that the users have of the repository and + its operators what exactly constitutes the "same" project. + +- It **MUST** point to the repositories that "own" the namespaces, not another + repository that is also tracking that namespace. + +- It **MUST** point to a project with the exact same name (after normalization). + +- It **MUST** point to the actual URLs for that project, not the base URL for + the extended repositories. + +It is **NOT** required that every name in a repository tracks the same +repository, or that they all track a repository at all. Mixed use repositories +where some names track a repository and some names do not are explicitly +allowed. + + +JSON +~~~~ + +.. code-block:: JSON + + { + "meta": { + "api-version": "1.2", + "tracks": ["https://pypi.org/simple/holygrail/", "https://test.pypi.org/simple/holygrail/"] + }, + "name": "holygrail", + "files": [ + { + "filename": "holygrail-1.0.tar.gz", + "url": "https://example.com/files/holygrail-1.0.tar.gz", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "yanked": "Had a vulnerability" + }, + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "https://example.com/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "dist-info-metadata": true + } + ] + } + + +HTML +~~~~ + +.. code-block:: HTML + + + + + + + + + + + + + + + +"Alternate Locations" Metadata +------------------------------ + +To enable a project to extend its namespace across multiple repositories, this +PEP allows a project owner to declare a list of "alternate locations" for their +project. This is exposed in JSON as the key ``alternate-locations`` and in HTML +as a meta element named ``pypi-alternate-locations``, which may be used multiple +times. + +There are a few key properties that **MUST** be observed when using this +metadata: + +- In order for this metadata to be trusted, there **MUST** be agreement between + all locations where that project is found as to what the alternate locations + are. +- When using alternate locations, clients **MUST** implicitly assume that the + url the response was fetched from was included in the list. This means that + if you fetch from ``https://pypi.org/simple/foo/`` and it has an + ``alternate-locations`` metadata that has the value + ``["https://example.com/simple/foo/"]``, then you **MUST** treat it as if it + had the value + ``["https://example.com/simple/foo/", "https://pypi.org/simple/foo/"]``. +- Order of the elements within the array does not have any particular meaning. + +When an installer encounters a project that is using the alternate locations +metadata it **SHOULD** consider that all repositories named are extending the +same namespace across multiple repositories. + +.. note:: + + This alternate locations metadata is project level metadata, not artifact + level metadata, which means it doesn't get included as part of the core + metadata spec, but rather it is something that each repository will have to + provide a configuration option for (if they choose to support it). + + +JSON +~~~~ + +.. code-block:: JSON + + { + "meta": { + "api-version": "1.2" + }, + "name": "holygrail", + "alternate-locations": ["https://pypi.org/simple/holygrail/", "https://test.pypi.org/simple/holygrail/"], + "files": [ + { + "filename": "holygrail-1.0.tar.gz", + "url": "https://example.com/files/holygrail-1.0.tar.gz", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "yanked": "Had a vulnerability" + }, + { + "filename": "holygrail-1.0-py3-none-any.whl", + "url": "https://example.com/files/holygrail-1.0-py3-none-any.whl", + "hashes": {"sha256": "...", "blake2b": "..."}, + "requires-python": ">=3.7", + "dist-info-metadata": true + } + ] + } + + +HTML +~~~~ + +.. code-block:: HTML + + + + + + + + + + + + + + + +Recommendations +=============== + +This section is non-normative; it provides recommendations to installers in how +to interpret this metadata that this PEP feels provides the best tradeoff +between protecting users by default and minimizing breakages to existing +workflows. These recommendations are not binding, and installers are free to +ignore them, or apply them selectively as they make sense in their specific +situations. + + +File Discovery Algorithm +------------------------ + +.. note:: + + This algorithm is written based on how pip currently discovers files; + other installers may adapt this based on their own discovery procedures. + +Currently the "standard" file discovery algorithm looks something like this: + +1. Generate a list of all files across all configured repositories. +2. Filter out any files that do not match known hashes from a lockfile or + requirements file. +3. Filter out any files that do not match the current platform, Python version, + etc. +4. Pass that list of files into the resolver where it will attempt to resolve + the "best" match out of those files, irrespective of which repository it came + from. + +It is recommended that installers change their file discovery algorithm to take +into account the new metadata, and instead do: + +1. Generate a list of all files across all configured repositories. + +2. Filter out any files that do not match known hashes from a lockfile or + requirements file. + +3. If the end user has explicitly told the installer to fetch the project from + specific repositories, filter out all other repositories and skip to 5. + +4. Look to see if the discovered files span multiple repositories; if they do + then determine if either "Tracks" or "Alternate Locations" metadata allows + safely merging *ALL* of the repositories where files were discovered + together. If that metadata does **NOT** allow that, then generate an error, + otherwise continue. + + - **Note:** This only applies to *remote* repositories; repositories that + exist on the local filesystem **SHOULD** always be implicitly allowed to be + merged to any remote repository. + +5. Filter out any files that do not match the current platform, Python version, + etc. + +6. Pass that list of files into the resolver where it will attempt to resolve + the "best" match out of those files, irrespective of what repository it came + from. + +This is somewhat subtle, but the key things in the recommendation are: + +- Users who are using lock files or requirements files that include specific + hashes of artifacts that are "valid" are assumed to be protected by nature of + those hashes, since the rest of these recommendations would apply during + hash generation. Thus, we filter out unknown hashes up front. +- If the user has explicitly told the installer that it wants to fetch a project + from a certain set of repositories, then there is no reason to question that + and we assume that they've made sure it is safe to merge those namespaces. +- If the project in question only comes from a single repository, then there is + no chance of dependency confusion, so there's no reason to do anything but + allow. +- We check for the metadata in this PEP before filtering out based on platform, + Python version, etc., because we don't want errors that only show up on + certain platforms, Python versions, etc. +- If nothing tells us merging the namespaces is safe, we refuse to implicitly + assume it is, and generate an error instead. +- Otherwise we merge the namespaces, and continue on. + +This algorithm ensures that an installer never assumes that two disparate +namespaces can be flattened into one, which for all practical purposes +eliminates the possibility of any kind of dependency confusion attack, while +still giving power throughout the stack in a safe way to allow people to +explicitly declare when those disparate namespaces are actually one logical +namespace that can be safely merged. + +The above algorithm is mostly a conceptual model. In reality the algorithm may +end up being slightly different in order to be more privacy preserving and +faster, or even just adapted to fit a specific installer better. + + +Explicit Configuration for End Users +------------------------------------ + +This PEP avoids dictating or recommending a specific mechanism by which an +installer allows an end user to configure exactly what repositories they want a +specific package to be installed from. However, it does recommend that +installers do provide *some* mechanism for end users to provide that +configuration, as without it users can end up in a DoS situation in cases +like ``torchtriton`` where they're just completely broken unless they resolve +the namespace collision externally (get the name taken down on one repository, +stand up a personal repository that handles the merging, etc). + +This configuration also allows end users to pre-emptively secure themselves +during what is likely to be a long transition until the default behavior is +safe. + + +How to Communicate This +======================= + +.. note:: + + This example is pip specific and assumes specifics about how pip will + choose to implement this PEP; it's included as an example of how we can + communicate this change, and not intended to constrain pip or any other + installer in how they implement this. This may ultimately be the actual basis + for communication, and if so will need be edited for accuracy and clarity. + + This section should be read as if it were an entire "post" to communicate this + change that could be used for a blog post, email, or discourse post. + +There's a long-standing class of attacks that are called "dependency confusion" +attacks, which roughly boil down to an individual expected to get package ``A``, +but instead they got ``B``. In Python, this almost always happens due to the end +user having configured multiple repositories, where they expect package ``A`` to +come from repository ``X``, but someone is able to publish package ``B`` with +the same name as package ``A`` in repository ``Y``. + +There are a number of ways to mitigate against these attacks today, but they all +require that the end user explicitly go out of their way to protect themselves, +rather than it being inherently safe. + +In an effort to secure pip's users and protect them from these types of attacks, +we will be changing how pip discovers packages to install. + + +What is Changing? +----------------- + +When pip discovers that the same project is available from multiple remote +repositories, by default it will generate an error and refuse to proceed rather +than make a guess about which repository was the correct one to install from. + +Projects that natively publish to multiple repositories will be given the +ability to safely "link" their repositories together so that pip does not error +when those repositories are used together. + +End users of pip will be given the ability to explicitly define one or more +repositories that are valid for a specific project, causing pip to only consider +those repositories for that project, and avoiding generating an error +altogether. + +See TBD for more information. + + +Who is Affected? +---------------- + +Users who are installing from multiple remote (e.g. not present on the local +filesystem) repositories may be affected by having pip error instead of +successfully install if: + +- They install a project where the same "name" is being served by multiple + remote repositories. +- The project name that is available from multiple remote repositories has not + used one of the defined mechanisms to link those repositories together. +- The user invoking pip has not used the defined mechanism to explicitly control + what repositories are valid for a particular project. + +Users who are not using multiple remote repositories will not be affected at +all, which includes users who are only using a single remote repository, plus a +local filesystem "wheel house". + + +What do I need to do? +--------------------- + +As a pip User? +~~~~~~~~~~~~~~ + +If you're using only a single remote repository you do not have to do anything. + +If you're using multiple remote repositories, you can opt into the new behavior +by adding ``--use-feature=TBD`` to your pip invocation to see if any of your +dependencies are being served from multiple remote repositories. If they are, +you should audit them to determine why they are, and what the best remediation +step will be for you. + +Once this behavior becomes the default, you can opt out of it temporarily by +adding ``--use-deprecated=TBD`` to your pip invocation. + +If you're using projects that are not hosted on a public repository, but you +still have the public repository as a fallback, consider configuring pip with a +repository file to be explicit where that dependency is meant to come from to +prevent registration of that name in a public repository to cause pip to error +for you. + + +As a Project Owner? +~~~~~~~~~~~~~~~~~~~ + +If you only publish your project to a single repository, then you do not have to +do anything. + +If you publish your project to multiple repositories that are intended to be +used together at the same time, configure all repositories to serve the +alternate repository metadata to prevent breakages for your end users. + +If you publish your project to a single repository, but it is commonly used in +conjunction with other repositories, consider preemptively registering your +names with those repositories to prevent a third party from being able to cause +your users ``pip install`` invocations to start failing. This may not be +available if your project name is too generic or if the repositories have +policies that prevent defensive name squatting. + + +As a Repository Operator? +~~~~~~~~~~~~~~~~~~~~~~~~~ + +You'll need to decide how you intend for your repository to be used by your end +users and how you want them to use it. + +For private repositories that host private projects, it is recommended that you +mirror the public projects that your users depend on into your own repository, +taking care not to let a public project merge with a private project, and tell +your users to use the ``--index-url`` option to use only your repository. + +For public repositories that host public projects, you should implement the +alternate repository mechanism and enable the owners of those projects to +configure the list of repositories that their project is available from if they +make it available from more than one repository. + +For public repositories that "track" another repository, but provide +supplemental artifacts such as wheels built for a specific platform, you should +implement the "tracks" metadata for your repository. However, this information +**MUST NOT** be settable by end users who are publishing projects to your +repository. See TBD for more information. + + +Rejected Ideas +============== + +*Note: Some of these are somewhat specific to pip, but any solution that doesn't +work for pip isn't a particularly useful solution.* + + +Implicitly allow mirrors when the list of files are the same +------------------------------------------------------------ + +If every repository returns the exact same list of files, then it is safe to +consider those repositories to be the same namespace and implicitly merge them. +This would possibly mean that mirrors would be automatically allowed without any +work on any user or repository operator's part. + +Unfortunately, this has two failings that make it undesirable: + +- It only solves the case of mirrors that are exact copies of each other, but + not repositories that "track" another one, which ends up being a more generic + solution. +- Even in the case of exact mirrors, multiple repositories mirroring each other + is a distributed system will not always be fully consistent with each + other, effectively an eventually consistent system. This means that + repositories that relied on this implicit heuristic to work would have + sporadic failures due to drift between the source repository and the mirror + repositories. + + +Provide a mechanism to order the repositories +--------------------------------------------- + +Providing some mechanism to give the repositories an order, and then short +circuiting the discovery algorithm when it finds the first repository that +provides files for that project is another workable solution that is safe if the +order is specified correctly. + +However, this has been rejected for a number of reasons: + +- We've spent 15+ years educating users that the ordering of repositories being + specified is not meaningful, and they effectively have an undefined order. It + would be difficult to backpedal on that and start saying that now order + matters. +- Users can easily rearrange the order that they specify their repositories in + within a single location, but when loading repositories from multiple + locations (env var, conf file, requirements file, cli arguments) the order is + hard coded into pip. While it would be a deterministic and documented order, + there's no reason to assume it's the order that the user wants their + repositories to be defined in, forcing them to contort how they configure pip + so that the implicit ordering ends up being the correct one. +- The above can be mitigated by providing a way to explicitly declare the order + rather than by implicitly using the order they were defined in; however, that + then means that the protections are not provided unless the user does some + explicit configuration. +- Ordering assumes that one repository is *always* preferred over another + repository without any way to decide on a project by project basis. +- Relying on ordering is subtle; if I look at an ordering of repositories, I + have no way of knowing or ensuring in advance what names are going + to come from what repositories. I can only know in that moment what names are + provided by which repositories. +- Relying on ordering is fragile. There's no reason to assume that two disparate + repositories are not going to have random naming collisions—what happens if + I'm using a library from a lower priority repository and then a higher + priority repository happens to start having a colliding name? +- In cases where ordering does the wrong thing, it does so silently, with no + feedback given to the user. This is by design because it doesn't actually know + what the wrong or right thing is, it's just hoping that order will give the + right thing, and if it does then users are protected without any breakage. + However, when it does the wrong thing, users are left with a very confusing + behavior coming from pip, where it's just silently installing the wrong thing. + +There is a variant of this idea which effectively says that it's really just +PyPI's nature of open registration that causes the real problems, so if we treat +all repositories but the "default" one as equal priority, and then treat the +default one as a lower priority then we'll fix things. + +That is true in that it does improve things, but it has many of the same +problems as the general ordering idea (though not all of them). + +It also assumes that PyPI, or whatever repository is configured as the +"default", is the only repository with open registration of names. +However, projects like `Piwheels `_ exist +which users are expected to use in addition to PyPI, +which also effectively have open registration of names +since it tracks whatever names are registered on PyPI. + + +Rely on repository proxies +-------------------------- + +One possible solution is to instead of having the installer have to solve this, +to instead depend on repository proxies that can intelligently merge multiple +repositories safely. This could provide a better experience for people with +complex needs because they can have configuration and features that are +dedicated to the problem space. + +However, that has been rejected because: + +- It requires users to opt into using them, unless we also remove the facilities + to have more than one repository in installers to force users into using a + repository proxy when they need multiple repositories. + + - Removing facilities to have more than one repository configured has been + rejected because it would be too disruptive to end users. + +- A user may need different outcomes of merging multiple repositories in + different contexts, or may need to merge different, mutually exclusive + repositories. This means they'll need to actually set up multiple repository + proxies for each unique set of options. + +- It requires users to maintain infrastructure or it requires adding features in + installers to automatically spin up a repository for each invocation. + +- It doesn't actually change the requirement to need to have a solution to these + problems, it just shifts the responsibility of implementation from installers + to some repository proxy, but in either case we still need something that + figures out how to merge these disparate namespaces. + +- Ultimately, most users do not want to have to stand up a repository proxy just + to safely interact with multiple repositories. + + +Rely only on hash checking +-------------------------- + +Another possible solution is to rely on hash checking, since with hash checking +enabled users cannot get an artifact that they didn't expect; it doesn't matter +if the namespaces are incorrectly merged or not. + +This is certainly a solution; unfortunately it also suffers from problems that +make it unworkable: + +- It requires users to opt in to it, so users are still unprotected by default. +- It requires users to do a bunch of labor to manage their hashes, which is + something that most users are unlikely to be willing to do. +- It is difficult and verbose to get the protection when users are not using a + ``requirements.txt`` file as the source of their dependencies (this affects + build time dependencies, and dependencies provided at the command line). +- It only sort of solves the problem, in a way it just shifts the responsibility + of the problem to be whatever system is generating the hashes that the + installer would use. If that system isn't a human manually validating hashes, + which it's unlikely it would be, then we've just shifted the question of how + to merge these namespaces to whatever tool implements the maintenance of the + hashes. + + +Require all projects to exist in the "default" repository +--------------------------------------------------------- + +Another idea is that we can narrow the scope of ``--extra-index-url`` such that +its only supported use is to refer to supplemental repositories to the default +repository, effectively saying that the default repository defines the +namespace, and every additional repository just extends it with extra packages. + +The implementation of this would roughly be to require that the project **MUST** +be registered with the default repository in order for any additional +repositories to work. + +This sort of works if you successfully narrow the scope in that way, but +ultimately it has been rejected because: + +- Users are unlikely to understand or accept this reduced scope, and thus are + likely to attempt to continue to use it in the now unsupported fashion. + + - This is complicated by the fact that with the scope now narrowed, users who + have the excluded workflow no longer have any alternative besides setting up + a repository proxy, which takes infrastructure and effort that they + previously didn't have to do. + +- It assumes that just because a name in an "extra" repository is the same as in + the default repository, that they are the same project. If we were starting + from scratch in a brand new ecosystem then maybe we could make this assumption + from the start and make it stick, but it's going to be incredibly difficult to + get the ecosystem to adjust to that change. + + - This is a fundamental issue with this approach; the underlying problem that + drives dependency confusion is that we're taking disparate namespaces and + flattening them into one. This approach essentially just declares that OK, + and attempts to mitigate it by requiring everyone to register their names. + +- Because of the above assumption, in cases where a name in an extra repository + collides by accident with the default repository, it's going to appear to work + for those users, but they are going to be silently in a state of dependency + confusion. + + - This is made worse by the fact that the person who owns the name that is + allowing this to work is going to be completely unaware of the role that + they're playing for that user, and might possibly delete their project or + hand it off to someone else, potentially allowing them to inadvertently + allow a malicious user to take it over. + +- Users are likely to attempt to get back to a working state by registering + their names in their default repository as a defensive name squat. Their + ability to do this will depend on the specific policies of their default + repository, whether someone already has that name, whether it's too generic, + etc. As a best case scenario it will cause needless placeholder projects that + serve no purpose other than to secure some internal use of a name. + + +Move to Globally Unique Names +----------------------------- + +The main reason this problem exists is that we don't have globally unique names, +we have locally unique names that exist under multiple namespaces that we are +attempting to merge into a single flat namespace. If we could instead come up +with a way to have globally unique names, we could sidestep the entire issue. + +This idea has been rejected because: + +- Generating globally unique but secure names that are also meaningful to humans + is a nearly impossible feat without piggybacking off of some kind of + centralized database. To my knowledge the only systems that have managed to do + this end up piggybacking off of the domain system and refer to packages by + URLs with domains etc. +- Even if we come up with a mechanism to get globally unique names, our ability + to retrofit that into our decades old system is practically zero without + burning it all to the ground and starting over. The best we could probably do + is declare that all non globally unique names are implicitly names on the PyPI + domain name, and force everyone with a non PyPI package to rename their + package. +- This would upend so many core assumptions and fundamental parts of our current + system it's hard to even know where to start to list them. + + +Only recommend that installers offer explicit configuration +----------------------------------------------------------- + +One idea that has come up is to essentially just implement the explicit +configuration and don't make any other changes to anything else. The specific +proposal for a mapping policy is what actually inspired the explicit +configuration option, and created a file that looked something like: + +.. code-block:: JSON + + { + "repositories": { + "PyTorch": ["https://download.pytorch.org/whl/nightly"], + "PyPI": ["https://pypi.org/simple"] + }, + "mapping": [ + { + "paths": ["torch*"], + "repositories": ["PyTorch"], + "terminating": true + }, + { + "paths": ["*"], + "repositories": ["PyPI"] + } + ] + } + +The recommendation to have explicit configuration pushes the decision on how to +implement that onto each installer, allowing them to choose what works best for +their users. + +Ultimately only implementing some kind of explicit configuration was rejected +because by its nature it's opt in, so it doesn't protect average users who are +least capable to solve the problem with the existing tools; by adding additional +protections alongside the explicit configuration, we are able to protect all +users by default. + +Additionally, relying on only explicit configuration also means that every end +user has to resolve the same problem over and over again, even in cases like +mirrors of PyPI, Piwheels, PyTorch, etc. In each and every case they have to sit +there and make decisions (or find some example to cargo cult) in order to be +secure. Adding extra features into the mix allows us to centralize those +protections where we can, while still giving advanced end users the ability to +completely control their own destiny. + + +Scopes Ă  la npm +--------------- + +There's been some suggestion that +`scopes similar to how npm has implemented them `__ +may ultimately solve this. Ultimately scopes do not change anything about this +problem. As far as I know scopes in npm are not globally unique, they're tied to +a specific registry just like unscoped names are. However what scopes do enable +is an obvious mechanism for grouping related projects and the ability for a user +or organization on npm.org to claim an entire scope, which makes explicit +configuration significantly easier to handle because you can be assured that +there's a whole little slice of the namespace that wholly belongs to you, and +you can easily write a rule that assigns an entire scope to a specific non +public registry. + +Unfortunately, it basically ends up being an easier version of the idea to only +use explicit configuration, which works ok in npm because its not particularly +common for people to use their own registries, but in Python we encourage you to +do just that. + + +Define and Standardize the "Explicit Configuration" +--------------------------------------------------- + +This PEP recommends installers to have a mechanism for explicit configuration of +which repository a particular project comes from, but it does not define what +that mechanism is. We are purposefully leave that undefined, as it is closely +tied to the UX of each individual installer and we want to allow each individual +installer the ability to expose that configuration in whatever way that they see +fit for their particular use cases. + +Further, when the idea of defining that mechanism came up, none of the other +installers seemed particularly interested in having that mechanism defined for +them, suggesting that they were happy to treat that as part of their UX. + +Finally, that mechanism, if we did choose to define it, deserves it's own PEP +rather than baking it as part of the changes to the repository API in this PEP +and it can be a future PEP if we ultimately decide we do want to go down the +path of standardization for it. + + +Acknowledgements +================ + +Thanks to Trishank Kuppusamy for kick starting the discussion that lead to this +PEP with his `proposal `__. + +Thanks to Paul Moore, Pradyun Gedam, Steve Dower, and Trishank Kuppusamy for +providing early feedback and discussion on the ideas in this PEP. + +Thanks to Jelle Zijlstra, C.A.M. Gerlach, Hugo van Kemenade, and Stefano Rivera +for copy editing and improving the structure and quality of this PEP. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0709.rst b/peps/pep-0709.rst new file mode 100644 index 000000000..926afefef --- /dev/null +++ b/peps/pep-0709.rst @@ -0,0 +1,333 @@ +PEP: 709 +Title: Inlined comprehensions +Author: Carl Meyer +Sponsor: Guido van Rossum +Discussions-To: https://discuss.python.org/t/pep-709-inlined-comprehensions/24240 +Status: Accepted +Type: Standards Track +Content-Type: text/x-rst +Created: 24-Feb-2023 +Python-Version: 3.12 +Post-History: `25-Feb-2023 `__ +Resolution: https://discuss.python.org/t/pep-709-inlined-comprehensions/24240/36 + + +Abstract +======== + +Comprehensions are currently compiled as nested functions, which provides +isolation of the comprehension's iteration variable, but is inefficient at +runtime. This PEP proposes to inline list, dictionary, and set comprehensions +into the code where they are defined, and provide the expected isolation by +pushing/popping clashing locals on the stack. This change makes comprehensions +much faster: up to 2x faster for a microbenchmark of a comprehension alone, +translating to an 11% speedup for one sample benchmark derived from real-world +code that makes heavy use of comprehensions in the context of doing actual work. + + +Motivation +========== + +Comprehensions are a popular and widely-used feature of the Python language. +The nested-function compilation of comprehensions optimizes for compiler +simplicity at the expense of performance of user code. It is possible to +provide near-identical semantics (see `Backwards Compatibility`_) with much +better runtime performance for all users of comprehensions, with only a small +increase in compiler complexity. + + +Rationale +========= + +Inlining is a common compiler optimization in many languages. Generalized +inlining of function calls at compile time in Python is near-impossible, since +call targets may be patched at runtime. Comprehensions are a special case, +where we have a call target known statically in the compiler that can neither +be patched (barring undocumented and unsupported fiddling with bytecode +directly) nor escape. + +Inlining also permits other compiler optimizations of bytecode to be more +effective, because they can now "see through" the comprehension bytecode, +instead of it being an opaque call. + +Normally a performance improvement would not require a PEP. In this case, the +simplest and most efficient implementation results in some user-visible effects, +so this is not just a performance improvement, it is a (small) change to the +language. + + +Specification +============= + +Given a simple comprehension:: + + def f(lst): + return [x for x in lst] + +The compiler currently emits the following bytecode for the function ``f``: + +.. code-block:: text + + 1 0 RESUME 0 + + 2 2 LOAD_CONST 1 ( at 0x...) + 4 MAKE_FUNCTION 0 + 6 LOAD_FAST 0 (lst) + 8 GET_ITER + 10 CALL 0 + 20 RETURN_VALUE + + Disassembly of at 0x...>: + 2 0 RESUME 0 + 2 BUILD_LIST 0 + 4 LOAD_FAST 0 (.0) + >> 6 FOR_ITER 4 (to 18) + 10 STORE_FAST 1 (x) + 12 LOAD_FAST 1 (x) + 14 LIST_APPEND 2 + 16 JUMP_BACKWARD 6 (to 6) + >> 18 END_FOR + 20 RETURN_VALUE + +The bytecode for the comprehension is in a separate code object. Each time +``f()`` is called, a new single-use function object is allocated (by +``MAKE_FUNCTION``), called (allocating and then destroying a new frame on the +Python stack), and then immediately thrown away. + +Under this PEP, the compiler will emit the following bytecode for ``f()`` +instead: + +.. code-block:: text + + 1 0 RESUME 0 + + 2 2 LOAD_FAST 0 (lst) + 4 GET_ITER + 6 LOAD_FAST_AND_CLEAR 1 (x) + 8 SWAP 2 + 10 BUILD_LIST 0 + 12 SWAP 2 + >> 14 FOR_ITER 4 (to 26) + 18 STORE_FAST 1 (x) + 20 LOAD_FAST 1 (x) + 22 LIST_APPEND 2 + 24 JUMP_BACKWARD 6 (to 14) + >> 26 END_FOR + 28 SWAP 2 + 30 STORE_FAST 1 (x) + 32 RETURN_VALUE + +There is no longer a separate code object, nor creation of a single-use function +object, nor any need to create and destroy a Python frame. + +Isolation of the ``x`` iteration variable is achieved by the combination of the +new ``LOAD_FAST_AND_CLEAR`` opcode at offset ``6``, which saves any outer value +of ``x`` on the stack before running the comprehension, and ``30 STORE_FAST``, +which restores the outer value of ``x`` (if any) after running the +comprehension. + +If the comprehension accesses variables from the outer scope, inlining avoids +the need to place these variables in a cell, allowing the comprehension (and all +other code in the outer function) to access them as normal fast locals instead. +This provides further performance gains. + +In some cases, the comprehension iteration variable may be a global or cellvar +or freevar, rather than a simple function local, in the outer scope. In these +cases, the compiler also internally pushes and pops the scope information for +the variable when entering/leaving the comprehension, so that semantics are +maintained. For example, if the variable is a global outside the comprehension, +``LOAD_GLOBAL`` will still be used where it is referenced outside the +comprehension, but ``LOAD_FAST`` / ``STORE_FAST`` will be used within the +comprehension. If it is a cellvar/freevar outside the comprehension, the +``LOAD_FAST_AND_CLEAR`` / ``STORE_FAST`` used to save/restore it do not change +(there is no ``LOAD_DEREF_AND_CLEAR``), meaning that the entire cell (not just +the value within it) is saved/restored, so the comprehension does not write to +the outer cell. + +Comprehensions occurring in module or class scope are also inlined. In this +case, the comprehension will introduce usage of fast-locals (``LOAD_FAST`` / +``STORE_FAST``) for the comprehension iteration variable within the +comprehension only, in a scope where otherwise only ``LOAD_NAME`` / +``STORE_NAME`` would be used, maintaining isolation. + +In effect, comprehensions introduce a sub-scope where local variables are fully +isolated, but without the performance cost or stack frame entry of a call. + +Generator expressions are currently not inlined in the reference implementation +of this PEP. In the future, some generator expressions may be inlined, where the +returned generator object does not leak. + +Asynchronous comprehensions are inlined the same as synchronous ones; no special +handling is needed. + + +Backwards Compatibility +======================= + +Comprehension inlining will cause the following visible behavior changes. No +changes in the standard library or test suite were necessary to adapt to these +changes in the implementation, suggesting the impact in user code is likely to +be minimal. + +Specialized tools depending on undocumented details of compiler bytecode output +may of course be affected in ways beyond the below, but these tools already must +adapt to bytecode changes in each Python version. + +locals() includes outer variables +--------------------------------- + +Calling ``locals()`` within a comprehension will include all locals of the +function containing the comprehension. E.g. given the following function:: + + def f(lst): + return [locals() for x in lst] + +Calling ``f([1])`` in current Python will return:: + + [{'.0': , 'x': 1}] + +where ``.0`` is an internal implementation detail: the synthetic sole argument +to the comprehension "function". + +Under this PEP, it will instead return:: + + [{'lst': [1], 'x': 1}] + +This now includes the outer ``lst`` variable as a local, and eliminates the +synthetic ``.0``. + +No comprehension frame in tracebacks +------------------------------------ + +Under this PEP, a comprehension will no longer have its own dedicated frame in +a stack trace. For example, given this function:: + + def g(): + raise RuntimeError("boom") + + def f(): + return [g() for x in [1]] + +Currently, calling ``f()`` results in the following traceback: + +.. code-block:: text + + Traceback (most recent call last): + File "", line 1, in + File "", line 5, in f + File "", line 5, in + File "", line 2, in g + RuntimeError: boom + +Note the dedicated frame for ````. + +Under this PEP, the traceback looks like this instead: + +.. code-block:: text + + Traceback (most recent call last): + File "", line 1, in + File "", line 5, in f + File "", line 2, in g + RuntimeError: boom + +There is no longer an extra frame for the list comprehension. The frame for the +``f`` function has the correct line number for the comprehension, however, so +this simply makes the traceback more compact without losing any useful +information. + +It is theoretically possible that code using warnings with the ``stacklevel`` +argument could observe a behavior change due to the frame stack change. In +practice, however, this seems unlikely. It would require a warning raised in +library code that is always called through a comprehension in that same +library, where the warning is using a ``stacklevel`` of 3+ to bypass the +comprehension and its containing function and point to a calling frame outside +the library. In such a scenario it would usually be simpler and more reliable +to raise the warning closer to the calling code and bypass fewer frames. + +Tracing/profiling will no longer show a call/return for the comprehension +------------------------------------------------------------------------- + +Naturally, since list/dict/set comprehensions will no longer be implemented as a +call to a nested function, tracing/profiling using ``sys.settrace`` or +``sys.setprofile`` will also no longer reflect that a call and return have +occurred. + + +Impact on other Python implementations +====================================== + +Per comments from representatives of `GraalPython +`_ and +`PyPy `_, +they would likely feel the need to adapt to the observable behavior changes +here, given the likelihood that someone, at some point, will depend on them. +Thus, all else equal, fewer observable changes would be less work. But these +changes (at least in the case of GraalPython) should be manageable "without much +headache". + + +How to Teach This +================= + +It is not intuitively obvious that comprehension syntax will or should result +in creation and call of a nested function. For new users not already accustomed +to the prior behavior, I suspect the new behavior in this PEP will be more +intuitive and require less explanation. ("Why is there a ```` line in +my traceback when I didn't define any such function? What is this ``.0`` +variable I see in ``locals()``?") + + +Security Implications +===================== + +None known. + + +Reference Implementation +======================== + +This PEP has a reference implementation in the form of `a PR against the CPython main +branch `_ which passes all tests. + +The reference implementation performs the micro-benchmark ``./python -m pyperf +timeit -s 'l = [1]' '[x for x in l]'`` 1.96x faster than the ``main`` branch (in a +build compiled with ``--enable-optimizations``.) + +The reference implementation performs the ``comprehensions`` benchmark in the +`pyperformance `_ benchmark suite +(which is not a micro-benchmark of comprehensions alone, but tests +real-world-derived code doing realistic work using comprehensions) 11% faster +than ``main`` branch (again in optimized builds). Other benchmarks in +pyperformance (none of which use comprehensions heavily) don't show any impact +outside the noise. + +The implementation has no impact on non-comprehension code. + + +Rejected Ideas +============== + +More efficient comprehension calling, without inlining +------------------------------------------------------ + +An `alternate approach `_ +introduces a new opcode for "calling" a comprehension in streamlined fashion +without the need to create a throwaway function object, but still creating a new +Python frame. This avoids all of the visible effects listed under `Backwards +Compatibility`_, and provides roughly half of the performance benefit (1.5x +improvement on the microbenchmark, 4% improvement on ``comprehensions`` +benchmark in pyperformance.) It also requires adding a new pointer to the +``_PyInterpreterFrame`` struct and a new ``Py_INCREF`` on each frame +construction, meaning (unlike this PEP) it has a (very small) performance cost +for all code. It also provides less scope for future optimizations. + +This PEP takes the position that full inlining offers sufficient additional +performance to more than justify the behavior changes. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0710.rst b/peps/pep-0710.rst new file mode 100644 index 000000000..e5c826931 --- /dev/null +++ b/peps/pep-0710.rst @@ -0,0 +1,629 @@ +PEP: 710 +Title: Recording the provenance of installed packages +Author: FridolĂ­n PokornĂœ +Sponsor: Donald Stufft +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/pep-710-recording-the-provenance-of-installed-packages/25428 +Status: Draft +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 27-Mar-2023 +Post-History: `03-Dec-2021 `__, + `30-Jan-2023 `__, + `14-Mar-2023 `__, + `03-Apr-2023 `__, + +Abstract +======== + +This PEP describes a way to record the provenance of installed Python distributions. +The record is created by an installer and is available to users in +the form of a JSON file ``provenance_url.json`` in the ``.dist-info`` directory. +The mentioned JSON file captures additional metadata to allow recording a URL to a +:term:`distribution package` together with the installed distribution hash. This +proposal is built on top of :pep:`610` following +:ref:`its corresponding canonical PyPA spec ` and +complements ``direct_url.json`` with ``provenance_url.json`` for when packages +are identified by a name, and optionally a version. + +Motivation +========== + +Installing a Python :term:`Project` involves downloading a :term:`Distribution Package` +from a :term:`Package Index` +and extracting its content to an appropriate place. After the installation +process is done, information about the release artifact used as well as its source +is generally lost. However, there are use cases for keeping records of +distributions used for installing packages and their provenance. + +Python wheels can be built with different compiler flags or supporting +different wheel tags. In both cases, users might get into a situation in which +multiple wheels might be considered by installers (possibly from different +package indexes) and immediately finding out which wheel file was actually used +during the installation might be helpful. This way, developers can use +information about wheels to debug issues making sure the desired wheel was +actually installed. Another use case could be tools reporting software +installed, such as tools reporting a SBOM (Software Bill of Materials), that might +give more accurate reports. Yet another use case could be reconstruction of the +Python environment by pinning each installed package to a specific distribution +artifact consumed from a Python package index. + +Rationale +========= + +The motivation described in this PEP is an extension of that in :pep:`610`. +In addition to recording provenance information for packages installed using a direct URL, +installers should also do so for packages installed by name +(and optionally version) from Python package indexes. + +The idea described in this PEP originated in a tool called `micropipenv`_ +that is used to install +:term:`distribution packages ` in containerized +environments (see the reported issue `thoth-station/micropipenv#206`_). +Currently, the assembled containerized application does not implicitly carry +information about the provenance of installed distribution packages +(unless these are installed from full URLs and recorded via ``direct_url.json``). +This requires container image suppliers to link +container images with the corresponding build process, its configuration and +the application source code for checking requirements files in cases when +software present in containerized environments needs to be audited. + +The `subsequent discussion in the Discourse thread +`__ also brought up +pip's new ``--report`` option that can +`generate a detailed JSON report `__ about +the installation process. This option could help with the provenance problem +this PEP approaches. Nevertheless, this option needs to be *explicitly* passed +to pip to obtain the provenance information, and includes additional metadata that +might not be necessary for checking the provenance (such as Python version +requirements of each distribution package). Also, this option is +specific to pip as of the writing of this PEP. + +Note the current :ref:`spec for recording installed packages +` defines a ``RECORD`` file that +records installed files, but not the distribution artifact from which these +files were obtained. Auditing installed artifacts can be performed +based on matching the entries listed in the ``RECORD`` file. However, this +technique requires a pre-computed database of files each artifact provides or a +comparison with the actual artifact content. Both approaches are relatively +expensive and time consuming operations which could be eliminated with the +proposed ``provenance_url.json`` file. + +Recording provenance information for installed distribution packages, +both those obtained from direct URLs and by name/version from an index, +can simplify auditing Python environments in general, beyond just +the specific use case for containerized applications mentioned earlier. +A community project `pip-audit +`__ raised their possible interest in +`pypa/pip-audit#170`_. + +Specification +============= + +The keywords “MUST”, “MUST NOT”, “REQUIRED”, “SHOULD”, +“SHOULD NOT”, “RECOMMENDED”, “MAY”, and “OPTIONAL” +in this document are to be interpreted as described in :rfc:`2119`. + +The ``provenance_url.json`` file SHOULD be created in the ``.dist-info`` +directory by installers when installing a :term:`Distribution Package` +specified by name (and optionally by :term:`Version Specifier`). + +This file MUST NOT be created when installing a distribution package from a requirement +specifying a direct URL reference (including a VCS URL). + +Only one of the files ``provenance_url.json`` and ``direct_url.json`` (from :pep:`610`), +may be present in a given ``.dist-info`` directory; installers MUST NOT add both. + +The ``provenance_url.json`` JSON file MUST be a dictionary, compliant with +:rfc:`8259` and UTF-8 encoded. + +If present, it MUST contain exactly two keys. The first one is ``url``, with +type ``string``. The second key MUST be ``archive_info`` with a value defined +below. + +The value of the ``url`` key MUST be the URL from which the distribution package was downloaded. If a wheel is +built from a source distribution, the ``url`` value MUST be the URL from which +the source distribution was downloaded. If a wheel is downloaded and installed directly, +the ``url`` field MUST be the URL from which the wheel was downloaded. +As in the :ref:`direct URL origin specification`, the ``url`` value +MUST be stripped of any sensitive authentication information for security reasons. + +The user:password section of the URL MAY however be composed of environment +variables, matching the following regular expression: + +.. code-block:: text + + \$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})? + +Additionally, the user:password section of the URL MAY be a well-known, +non-security sensitive string. A typical example is ``git`` in the case of an +URL such as ``ssh://git@gitlab.com``. + +The value of ``archive_info`` MUST be a dictionary with a single key +``hashes``. The value of ``hashes`` is a dictionary mapping hash function names to a +hex-encoded digest of the file referenced by the ``url`` value. Multiple hashes +can be included, and it is up to the consumer to decide what to do with +multiple hashes (it may validate all of them or a subset of them, or nothing at +all). + +Each hash MUST be one of the single argument hashes provided by +:data:`py3.11:hashlib.algorithms_guaranteed`, excluding ``sha1`` and ``md5`` which MUST NOT be used. +As of Python 3.11, with ``shake_128`` and ``shake_256`` excluded +for being multi-argument, the allowed set of hashes is: + +.. code-block:: python + + >>> import hashlib + >>> sorted(hashlib.algorithms_guaranteed - {"shake_128", "shake_256", "sha1", "md5"}) + ['blake2b', 'blake2s', 'sha224', 'sha256', 'sha384', 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', 'sha512'] + +Each hash MUST be referenced by the canonical name of the hash, always lower case. + +Hashes ``sha1`` and ``md5`` MUST NOT be present, due to the security +limitations of these hash algorithms. Conversely, hash ``sha256`` SHOULD +be included. + +Installers that cache distribution packages from an index SHOULD keep +information related to the cached distribution artifact, so that +the ``provenance_url.json`` file can be created even when installing distribution packages +from the installer's cache. + +Backwards Compatibility +======================= + +Following the :ref:`packaging:recording-installed-packages` specification, +installers may keep additional installer-specific files in the ``.dist-info`` +directory. To make sure this PEP does not cause any backwards compatibility +issues, a `comprehensive survey of installers and libraries <710-tool-survey_>`_ +found no current tools that are using a similarly-named file, +or other major feasibility concerns. + +The :ref:`Wheel specification ` lists files that can be +present in the ``.dist-info`` directory. None of these file names collide with +the proposed ``provenance_url.json`` file from this PEP. + +Presence of provenance_url.json in installers and libraries +----------------------------------------------------------- + +A comprehensive survey of the existing installers, libraries, and dependency +managers in the Python ecosystem analyzed the implications of adding support for +``provenance_url.json`` to each tool. +In summary, no major backwards compatibility issues, conflicts or feasibility blockers +were found as of the time of writing of this PEP. More details about the survey +can be found in the `Appendix: Survey of installers and libraries`_ section. + +Compatibility with direct_url.json +---------------------------------- + +This proposal does not make any changes to the ``direct_url.json`` file +described in :pep:`610` and :ref:`its corresponding canonical PyPA spec +`. + +The content of ``provenance_url.json`` file was designed in a way to eventually +allow installers reuse some of the logic supporting ``direct_url.json`` when a +direct URL refers to a source archive or a wheel. + +The main difference between the ``provenance_url.json`` and ``direct_url.json`` +files are the mandatory keys and their values in the ``provenance_url.json`` file. +This helps make sure consumers of the ``provenance_url.json`` file can rely +on its content, if the file is present in the ``.dist-info`` directory. + +Security Implications +===================== + +One of the main security features of the ``provenance_url.json`` file is the +ability to audit installed artifacts in Python environments. Tools can check +which Python package indexes were used to install Python :term:`distribution +packages ` as well as the hash digests of their release +artifacts. + +As an example, we can take the recent compromised dependency chain in `the +PyTorch incident `__. +The PyTorch index provided a package named ``torchtriton``. An attacker +published ``torchtriton`` on PyPI, which ran a malicious binary. By checking +the URL of the installed Python distribution stated in the +``provenance_url.json`` file, tools can automatically check the source of the +installed Python distribution. In case of the PyTorch incident, the URL of +``torchtriton`` should point to the PyTorch index, not PyPI. Tools can help +identifying such malicious Python distributions installed by checking the +installed Python distribution URL. A more exact check can include also the hash +of the installed Python distribution stated in the ``provenance_url.json`` +file. Such checks on hashes can be helpful for mirrored Python package indexes +where Python distributions are not distinguishable by their source URLs, making +sure only desired Python package distributions are installed. + +A malicious actor can intentionally adjust the content of +``provenance_url.json`` to possibly hide provenance information of the +installed Python distribution. A security check which would uncover such +malicious activity is beyond scope of this PEP as it would require monitoring +actions on the filesystem and eventually reviewing user or file permissions. + +How to Teach This +================= + +The ``provenance_url.json`` metadata file is intended for tools and is not +directly visible to end users. + +Examples +======== + +Examples of a valid provenance_url.json +--------------------------------------- + +A valid ``provenance_url.json`` list multiple hashes: + +.. code-block:: json + + { + "archive_info": { + "hashes": { + "blake2s": "fffeaf3d0bd71dc960ca2113af890a2f2198f2466f8cd58ce4b77c1fc54601ff", + "sha256": "236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f", + "sha3_256": "c856930e0f707266d30e5b48c667a843d45e79bb30473c464e92dfa158285eab", + "sha512": "6bad5536c30a0b2d5905318a1592948929fbac9baf3bcf2e7faeaf90f445f82bc2b656d0a89070d8a6a9395761f4793c83187bd640c64b2656a112b5be41f73d" + } + }, + "url": "https://files.pythonhosted.org/packages/07/51/2c0959c5adf988c44d9e1e0d940f5b074516ecc87e96b1af25f59de9ba38/pip-23.0.1-py3-none-any.whl" + } + +A valid ``provenance_url.json`` listing a single hash entry: + +.. code-block:: json + + { + "archive_info": { + "hashes": { + "sha256": "236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f" + } + }, + "url": "https://files.pythonhosted.org/packages/07/51/2c0959c5adf988c44d9e1e0d940f5b074516ecc87e96b1af25f59de9ba38/pip-23.0.1-py3-none-any.whl" + } + +A valid ``provenance_url.json`` listing a source distribution which was used to +build and install a wheel: + +.. code-block:: json + + { + "archive_info": { + "hashes": { + "sha256": "8bfe29f17c10e2f2e619de8033a07a224058d96b3bfe2ed61777596f7ffd7fa9" + } + }, + "url": "https://files.pythonhosted.org/packages/1d/43/ad8ae671de795ec2eafd86515ef9842ab68455009d864c058d0c3dcf680d/micropipenv-0.0.1.tar.gz" + } + +Examples of an invalid provenance_url.json +------------------------------------------ + +The following example includes a ``hash`` key in the ``archive_info`` dictionary +as originally designed in :pep:`610` and the data structure documented in +:ref:`packaging:direct-url`. +The ``hash`` key MUST NOT be present to prevent from any possible confusion +with ``hashes`` and additional checks that would be required to keep hash +values in sync. + +.. code-block:: json + + { + "archive_info": { + "hash": "sha256=236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f", + "hashes": { + "sha256": "236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f" + } + }, + "url": "https://files.pythonhosted.org/packages/07/51/2c0959c5adf988c44d9e1e0d940f5b074516ecc87e96b1af25f59de9ba38/pip-23.0.1-py3-none-any.whl" + } + +Another example demonstrates an invalid hash name. The referenced hash name does not +correspond to the canonical hash names described in this PEP and +in the Python docs under :attr:`py3.11:hashlib.hash.name`. + +.. code-block:: json + + { + "archive_info": { + "hashes": { + "SHA-256": "236bcb61156d76c4b8a05821b988c7b8c35bf0da28a4b614e8d6ab5212c25c6f" + } + }, + "url": "https://files.pythonhosted.org/packages/07/51/2c0959c5adf988c44d9e1e0d940f5b074516ecc87e96b1af25f59de9ba38/pip-23.0.1-py3-none-any.whl" + } + + +Example pip commands and their effect on provenance_url.json and direct_url.json +-------------------------------------------------------------------------------- + +These commands generate a ``direct_url.json`` file but do not generate a +``provenance_url.json`` file. These examples follow examples from :pep:`610`: + +* ``pip install https://example.com/app-1.0.tgz`` +* ``pip install https://example.com/app-1.0.whl`` +* ``pip install "git+https://example.com/repo/app.git#egg=app&subdirectory=setup"`` +* ``pip install ./app`` +* ``pip install file:///home/user/app`` +* ``pip install --editable "git+https://example.com/repo/app.git#egg=app&subdirectory=setup"`` (in which case, ``url`` will be the local directory where the git repository has been cloned to, and ``dir_info`` will be present with ``"editable": true`` and no ``vcs_info`` will be set) +* ``pip install -e ./app`` + +Commands that generate a ``provenance_url.json`` file but do not generate +a ``direct_url.json`` file: + +* ``pip install app`` +* ``pip install app~=2.2.0`` +* ``pip install app --no-index --find-links "https://example.com/"`` + +This behaviour can be tested using changes to pip implemented in the PR +`pypa/pip#11865`_. + +Reference Implementation +======================== + +A proof-of-concept for creating the ``provenance_url.json`` metadata file when +installing a Python :term:`Distribution Package` is available in the PR to pip +`pypa/pip#11865`_. It reuses the already available implementation for the +:ref:`direct URL data structure ` to provide +the ``provenance_url.json`` metadata file for cases when ``direct_url.json`` is not +created. + +A prototype called `pip-preserve `_ was developed to +demonstrate creation of ``requirements.txt`` files considering ``direct_url.json`` +and ``provenance_url.json`` metadata files. This tool mimics the ``pip +freeze`` functionality, but the listing of installed packages also includes +the hashes of the Python distribution artifacts. + +To further support this proposal, `pip-sbom `_ demonstrates creation +of SBOM in the SPDX format. The tool uses information stored in the ``provenance_url.json`` +file. + +Rejected Ideas +============== + +Naming the file direct_url.json instead of provenance_url.json +-------------------------------------------------------------- + +To preserve backwards compatibility with the +:ref:`Direct URL Origin specification `, +the file cannot be named ``direct_url.json``, as per the text of that specification: + + This file MUST NOT be created when installing a distribution from an other + type of requirement (i.e. name plus version specifier). + +Such a change might introduce backwards compatibility issues for consumers of +``direct_url.json`` who rely on its presence only when distributions are +installed using a direct URL reference. + +Deprecating direct_url.json and using only provenance_url.json +-------------------------------------------------------------- + +File ``direct_url.json`` is already well established with :pep:`610` being accepted and is +already used by installers. For example, ``pip`` uses ``direct_url.json`` to +report a direct URL reference on ``pip freeze``. Deprecating +``direct_url.json`` would require additional changes to the ``pip freeze`` +implementation in pip (see PR `fridex/pip#2`_) and could introduce backwards compatibility +issues for already existing ``direct_url.json`` consumers. + +Keeping the hash key in the archive_info dictionary +--------------------------------------------------- + +:pep:`610` and :ref:`its corresponding canonical PyPA spec ` +discuss the possibility to include the ``hash`` key alongside the ``hashes`` key in the +``archive_info`` dictionary. This PEP explicitly does not include the ``hash`` key in +the ``provenance_url.json`` file and allows only the ``hashes`` key to be present. +By doing so we eliminate possible redundancy in the file, possible confusion, +and any additional checks that would need to be done to make sure the hashes are in +sync. + +Making the hashes key optional +------------------------------ + +:pep:`610` and :ref:`its corresponding canonical PyPA spec ` +recommend including the ``hashes`` key of the ``archive_info`` in the +``direct_url.json`` file but it is not required (per the :rfc:`2119` language): + + A hashes key SHOULD be present as a dictionary mapping a hash name to a hex + encoded digest of the file. + +This PEP requires the ``hashes`` key be included in ``archive_info`` +in the ``provenance_url.json`` file if that file is created; per this PEP: + + The value of ``archive_info`` MUST be a dictionary with a single key + ``hashes``. + +By doing so, consumers of ``provenance_url.json`` can check +artifact digests when the ``provenance_url.json`` file is created by installers. + +Open Issues +=========== + +Availability of the provenance_url.json file in Conda +----------------------------------------------------- + +We would like to get feedback on the ``provenance_url.json`` file from the Conda +maintainers. It is not clear whether Conda would like to adopt the +``provenance_url.json`` file. Conda already stores provenance related +information (similar to the provenance information proposed in this PEP) in +JSON files located in the ``conda-meta`` directory `following its actions +during installation +`__. + +Using provenance_url.json in downstream installers +-------------------------------------------------- + +The proposed ``provenance_url.json`` file was meant to be adopted primarily by +Python installers. Other installers, such as APT or DNF, might record the +provenance of the installed downstream Python distributions in their own +way specific to downstream package management. The proposed file is +not expected to be created by these downstream package installers and thus they +were intentionally left out of this PEP. However, any input by developers or +maintainers of these installers is valuable to possibly enrich the +``provenance_url.json`` file with information that would help in some way. + +.. _710-tool-survey: + +Appendix: Survey of installers and libraries +============================================ + +pip +--- + +The function from pip's internal API responsible for installing wheels, named +`_install_wheel +`__, +does not store any ``provenance_url.json`` file in the ``.dist-info`` +directory. Additionally, a prototype introducing the mentioned file to pip in +`pypa/pip#11865`_ demonstrates incorporating logic for handling the +``provenance_url.json`` file in pip's source code. + +As pip is used by some of the tools mentioned below to install Python package +distributions, findings for pip apply to these tools, as well as pip does not +allow parametrizing creation of files in the ``.dist-info`` directory in its +internal API. Most of the tools mentioned below that use pip invoke pip as a +subprocess which has no effect on the eventual presence of the +``provenance_url.json`` file in the ``.dist-info`` directory. + +distlib +------- + +`distlib`_ implements low-level functionality to manipulate the +``dist-info`` directory. The database of installed distributions does not use +any file named ``provenance_url.json``, based on `the distlib's source code +`__. + +Pipenv +------ + +`Pipenv`_ uses pip `to install Python package distributions +`__. +There wasn't any additional identified logic that would cause backwards +compatibility issues when introducing the ``provenance_url.json`` file in the +``.dist-info`` directory. + +installer +--------- + +`installer`_ does not create a ``provenance_url.json`` file explicitly. +Nevertheless, as per the :ref:`Recording Installed Projects ` +specification, installer allows passing the ``additional_metadata`` argument to +create a file in the ``.dist-info`` directory - see `the source code +`__. +To avoid any backwards compatibility issues, any library or tool using +installer must not request creating the ``provenance_url.json`` file using the +mentioned ``additional_metadata`` argument. + +Poetry +------ + +The installation logic in `Poetry`_ depends on the +``installer.modern-installer`` configuration option (`see docs +`__). + +For cases when the ``installer.modern-installer`` configuration option is set +to ``false``, Poetry uses `pip for installing Python package distributions +`__. + +On the other hand, when ``installer.modern-installer`` configuration option is +set to ``true``, Poetry uses `installer to install Python package distributions +`__. +As can be seen from the linked sources, there isn't passed any additional +metadata file named ``provenance_url.json`` that would cause compatibility +issues with this PEP. + +Conda +----- + +`Conda`_ does not create any ``provenance_url.json`` file +`when Python package distributions are installed +`__. + +Hatch +----- + +`Hatch`_ uses pip `to install project dependencies +`__. + +micropipenv +----------- + +As `micropipenv`_ is a wrapper on top of pip, it uses +pip to install Python distributions, for both `lock files +`__ +as well as `for requirements files +`__. + +Thamos +------ + +`Thamos`_ uses micropipenv `to install Python package +distributions +`__, +hence any findings for micropipenv apply for Thamos. + +PDM +--- + +`PDM`_ uses installer `to install binary distributions +`__. +The only additional metadata file it eventually creates in the ``.dist-info`` +directory is `the REFER_TO file +`__. + +References +========== + +.. _pypa/pip#11865: https://github.com/pypa/pip/pull/11865 + +.. _fridex/pip#2: https://github.com/fridex/pip/pull/2/ + +.. _pip_preserve: https://pypi.org/project/pip-preserve/ + +.. _pip_sbom: https://github.com/sethmlarson/pip-sbom + +.. _thoth-station/micropipenv#206: https://github.com/thoth-station/micropipenv/issues/206 + +.. _pypa/pip-audit#170: https://github.com/pypa/pip-audit/issues/170 + +.. _pip_installation_report: https://pip.pypa.io/en/stable/reference/installation-report/ + +.. _distlib: https://distlib.readthedocs.io/ + +.. _Pipenv: https://pipenv.pypa.io/ + +.. _installer: https://github.com/pypa/installer + +.. _Poetry: https://python-poetry.org/ + +.. _Conda: https://docs.conda.io/ + +.. _Hatch: https://hatch.pypa.io/ + +.. _micropipenv: https://github.com/thoth-station/micropipenv + +.. _Thamos: https://github.com/thoth-station/thamos/ + +.. _PDM: https://pdm.fming.dev/ + +Acknowledgements +================ + +Thanks to Dustin Ingram, Brett Cannon, and Paul Moore for the initial discussion in +which this idea originated. + +Thanks to Donald Stufft, Ofek Lev, and Trishank Kuppusamy for early feedback +and support to work on this PEP. + +Thanks to Gregory P. Smith, StĂ©phane Bidoul, and C.A.M. Gerlach for +reviewing this PEP and providing valuable suggestions. + +Thanks to Seth Michael Larson for providing valuable suggestions and for +the proposed pip-sbom prototype. + +Thanks to StĂ©phane Bidoul and Chris Jerdonek for :pep:`610`. + +Last, but not least, thanks to Donald Stufft for sponsoring this PEP. + +Copyright +========= + +This document is placed in the public domain or under the CC0-1.0-Universal +license, whichever is more permissive. diff --git a/peps/pep-0711.rst b/peps/pep-0711.rst new file mode 100644 index 000000000..bff113317 --- /dev/null +++ b/peps/pep-0711.rst @@ -0,0 +1,584 @@ +PEP: 711 +Title: PyBI: a standard format for distributing Python Binaries +Author: Nathaniel J. Smith +PEP-Delegate: TODO +Discussions-To: https://discuss.python.org/t/pep-711-pybi-a-standard-format-for-distributing-python-binaries/25547 +Status: Draft +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 06-Apr-2023 +Post-History: `06-Apr-2023 `__ + + +Abstract +======== + +“Like wheels, but instead of a pre-built python package, it’s a +pre-built python interpreter” + + +Motivation +========== + +End goal: Pypi.org has pre-built packages for all Python versions on all +popular platforms, so automated tools can easily grab any of them and +set it up. It becomes quick and easy to try Python prereleases, pin +Python versions in CI, make a temporary environment to reproduce a bug +report that only happens on a specific Python point release, etc. + +First step (this PEP): define a standard packaging file format to hold pre-built +Python interpreters, that reuses existing Python packaging standards as much as +possible. + + +Examples +======== + +Example pybi builds are available at `pybi.vorpus.org +`__. They're zip files, so you can unpack them and poke +around inside if you want to get a feel for how they're laid out. + +You can also look at the `tooling I used to create them +`__. + + +Specification +============= + +Filename +-------- + +Filename: ``{distribution}-{version}[-{build tag}]-{platform tag}.pybi`` + +This matches the wheel file format defined in :pep:`427`, except dropping the +``{python tag}`` and ``{abi tag}`` and changing the extension from ``.whl`` → +``.pybi``. + +For example: + +- ``cpython-3.9.3-manylinux_2014.pybi`` +- ``cpython-3.10b2-win_amd64.pybi`` + +Just like for wheels, if a pybi supports multiple platforms, you can +separate them by dots to make a “compressed tag set”: + +- ``cpython-3.9.5-macosx_11_0_x86_64.macosx_11_0_arm64.pybi`` + +(Though in practice this probably won’t be used much, e.g. the above +filename is more idiomatically written as +``cpython-3.9.5-macosx_11_0_universal2.pybi``.) + + +File contents +------------- + +A ``.pybi`` file is a zip file, that can be unpacked directly into an +arbitrary location and then used as a self-contained Python environment. +There’s no ``.data`` directory or install scheme keys, because the +Python environment knows which install scheme it’s using, so it can just +put things in the right places to start with. + +The “arbitrary location” part is important: the pybi can’t contain any +hardcoded absolute paths. In particular, any preinstalled scripts MUST +NOT embed absolute paths in their shebang lines. + +Similar to wheels’ ``-.dist-info`` directory, the pybi archive +must contain a top-level directory named ``pybi-info/``. (Rationale: calling it +``pybi-info`` instead ``dist-info`` makes sure that tools don’t get confused +about which kind of metadata they’re looking at; leaving off the +``{name}-{version}`` part is fine because only one pybi can be installed into a +given directory.) The ``pybi-info/`` directory contains at least the following +files: + +- ``.../PYBI``: metadata about the archive itself, in the same + RFC822-ish format as ``METADATA`` and ``WHEEL`` files: + + :: + + Pybi-Version: 1.0 + Generator: {name} {version} + Tag: {platform tag} + Tag: {another platform tag} + Tag: {...and so on...} + Build: 1 # optional + +- ``.../RECORD``: same as in wheels, except see the note about + symlinks, below. + +- ``.../METADATA``: In the same format as described in the current core + metadata spec, except that the following keys are forbidden because + they don’t make sense: + + - ``Requires-Dist`` + - ``Provides-Extra`` + - ``Requires-Python`` + + And also there are some new, required keys described below. + +Pybi-specific core metadata +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here's an example of the new ``METADATA`` fields, before we give the full details:: + + Pybi-Environment-Marker-Variables: {"implementation_name": "cpython", "implementation_version": "3.10.8", "os_name": "posix", "platform_machine": "x86_64", "platform_system": "Linux", "python_full_version": "3.10.8", "platform_python_implementation": "CPython", "python_version": "3.10", "sys_platform": "linux"} + Pybi-Paths: {"stdlib": "lib/python3.10", "platstdlib": "lib/python3.10", "purelib": "lib/python3.10/site-packages", "platlib": "lib/python3.10/site-packages", "include": "include/python3.10", "platinclude": "include/python3.10", "scripts": "bin", "data": "."} + Pybi-Wheel-Tag: cp310-cp310-PLATFORM + Pybi-Wheel-Tag: cp310-abi3-PLATFORM + Pybi-Wheel-Tag: cp310-none-PLATFORM + Pybi-Wheel-Tag: cp39-abi3-PLATFORM + Pybi-Wheel-Tag: cp38-abi3-PLATFORM + Pybi-Wheel-Tag: cp37-abi3-PLATFORM + Pybi-Wheel-Tag: cp36-abi3-PLATFORM + Pybi-Wheel-Tag: cp35-abi3-PLATFORM + Pybi-Wheel-Tag: cp34-abi3-PLATFORM + Pybi-Wheel-Tag: cp33-abi3-PLATFORM + Pybi-Wheel-Tag: cp32-abi3-PLATFORM + Pybi-Wheel-Tag: py310-none-PLATFORM + Pybi-Wheel-Tag: py3-none-PLATFORM + Pybi-Wheel-Tag: py39-none-PLATFORM + Pybi-Wheel-Tag: py38-none-PLATFORM + Pybi-Wheel-Tag: py37-none-PLATFORM + Pybi-Wheel-Tag: py36-none-PLATFORM + Pybi-Wheel-Tag: py35-none-PLATFORM + Pybi-Wheel-Tag: py34-none-PLATFORM + Pybi-Wheel-Tag: py33-none-PLATFORM + Pybi-Wheel-Tag: py32-none-PLATFORM + Pybi-Wheel-Tag: py31-none-PLATFORM + Pybi-Wheel-Tag: py30-none-PLATFORM + Pybi-Wheel-Tag: py310-none-any + Pybi-Wheel-Tag: py3-none-any + Pybi-Wheel-Tag: py39-none-any + Pybi-Wheel-Tag: py38-none-any + Pybi-Wheel-Tag: py37-none-any + Pybi-Wheel-Tag: py36-none-any + Pybi-Wheel-Tag: py35-none-any + Pybi-Wheel-Tag: py34-none-any + Pybi-Wheel-Tag: py33-none-any + Pybi-Wheel-Tag: py32-none-any + Pybi-Wheel-Tag: py31-none-any + Pybi-Wheel-Tag: py30-none-any + +Specification: + +- ``Pybi-Environment-Marker-Variables``: The value of all PEP 508 + environment marker variables that are static across installs of this + Pybi, as a JSON dict. So for example: + + - ``python_version`` will always be present, because a Python 3.10 package + always has ``python_version == "3.10"``. + + - ``platform_version`` will generally not be present, because it gives + detailed information about the OS where Python is running, for example:: + + #60-Ubuntu SMP Thu May 6 07:46:32 UTC 2021 + + ``platform_release`` has similar issues. + + - ``platform_machine`` will *usually* be present, except for macOS universal2 + pybis: these can potentially be run in either x86-64 or arm64 mode, and we + don't know which until the interpreter is actually invoked, so we can't + record it in static metadata. + + **Rationale:** In many cases, this should allow a resolver running on Linux + to compute package pins for a Python environment on Windows, or vice-versa, + so long as the resolver has access to the target platform’s .pybi file. (Note + that ``Requires-Python`` constraints can be checked by using the + ``python_full_version`` value.) While we have to leave out a few keys + sometimes, they're either fairly useless (``platform_version``, + ``platform_release``) or can be reconstructed by the resolver + (``platform_machine``). + + The markers are also just generally useful information to have + accessible. For example, if you have a ``pypy3-7.3.2`` pybi, and you + want to know what version of the Python language that supports, then + that’s recorded in the ``python_version`` marker. + + (Note: we may want to deprecate/remove ``platform_version`` and + ``platform_release``? They're problematic and I can't figure out any cases + where they're useful. But that's out of scope of this particular PEP.) + +- ``Pybi-Paths``: The install paths needed to install wheels (same keys + as ``sysconfig.get_paths()``), as relative paths starting at the root + of the zip file, as a JSON dict. + + These paths MUST be written in Unix format, using forward slashes as + a separator, not backslashes. + + It must be possible to invoke the Python interpreter by running + ``{paths["scripts"]}/python``. If there are alternative interpreter + entry points (e.g. ``pythonw`` for Windows GUI apps), then they + should also be in that directory under their conventional names, with + no version number attached. (You can *also* have a ``python3.11`` + symlink if you want; there’s no rule against that. It’s just that + ``python`` has to exist and work.) + + **Rationale:** ``Pybi-Paths`` and ``Pybi-Wheel-Tag``\ s (see below) are + together enough to let an installer choose wheels and install them into an + unpacked pybi environment, without invoking Python. Besides, we need to write + down the interpreter location somewhere, so it’s two birds with one stone. + +- ``Pybi-Wheel-Tag``: The wheel tags supported by this interpreter, in + preference order (most-preferred first, least-preferred last), except + that the special platform tag ``PLATFORM`` should replace any + platform tags that depend on the final installation system. + + **Discussion:** It would be niceℱ if installers could compute a pybi’s + corresponding wheel tags ahead of time, so that they could install + wheels into the unpacked pybi without needing to actually invoke the + python interpreter to query its tags – both for efficiency and to + allow for more exotic use cases like setting up a Windows environment + from a Linux host. + + But unfortunately, it’s impossible to compute the full set of + platform tags supported by a Python installation ahead of time, + because they can depend on the final system: + + - A pybi tagged ``manylinux_2_12_x86_64`` can always use wheels + tagged as ``manylinux_2_12_x86_64``. It also *might* be able to + use wheels tagged ``manylinux_2_17_x86_64``, but only if the final + installation system has glibc 2.17+. + + - A pybi tagged ``macosx_11_0_universal2`` (= x86-64 + arm64 support + in the same binary) might be able to use wheels tagged as + ``macosx_11_0_arm64``, but only if it’s installed on an “Apple + Silicon” machine and running in arm64 mode. + + In these two cases, an installation tool can still work out the + appropriate set of wheel tags by computing the local platform tags, + taking the wheel tag templates from ``Pybi-Wheel-Tag``, and swapping + in the actual supported platforms in place of the magic ``PLATFORM`` + string. + + However, there are other cases that are even more complicated: + + - You can (usually) run both 32- and 64-bit apps on 64-bit Windows. So a pybi + installer might compute the set of allowable pybi tags on the current + platform as [``win32``, ``win_amd64``]. But you can’t then just take that + set and swap it into the pybi’s wheel tag template or you get nonsense: + + :: + + [ + "cp39-cp39-win32", + "cp39-cp39-win_amd64", + "cp39-abi3-win32", + "cp39-abi3-win_amd64", + ... + ] + + To handle this, the installer needs to somehow understand that a + ``manylinux_2_12_x86_64`` pybi can use a ``manylinux_2_17_x86_64`` wheel + as long as those are both valid tags on the current machine, but a + ``win32`` pybi *can’t* use a ``win_amd64`` wheel, even if those are both + valid tags on the current machine. + + - A pybi tagged ``macosx_11_0_universal2`` might be able to use + wheels tagged as ``macosx_11_0_x86_64``, but only if it’s + installed on an x86-64 machine *or* it’s installed on an ARM + machine *and* the interpreter is invoked with the magic + incantation that tells macOS to run a binary in x86-64 mode. So + how the installer plans to invoke the pybi matters too! + + So actually using ``Pybi-Wheel-Tag`` values is less trivial than it + might seem, and they’re probably only useful with fairly + sophisticated tooling. But, smart pybi installers will already have + to understand a lot of these platform compatibility issues in order + to select a working pybi, and for the cross-platform + pinning/environment building case, users can potentially provide + whatever information is needed to disambiguate exactly what platform + they’re targeting. So, it’s still useful enough to include in the PyBI + metadata -- tools that don't find it useful can simply ignore it. + +You can probably generate these metadata values by running this script on the +built interpreter: + +.. code:: python + + import packaging.markers + import packaging.tags + import sysconfig + import os.path + import json + import sys + + marker_vars = packaging.markers.default_environment() + # Delete any keys that depend on the final installation + del marker_vars["platform_release"] + del marker_vars["platform_version"] + # Darwin binaries are often multi-arch, so play it safe and + # delete the architecture marker. (Better would be to only + # do this if the pybi actually is multi-arch.) + if marker_vars["sys_platform"] == "darwin": + del marker_vars["platform_machine"] + + # Copied and tweaked version of packaging.tags.sys_tags + tags = [] + interp_name = packaging.tags.interpreter_name() + if interp_name == "cp": + tags += list(packaging.tags.cpython_tags(platforms=["xyzzy"])) + else: + tags += list(packaging.tags.generic_tags(platforms=["xyzzy"])) + + tags += list(packaging.tags.compatible_tags(platforms=["xyzzy"])) + + # Gross hack: packaging.tags normalizes platforms by lowercasing them, + # so we generate the tags with a unique string and then replace it + # with our special uppercase placeholder. + str_tags = [str(t).replace("xyzzy", "PLATFORM") for t in tags] + + (base_path,) = sysconfig.get_config_vars("installed_base") + # For some reason, macOS framework builds report their + # installed_base as a directory deep inside the framework. + while "Python.framework" in base_path: + base_path = os.path.dirname(base_path) + paths = {key: os.path.relpath(path, base_path).replace("\\", "/") for (key, path) in sysconfig.get_paths().items()} + + json.dump({"marker_vars": marker_vars, "tags": str_tags, "paths": paths}, sys.stdout) + +This emits a JSON dict on stdout with separate entries for each set of +pybi-specific tags. + + +Symlinks +-------- + +Currently, symlinks are used by default in all Unix Python installs (e.g., +``bin/python3 -> bin/python3.9``). And furthermore, symlinks are *required* to +store macOS framework builds in ``.pybi`` files. So, unlike wheel files, we +absolutely have to support symlinks in ``.pybi`` files for them to be useful at +all. + + +Representing symlinks in zip files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The de-facto standard for representing symlinks in zip files is the +Info-Zip symlink extension, which works as follows: + +- The symlink’s target path is stored as if it were the file contents +- The top 4 bits of the Unix permissions field are set to ``0xa``, + i.e.: ``permissions & 0xf000 == 0xa000`` +- The Unix permissions field, in turn, is stored as the top 16 bits of + the “external attributes” field. + +So if using Python’s ``zipfile`` module, you can check whether a +``ZipInfo`` represents a symlink by doing: + +.. code:: python + + (zip_info.external_attr >> 16) & 0xf000 == 0xa000 + +Or if using Rust’s ``zip`` crate, the equivalent check is: + +.. code:: rust + + fn is_symlink(zip_file: &zip::ZipFile) -> bool { + match zip_file.unix_mode() { + Some(mode) => mode & 0xf000 == 0xa000, + None => false, + } + } + +If you’re on Unix, your ``zip`` and ``unzip`` commands probably understands this +format already. + + +Representing symlinks in RECORD files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Normally, a ``RECORD`` file lists each file + its hash + its length: + +.. code:: text + + my/favorite/file,sha256=...,12345 + +For symlinks, we instead write: + +.. code:: text + + name/of/symlink,symlink=path/to/symlink/target, + +That is: we use a special “hash function” called ``symlink``, and then +store the actual symlink target as the “hash value”. And the length is +left empty. + +**Rationale:** we’re already committed to the ``RECORD`` file containing a +redundant check on everything in the main archive, so for symlinks we at least +need to store some kind of hash, plus some kind of flag to indicate that this is +a symlink. Given that symlink target strings are roughly the same size as a +hash, we might as well store them directly. This also makes the symlink +information easier to access for tools that don’t understand the Info-Zip +symlink extension, and makes it possible to losslessly unpack and repack a Unix +pybi on a Windows system, which someone might find handy at some point. + + +Storing symlinks in ``pybi`` files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a pybi creator stores a symlink, they MUST use both of the +mechanisms defined above: storing it in the zip archive directly using +the Info-Zip representation, and also recording it in the ``RECORD`` +file. + +Pybi consumers SHOULD validate that the symlinks in the archive and +``RECORD`` file are consistent with each other. + +We also considered using *only* the ``RECORD`` file to store symlinks, +but then the vanilla ``unzip`` tool wouldn’t be able to unpack them, and +that would make it hard to install a pybi from a shell script. + + +Limitations +~~~~~~~~~~~ + +Symlinks enable a lot of potential messiness. To keep things under +control, we impose the following restrictions: + +- Symlinks MUST NOT be used in ``.pybi``\ s targeting Windows, or other + platforms that are missing first-class symlink support. + +- Symlinks MUST NOT be used inside the ``pybi-info`` directory. + (Rationale: there’s no need, and it makes things simpler for + resolvers that need to extract info from ``pybi-info`` without + unpacking the whole archive.) + +- Symlink targets MUST be relative paths, and MUST be inside the pybi + directory. + +- If ``A/B/...`` is recorded as a symlink in the archive, then there + MUST NOT be any other entries in the archive named like + ``A/B/.../C``. + + For example, if an archive has a symlink ``foo -> bar``, and then + later in the archive there’s a regular file named ``foo/blah.py``, + then a naive unpacker could potentially end up writing a file called + ``bar/blah.py``. Don’t be naive. + +Unpackers MUST verify that these rules are followed, because without +them attackers could create evil symlinks like ``foo -> /etc/passwd`` or +``foo -> ../../../../../etc`` + ``foo/passwd -> ...`` and cause havoc. + + +Non-normative comments +====================== + +Why not just use conda? +----------------------- + +This isn't really in the scope of this PEP, but since conda is a popular way to +distribute binary Python interpreters, it's a natural question. + +The simple answer is: conda is great! But, there are lots of python users who +aren't conda users, and they deserve nice things too. This PEP just gives them +another option. + +The deeper answer is: the maintainers who upload packages to PyPI are the +backbone of the Python ecosystem. They're the first audience for Python +packaging tools. And one thing they want is to upload a package once, and have +it be accessible across all the different ways Python is deployed: in Debian and +Fedora and Homebrew and FreeBSD, in Conda environments, in big companies' +monorepos, in Nix, in Blender plugins, in RenPy games, ..... you get the idea. + +All of these environments have their own tooling and strategies for managing +packages and dependencies. So what's special about PyPI and wheels is that +they're designed to describe dependencies in a *standard, abstract way*, that +all these downstream systems can consume and convert into their local +conventions. That's why package maintainers use Python-specific metadata and +upload to PyPI: because it lets them address all of those systems +simultaneously. Every time you build a Python package for conda, there's an +intermediate wheel that's generated, because wheels are the common language that +Python package build systems and conda can use to talk to each other. + +But then, if you're a maintainer releasing an sdist+wheels, then you naturally +want to test what you're releasing, which may depend on arbitrary PyPI packages +and versions. So you need tools that build Python environments directly from +PyPI, and conda is fundamentally not designed to do that. So conda and pip are +both necessary for different cases, and this proposal happens to be targeting +the pip side of that equation. + + +Sdists (or not) +--------------- + +It might be cool to have an “sdist” equivalent for pybis, i.e., some +kind of format for a Python source release that’s structured-enough to +let tools automatically fetch and build it into a pybi, for platforms +where prebuilt pybis aren’t available. But, this isn’t necessary for the +MVP and opens a can of worms, so let’s worry about it later. + + +What packages should be bundled inside a pybi? +---------------------------------------------- + +Pybi builders have the power to pick and choose what exactly goes inside. For +example, you could include some preinstalled packages in the pybi’s +``site-packages`` directory, or prune out bits of the stdlib that you don’t +want. We can’t stop you! Though if you do preinstall packages, then it's +strongly recommended to also include the correct metadata (``.dist-info`` etc.), +so that it’s possible for Pip or other tools to understand out what’s going on. + +For my prototype “general purpose” pybi’s, what I chose is: + +- Make sure ``site-packages`` is *empty*. + + **Rationale:** for traditional standalone python installers that are targeted + at end-users, you probably want to include at least ``pip``, to avoid + bootstrapping issues (:pep:`453`). But pybis are different: they’re designed + to be installed by “smart” tooling, that consume the pybi as part of some + kind of larger automated deployment process. It’s easier for these installers + to start from a blank slate and then add whatever they need, than for them to + start with some preinstalled packages that they may or may not want. (And + besides, you can still run ``python -m ensurepip``.) + +- Include the full stdlib, *except* for ``test``. + + **Rationale:** the top-level ``test`` module contains CPython’s own test + suite. It’s huge (CPython without ``test`` is ~37 MB, then ``test`` + adds another ~25 MB on top of that!), and essentially never used by + regular user code. Also, as precedent, the official nuget packages, + the official manylinux images, and multiple Linux distributions all + leave it out, and this hasn’t caused any major problems. + + So this seems like the best way to balance broad compatibility with + reasonable download/install sizes. + +- I’m not shipping any ``.pyc`` files. They take up space in the + download, can be generated on the final system at minimal cost, and + dropping them removes a source of location-dependence. (``.pyc`` + files store the absolute path of the corresponding ``.py`` file and + include it in tracebacks; but, pybis are relocatable, so the correct + path isn’t known until after install.) + + +Backwards Compatibility +======================= + +No backwards compatibility considerations. + + +Security Implications +===================== + +No security implications, beyond the fact that anyone who takes it upon +themselves to distribute binaries has to come up with a plan to manage their +security (e.g., whether they roll a new build after an OpenSSL CVE drops). But +collectively, we core Python folks are already maintaining binary builds for all +major platforms (macOS + Windows through python.org, and Linux builds through +the official manylinux image), so even if we do start releasing official CPython +builds on PyPI it doesn't really raise any new security issues. + + +How to Teach This +================= + +This isn't targeted at end-users; their experience will simply be that e.g. +their pyenv or tox invocation magically gets faster and more reliable (if those +projects' maintainers decide to take advantage of this PEP). + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0712.rst b/peps/pep-0712.rst new file mode 100644 index 000000000..cbf9f51b5 --- /dev/null +++ b/peps/pep-0712.rst @@ -0,0 +1,304 @@ +PEP: 712 +Title: Adding a "converter" parameter to dataclasses.field +Author: Joshua Cannon +Sponsor: Eric V. Smith +Discussions-To: https://discuss.python.org/t/pep-712-adding-a-converter-parameter-to-dataclasses-field/26126 +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 01-Jan-2023 +Python-Version: 3.13 +Post-History: `27-Dec-2022 `__, + `19-Jan-2023 `__, + `23-Apr-2023 `__, + +Abstract +======== + +:pep:`557` added :mod:`dataclasses` to the Python stdlib. :pep:`681` added +:func:`~py3.11:typing.dataclass_transform` to help type checkers understand +several common dataclass-like libraries, such as attrs, Pydantic, and object +relational mapper (ORM) packages such as SQLAlchemy and Django. + +A common feature other libraries provide over the standard library +implementation is the ability for the library to convert arguments given at +initialization time into the types expected for each field using a +user-provided conversion function. + +Therefore, this PEP adds a ``converter`` parameter to :func:`dataclasses.field` +(along with the requisite changes to :class:`dataclasses.Field` and +:func:`~py3.11:typing.dataclass_transform`) to specify the function to use to +convert the input value for each field to the representation to be stored in +the dataclass. + +Motivation +========== + +There is no existing, standard way for :mod:`dataclasses` or third-party +dataclass-like libraries to support argument conversion in a type-checkable +way. To work around this limitation, library authors/users are forced to choose +to: + +* Opt-in to a custom Mypy plugin. These plugins help Mypy understand the + conversion semantics, but not other tools. +* Shift conversion responsibility onto the caller of the dataclass + constructor. This can make constructing certain dataclasses unnecessarily + verbose and repetitive. +* Provide a custom ``__init__`` which declares "wider" parameter types and + converts them when setting the appropriate attribute. This not only duplicates + the typing annotations between the converter and ``__init__``, but also opts + the user out of many of the features :mod:`dataclasses` provides. +* Provide a custom ``__init__`` but without meaningful type annotations + for the parameter types requiring conversion. + +None of these choices are ideal. + +Rationale +========= + +Adding argument conversion semantics is useful and beneficial enough that most +dataclass-like libraries provide support. Adding this feature to the standard +library means more users are able to opt-in to these benefits without requiring +third-party libraries. Additionally third-party libraries are able to clue +type-checkers into their own conversion semantics through added support in +:func:`~py3.11:typing.dataclass_transform`, meaning users of those libraries +benefit as well. + +Specification +============= + +New ``converter`` parameter +--------------------------- + +This specification introduces a new parameter named ``converter`` to the +:func:`dataclasses.field` function. If provided, it represents a single-argument +callable used to convert all values when assigning to the associated attribute. + +For frozen dataclasses, the converter is only used inside a ``dataclass``-synthesized +``__init__`` when setting the attribute. For non-frozen dataclasses, the converter +is used for all attribute assignment (E.g. ``obj.attr = value``), which includes +assignment of default values. + +The converter is not used when reading attributes, as the attributes should already +have been converted. + +Adding this parameter also implies the following changes: + +* A ``converter`` attribute will be added to :class:`dataclasses.Field`. +* ``converter`` will be added to :func:`~py3.11:typing.dataclass_transform`'s + list of supported field specifier parameters. + +Example +''''''' + +.. code-block:: python + + @dataclasses.dataclass + class InventoryItem: + # `converter` as a type (including a GenericAlias). + id: int = dataclasses.field(converter=int) + skus: tuple[int, ...] = dataclasses.field(converter=tuple[int, ...]) + # `converter` as a callable. + vendor: str | None = dataclasses.field(converter=str_or_none)) + names: tuple[str, ...] = dataclasses.field( + converter=lambda names: tuple(map(str.lower, names)) + ) # Note that lambdas are supported, but discouraged as they are untyped. + + # The default value is also converted; therefore the following is not a + # type error. + stock_image_path: pathlib.PurePosixPath = dataclasses.field( + converter=pathlib.PurePosixPath, default="assets/unknown.png" + ) + + # Default value conversion extends to `default_factory`; + # therefore the following is also not a type error. + shelves: tuple = dataclasses.field( + converter=tuple, default_factory=list + ) + + item1 = InventoryItem( + "1", + [234, 765], + None, + ["PYTHON PLUSHIE", "FLUFFY SNAKE"] + ) + # item1's repr would be (with added newlines for readability): + # InventoryItem( + # id=1, + # skus=(234, 765), + # vendor=None, + # names=('PYTHON PLUSHIE', 'FLUFFY SNAKE'), + # stock_image_path=PurePosixPath('assets/unknown.png'), + # shelves=() + # ) + + # Attribute assignment also participates in conversion. + item1.skus = [555] + # item1's skus attribute is now (555,). + + +Impact on typing +---------------- + +A ``converter`` must be a callable that accepts a single positional argument, and +the parameter type corresponding to this positional argument provides the type +of the the synthesized ``__init__`` parameter associated with the field. + +In other words, the argument provided for the converter parameter must be +compatible with ``Callable[[T], X]`` where ``T`` is the input type for +the converter and ``X`` is the output type of the converter. + +Type-checking ``default`` and ``default_factory`` +''''''''''''''''''''''''''''''''''''''''''''''''' + +Because default values are unconditionally converted using ``converter``, if +an argument for ``converter`` is provided alongside either ``default`` or +``default_factory``, the type of the default (the ``default`` argument if +provided, otherwise the return value of ``default_factory``) should be checked +using the type of the single argument to the ``converter`` callable. + +Converter return type +''''''''''''''''''''' + +The return type of the callable must be a type that's compatible with the +field's declared type. This includes the field's type exactly, but can also be +a type that's more specialized (such as a converter returning a ``list[int]`` +for a field annotated as ``list``, or a converter returning an ``int`` for a +field annotated as ``int | str``). + +Indirection of allowable argument types +--------------------------------------- + +One downside introduced by this PEP is that knowing what argument types are +allowed in the dataclass' ``__init__`` and during attribute assignment is not +immediately obvious from reading the dataclass. The allowable types are defined +by the converter. + +This is true when reading code from source, however typing-related aides such +as ``typing.reveal_type`` and "IntelliSense" in an IDE should make it easy to know +exactly what types are allowed without having to read any source code. + + +Backward Compatibility +====================== + +These changes don't introduce any compatibility problems since they +only introduce opt-in new features. + +Security Implications +====================== + +There are no direct security concerns with these changes. + +How to Teach This +================= + +Documentation and examples explaining the new parameter and behavior will be +added to the relevant sections of the docs site (primarily on +:mod:`dataclasses`) and linked from the *What's New* document. + +The added documentation/examples will also cover the "common pitfalls" that +users of converters are likely to encounter. Such pitfalls include: + +* Needing to handle ``None``/sentinel values. +* Needing to handle values that are already of the correct type. +* Avoiding lambdas for converters, as the synthesized ``__init__`` + parameter's type will become ``Any``. +* Forgetting to convert values in the bodies of user-defined ``__init__`` in + frozen dataclasses. +* Forgetting to convert values in the bodies of user-defined ``__setattr__`` in + non-frozen dataclasses. + +Reference Implementation +======================== + +The attrs library `already includes `__ a ``converter`` +parameter containing converter semantics. + +CPython support is implemented on `a branch in the author's fork `__. + +Rejected Ideas +============== + +Just adding "converter" to ``typing.dataclass_transform``'s ``field_specifiers`` +-------------------------------------------------------------------------------- + +The idea of isolating this addition to +:func:`~py3.11:typing.dataclass_transform` was briefly +`discussed on Typing-SIG `__ where it was suggested +to broaden this to :mod:`dataclasses` more generally. + +Additionally, adding this to :mod:`dataclasses` ensures anyone can reap the +benefits without requiring additional libraries. + +Not converting default values +----------------------------- + +There are pros and cons with both converting and not converting default values. +Leaving default values as-is allows type-checkers and dataclass authors to +expect that the type of the default matches the type of the field. However, +converting default values has three large advantages: + +1. Consistency. Unconditionally converting all values that are assigned to the + attribute, involves fewer "special rules" that users must remember. + +2. Simpler defaults. Allowing the default value to have the same type as + user-provided values means dataclass authors get the same conveniences as + their callers. + +3. Compatibility with attrs. Attrs unconditionally uses the converter to + convert default values. + +Automatic conversion using the field's type +------------------------------------------- + +One idea could be to allow the type of the field specified (e.g. ``str`` or +``int``) to be used as a converter for each argument provided. +`Pydantic's data conversion `__ has semantics which +appear to be similar to this approach. + +This works well for fairly simple types, but leads to ambiguity in expected +behavior for complex types such as generics. E.g. For ``tuple[int, ...]`` it is +ambiguous if the converter is supposed to simply convert an iterable to a tuple, +or if it is additionally supposed to convert each element type to ``int``. Or +for ``int | None``, which isn't callable. + +Deducing the attribute type from the return type of the converter +----------------------------------------------------------------- + +Another idea would be to allow the user to omit the attribute's type annotation +if providing a ``field`` with a ``converter`` argument. Although this would +reduce the common repetition this PEP introduces (e.g. ``x: str = field(converter=str)``), +it isn't clear how to best support this while maintaining the current dataclass +semantics (namely, that the attribute order is preserved for things like the +synthesized ``__init__``, or ``dataclasses.fields``). This is because there isn't +an easy way in Python (today) to get the annotation-only attributes interspersed +with un-annotated attributes in the order they were defined. + +A sentinel annotation could be applied (e.g. ``x: FromConverter = ...``), +however this breaks a fundamental assumption of type annotations. + +Lastly, this is feasible if *all* fields (including those without a converter) +were assigned to ``dataclasses.field``, which means the class' own namespace +holds the order, however this trades repetition of type+converter with +repetition of field assignment. The end result is no gain or loss of repetition, +but with the added complexity of dataclasses semantics. + +This PEP doesn't suggest it can't or shouldn't be done. Just that it isn't +included in this PEP. + + +References +========== + +.. _attrs-converters: https://www.attrs.org/en/21.2.0/examples.html#conversion +.. _cpython-branch: https://github.com/thejcannon/cpython/tree/converter +.. _only-dataclass-transform: https://mail.python.org/archives/list/typing-sig@python.org/thread/NWZQIINJQZDOCZGO6TGCUP2PNW4PEKNY/ +.. _pydantic-data-conversion: https://docs.pydantic.dev/usage/models/#data-conversion + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0713.rst b/peps/pep-0713.rst new file mode 100644 index 000000000..65d117b1d --- /dev/null +++ b/peps/pep-0713.rst @@ -0,0 +1,207 @@ +PEP: 713 +Title: Callable Modules +Author: Amethyst Reese +Sponsor: Ɓukasz Langa +Discussions-To: https://discuss.python.org/t/pep-713-callable-modules/26127 +Status: Rejected +Type: Standards Track +Content-Type: text/x-rst +Created: 20-Apr-2023 +Python-Version: 3.12 +Post-History: `23-Apr-2023 `__ +Resolution: https://discuss.python.org/t/26127/86 + + +Rejection Notice +================ + +The Steering Council didn't feel that there was a compelling reason to +have this PEP, even though it clearly could be done from a consistency +point of view. +If this idea comes up again in the future, this is a useful prior +discussion to refer to. + + +Abstract +======== + +Modules are currently not directly callable. Classes can define a ``__call__`` +method that makes instance objects callable, but defining a similarly named +function in the global module scope has no effect, and that function can +only be called by importing or referencing it directly as ``module.__call__``. +:pep:`562` added support for :meth:`~object.__getattr__` and :meth:`~object.__dir__` for modules, but +defining ``__getattr__`` to return a value for ``__call__`` still does not +make a module callable. + +This PEP proposes support for making modules directly callable by defining +a ``__call__`` object in the module's global namespace, either as a standard +function, or an arbitrary callable object. + + +Motivation +========== + +Many modules have only a single primary interface to their functionality. +In many cases, that interface is a single callable object, where being able +to import and use the module directly as a callable provides a more "Pythonic" +interface for users:: + + # user.py + + import fancy + + @fancy + def func(...): + ... + +Currently, providing this style of interface requires modifying the module +object at runtime to make it callable. + +This is commonly done by replacing the module object in ``sys.modules`` with +a callable alternative (such as a function or class instance):: + + # fancy.py + + def fancy(...): + ... + + sys.modules[__name__] = fancy + +This has the effect of making the original module effectively unreachable +without further hooks from the author, even with ``from module import member``. +It also results in a "module" object that is missing all of the special module +attributes, including ``__doc__``, ``__package__``, ``__path__``, etc. + +Alternatively, a module author can choose to override the module's ``__class__`` +property with a custom type that provides a callable interface:: + + # fancy.py + + def fancy(...): + ... + + class FancyModule(types.ModuleType): + def __call__(self, ...): + return fancy(...) + + sys.modules[__name__].__class__ = FancyModule + +The downside of either approach is that it not only results in extra +boilerplate, but also results in type checker failures because they don't +recognize that the module is callable at runtime: + +.. code-block:: console + + $ mypy user.py + user.py:3: error: Module not callable [operator] + Found 1 error in 1 file (checked 1 source file) + + +Specification +============= + +When a module object is called, and a ``__call__`` object is found (either +as the result of a ``__getattr__`` or ``__dict__`` lookup), then that object +will be called with the given arguments. + +If a ``__call__`` object is not found, then a ``TypeError`` will be raised, +matching the existing behavior. + +All of these examples would be considered valid, callable modules: + +.. code-block:: python + + # hello.py + + def __call__(...): + pass + +.. code-block:: python + + # hello.py + + class Hello: + pass + + __call__ = Hello + +.. code-block:: python + + # hello.py + + def hello(): + pass + + def __getattr__(name): + if name == "__call__": + return hello + +The first two styles should generally be preferred, as it allows for easier +static analysis from tools like type checkers, though the third form would be +allowed in order to make the implementation more consistent. + +The intent is to allow arbitrary callable object to be assigned to the module's +``__call__`` property or returned by the module's ``__getattr__`` method, +enabling module authors to pick the most suitable mechanism for making their +module callable by users. + + +Backwards Compatibility and Impact on Performance +================================================= + +This PEP is not expected to cause any backwards incompatibility. Any modules +that already contain a ``__call__`` object will continue to function the same +as before, though with the additional ability to be called directly. It is +considered unlikely that modules with an existing ``__call__`` object would +depend on the existing behavior of raising ``TypeError`` when called. + +Performance implications of this PEP are minimal, as it defines a new interface. +Calling a module would trigger a lookup for the name ``__call__`` on a module +object. Existing workarounds for creating callable modules already depend on +this behavior for generic objects, resulting in similar performance for these +callable modules. + +Type checkers will likely need to be updated accordingly to treat modules with +a ``__call__`` object as callable. This should be possible to support in type +checkers when checking code targeted at older Python versions that do not +support callable modules, with the expectation that these modules would also +include one of the workarounds mentioned earlier to make the module callable. + + +How to Teach This +================= + +The documentation for :external+python:ref:`callable types ` will +include modules in the list, with a link to :meth:`~object.__call__`. +The :external+python:ref:`callable-types` documentation will include a section +covering callable modules, with example code, similar to the section for +`customizing module attribute access`__. + +__ https://docs.python.org/3/reference/datamodel.html#customizing-module-attribute-access + + +Reference Implementation +======================== + +The proposed implementation of callable modules is available in +`CPython PR #103742 `_. + + +Rejected Ideas +============== + +Given the introduction of ``__getattr__`` and ``__dir__``, and the proposal +to enable use of ``__call__``, it was considered if it was worth allowing use +of *all* :external+python:ref:`specialnames` for modules, such as ``__or__`` +and ``__iter__``. While this would not be completely undesired, it increases +the potential for backward compatibility concerns, and these other special +methods are likely to provide less utility to library authors in comparison +to ``__call__``. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + diff --git a/peps/pep-0714.rst b/peps/pep-0714.rst new file mode 100644 index 000000000..00542d5e5 --- /dev/null +++ b/peps/pep-0714.rst @@ -0,0 +1,275 @@ +PEP: 714 +Title: Rename dist-info-metadata in the Simple API +Author: Donald Stufft +PEP-Delegate: Paul Moore +Discussions-To: https://discuss.python.org/t/27471 +Status: Accepted +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 06-Jun-2023 +Post-History: `06-Jun-2023 `__ +Resolution: https://discuss.python.org/t/27471/19 + + +Abstract +======== + +This PEP renames the metadata provided by :pep:`658` in both HTML and JSON +formats of the Simple API and provides guidelines for both clients and servers +in how to handle the renaming. + + +Motivation +========== + +:pep:`658` specified a mechanism to host the core metadata files from an +artifact available through the Simple API such that a client could fetch the +metadata and use it without having to download the entire artifact. Later +:pep:`691` was written to add the ability to use JSON rather than HTML on the +Simple API, which included support for the :pep:`658` metadata. + +Unfortunately, PyPI did not support :pep:`658` until just +`recently `__, which released with +a `bug `__ where the +``dist-info-metadata`` key from :pep:`658` was incorrectly named in the JSON +representation, to be ``data-dist-info-metadata``. However, when +attempting to fix that bug, it was discovered that pip *also* had a +`bug `__, where any use of +``dist-info-metadata`` in the JSON representation would cause pip to hard fail +with an exception. + +The bug in pip has existed since at least ``v22.3``, which means that it has +been released for approximately 8 months, long enough to have been pulled into +Python releases, downstream Linux releases, baked into containers, virtual +environments, etc. + +This puts us in an awkward position of having a bug on PyPI that cannot be fixed +without breaking pip, due to a bug in pip, but that version of pip is old enough +to have been widely deployed. To make matters worse, a version of pip that is +broken in this way cannot install *anything* from PyPI once it fixes its bug, +including installing a new, fixed version of pip. + + +Rationale +========= + +There are 3 main options for a path forward for fixing these bugs: + +1. Do not change the spec, fix the bug in pip, wait some amount of time, then + fix the bug in PyPI, breaking anyone using an unfixed pip such that they + cannot even install a new pip from PyPI. +2. Do the same as (1), but special case PyPI so it does not emit the :pep:`658` + metadata for pip, even if it is available. This allows people to upgrade pip + if they're on a broken version, but nothing else. +3. Change the spec to avoid the key that pip can't handle currently, allowing + PyPI to emit that key and a new version of pip to be released to take + advantage of that key. + +This PEP chooses (3), but goes a little further and also renames the key in the +HTML representation. + +Typically we do not change specs because of bugs that only affect one particular +implementation, unless the spec itself is at fault, which isn't the case here: +the spec is fine and these are just genuine bugs in pip and PyPI. + +However, we choose to do this for 4 reasons: + +1. Bugs that affect pip and PyPI together represent an outsized amount of impact + compared to any other client or repository combination. +2. The impact of being broken is that installs do not function, at all, rather + than degrading gracefully in some way. +3. The feature that is being blocked by these bugs is of large importance to + the ability to quickly and efficiently resolve dependencies from PyPI with + pip, and having to delay it for a long period of time while we wait for the + broken versions of pip to fall out of use would be of detriment to the entire + ecosystem. +4. The downsides of changing the spec are fairly limited, given that we do not + believe that support for this is widespread, so it affects only a limited + number of projects. + + +Specification +============= + +The keywords "**MUST**", "**MUST NOT**", "**REQUIRED**", "**SHALL**", +"**SHALL NOT**", "**SHOULD**", "**SHOULD NOT**", "**RECOMMENDED**", "**MAY**", +and "**OPTIONAL**"" in this document are to be interpreted as described in +:rfc:`RFC 2119 <2119>`. + + +Servers +------- + +The :pep:`658` metadata, when used in the HTML representation of the Simple API, +**MUST** be emitted using the attribute name ``data-core-metadata``, with the +supported values remaining the same. + +The :pep:`658` metadata, when used in the :pep:`691` JSON representation of the +Simple API, **MUST** be emitted using the key ``core-metadata``, with the +supported values remaining the same. + +To support clients that used the previous key names, the HTML representation +**MAY** also be emitted using the ``data-dist-info-metadata``, and if it does +so it **MUST** match the value of ``data-core-metadata``. + + + +Clients +------- + +Clients consuming any of the HTML representations of the Simple API **MUST** +read the :pep:`658` metadata from the key ``data-core-metadata`` if it is +present. They **MAY** optionally use the legacy ``data-dist-info-metadata`` if +it is present but ``data-core-metadata`` is not. + +Clients consuming the JSON representation of the Simple API **MUST** read the +:pep:`658` metadata from the key ``core-metadata`` if it is present. They +**MAY** optionally use the legacy ``dist-info-metadata`` key if it is present +but ``core-metadata`` is not. + + +Backwards Compatibility +======================= + +There is a minor compatibility break in this PEP, in that clients that currently +correctly handle the existing metadata keys will not automatically understand +the newer metadata keys, but they should degrade gracefully, and simply act +as if the :pep:`658` metadata does not exist. + +Otherwise there should be no compatibility problems with this PEP. + + +Rejected Ideas +============== + +Leave the spec unchanged, and cope with fixing in PyPI and/or pip +----------------------------------------------------------------- + +We believe that the improvements brought by :pep:`658` are very important to +improving the performance of resolving dependencies from PyPI, and would like to +be able to deploy it as quickly as we can. + +Unfortunately the nature of these bugs is that we cannot deploy them as is +without breaking widely deployed and used versions of pip. The breakages in +this case would be bad enough that affected users would not even be able to +directly upgrade their version of pip to fix it, but would have to manually +fetch pip another way first (e.g. ``get-pip.py``). + +This is something that PyPI would be unwilling to do without some way to +mitigate those breakages for those users. Without some reasonable mitigation +strategy, we would have to wait until those versions of pip are no longer in use +on PyPI, which would likely be 5+ years from now. + +There are a few possible mitigation strategies that we could use, but we've +rejected them as well. + + +Mitigation: Special Case pip +++++++++++++++++++++++++++++ + +The breakages are particularly bad in that it prevents users from even upgrading +pip to get an unbroken version of pip, so a command like +``pip install --upgrade pip`` would fail. We could mitigate this by having PyPI +special case pip itself, so that the JSON endpoint never returns the :pep:`658` +metadata and the above still works. + +This PEP rejects this idea because while the simple command that only upgrades +pip would work, if the user included *anything* else in that command to upgrade +then the command would go back to failing, which we consider to be still too +large of a breakage. + +Additionally, while this bug happens to be getting exposed right now with PyPI, +it is really a bug that would happen with any :pep:`691` repository that +correctly exposed the :pep:`658` metadata. This would mean that every repository +would have to carry this special case for pip. + + +Mitigation: Have the server use User-Agent Detection +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +pip puts its version number into its ``User-Agent``, which means that the server +could detect the version number and serve different responses based on that +version number so that we don't serve the :pep:`658` metadata to versions of pip +that are broken. + +This PEP rejects this idea because supporting ``User-Agent`` detection is too +difficult to implement in a reasonable way. + +1. On PyPI we rely heavily on caching the Simple API in our CDN. If we varied + the responses based on ``User-Agent``, then our CDN cache would have an + explosion of cache keys for the same content, which would make it more likely + that any particular request would not be cached and fall back to hitting + our backend servers, which would have to scale much higher to support the + load. +2. PyPI *could* support the ``User-Agent`` detection idea by mutating the + ``Accept`` header of the request so that those versions appear to only + accept the HTML version, allowing us to maintain the CDNs cache keys. This + doesn't affect any downstream caches of PyPI though, including pip's HTTP + cache which would possibly have JSON versions cached for those requests and + we wouldn't emit a ``Vary`` on ``User-Agent`` for them to know that it isn't + acceptable to share those caches, and adding a ``Vary: User-Agent`` for + downstream caches would have the same problem as (1), but for downstream + caches instead of our CDN cache. +3. The pip bug ultimately isn't PyPI specific, it affects any repository that + implements :pep:`691` and :pep:`658` together. This would mean that + workarounds that rely on implementation specific fixes have to be replicated + for each repository that implements both, which may not be easy or possible + in all cases (static mirrors may not be able to do this ``User-Agent`` + detection for instance). + + +Only change the JSON key +------------------------ + +The bug in pip only affects the JSON representation of the Simple API, so we only +*need* to actually change the key in the JSON, and we could leave the existing +HTML keys alone. + +This PEP rejects doing that because we believe that in the long term, having +the HTML and JSON key names diverge would make mistakes like this more likely +and make implementing and understanding the spec more confusing. + +The main reason that we would want to not change the HTML keys is to not lose +:pep:`658` support in any HTML only clients or repositories that might already +support it. This PEP mitigates that breakage by allowing both clients and +servers to continue to support both keys, with a recommendation of when and +how to do that. + + +Recommendations +=============== + +The recommendations in this section, other than this notice itself, are +non-normative, and represent what the PEP authors believe to be the best default +implementation decisions for something implementing this PEP, but it does not +represent any sort of requirement to match these decisions. + + +Servers +------- + +We recommend that servers *only* emit the newer keys, particularly for the JSON +representation of the Simple API since the bug itself only affected JSON. + +Servers that wish to support :pep:`658` in clients that use HTML and have it +implemented, can safely emit both keys *only* in HTML. + +Servers should not emit the old keys in JSON unless they know that no broken +versions of pip will be used to access their server. + + +Clients +------- + +We recommend that clients support both keys, for both HTML and JSON, preferring +the newer key as this PEP requires. This will allow clients to support +repositories that already have correctly implemented :pep:`658` and :pep:`691` +but have not implemented this PEP. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0715.rst b/peps/pep-0715.rst new file mode 100644 index 000000000..f84967822 --- /dev/null +++ b/peps/pep-0715.rst @@ -0,0 +1,171 @@ +PEP: 715 +Title: Disabling bdist_egg distribution uploads on PyPI +Author: William Woodruff +Sponsor: Donald Stufft +PEP-Delegate: Donald Stufft +Discussions-To: https://discuss.python.org/t/27610 +Status: Final +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 06-Jun-2023 +Post-History: `09-Jun-2023 `__ +Resolution: https://discuss.python.org/t/pep-715-disabling-bdist-egg-distribution-uploads-on-pypi/27610/13 + +Abstract +======== + +This PEP recommends deprecating and then disabling new uploads of the +``bdist_egg`` distribution type on PyPI. In a parallel move, this PEP recommends +deprecating and then disabling new uploads of distribution filenames that have +the ``.egg`` suffix. + +After this PEP, PyPI will only accept new uploads of the ``sdist`` +and ``bdist_wheel`` types, corresponding to files with ``.tar.gz``/``.zip`` and +``.whl`` suffixes respectively. + +This PEP does not recommend removing or otherwise affecting any previously +uploaded ``bdist_egg`` distributions or files with the ``.egg`` suffix. + +Rationale +========= + +Previous Work +------------- + +The groundwork for this proposal was established with :pep:`527`, which +proposed deprecating and eventually removing upload support for a handful +of un(der)used file extensions and distribution types. + +In particular, :pep:`527` proposed the removal of the ``bdist_dumb``, +``bdist_rpm``, ``bdist_dmg``, ``bdist_msi``, and ``bdist_wininst`` distribution +types, as well as the ``.tar``, ``.tar.bz2``, ``.tar.xz``, ``.tar.Z``, +``.tgz``, and ``.tbz`` file extensions on distribution filenames. + +:pep:`527` was fully enacted with +`PR #7529 `_ to Warehouse, +which was merged on 13 April 2020. + +The ``bdist_egg`` format +------------------------ + +The ``bdist_egg`` filetype identifies distributions in the +:term:`egg format `. The +egg format was introduced by setuptools in 2004 and is roughly equivalent +in functionality to the +:term:`wheel format ` +first introduced by :pep:`427` in 2012 +as :ref:`the standardized format ` +for :term:`built distributions `. + +Despite its longevity, the egg format has had +`limited adoption on PyPI `_. +Some observations from that issue: + +* In the month of May 2023, ``bdist_egg`` uploads accounted for 0.2% of all + distribution uploads to PyPI; +* ``pip`` deprecated its ``--egg`` option in 2016; +* ``setuptools`` has considered egg support deprecated since 2019; +* ``build`` only supports the ``sdist`` and ``bdist_wheel`` filetypes. + +Given the above, this PEP proposes the removal of the ``bdist_egg`` format +under the same justifications presented in :pep:`527`, namely: + +* Egg distributions are of limited use to the broader ecosystem and + therefore represent a non-reciprocal maintenance burden; +* Having an additional built distribution format + is confusing to end users, who may + mistakenly pick it over the wheel format; + +This PEP *additionally* offers an argument for removal rooted in +standardization and duplication: the egg format is not standardized by any +PEP or other community standard, and overlaps heavily with its standardized +and well-supported alternative (wheel). + +The ``.egg`` file extension +--------------------------- + +The ``.egg`` file extension is used exclusively for distributions of the +``bdist_egg`` format. As such, it serves no purpose in a scenario where +PyPI disables new distribution uploads for ``bdist_egg`` distributions. + +Removal Process +=============== + +This PEP does **NOT** propose removing any existing files from PyPI, only +disallowing new ones from being uploaded. + +PyPI will provide a deprecation period of one month. At the beginning +of the deprecation period, maintainers of projects that have uploaded one or +more egg distributions since 1 Jan 2023 will receive a one-time email informing +them of the upcoming end of support for egg distribution uploads. + +During the deprecation period, users will continue to be allowed to upload egg +distributions to new and existing projects. Uploading an egg distribution +during this period will also send all maintainers of the project +a similar email as above, reminding them of the upcoming end of support. + +After the deprecation period, support for uploading egg distributions will +cease to exist on PyPI. + +Prior Art +--------- + +The removal process above was based on that of :pep:`527`, with the following +changes: + +* All projects will be continue to be allowed to upload eggs + during the deprecation period, not just those that have done so previously. +* Uploading an egg during the deprecation period will also trigger an email + to maintainers, in addition to the one-time email at the beginning. + +Backwards Compatibility +======================= + +Limited Impact +-------------- + +As noted in the rationale section, this PEP is expected to have no impact +on the overwhelming majority of PyPI users and projects, and there has been +`substantial community coordination `_ +over the past 1 1/2 years to minimize the impact on the few last use cases. + +Hosted Files +------------ + +This PEP does **NOT** propose the removal of any egg distributions that have +already been uploaded to PyPI. All previously uploaded egg distributions will +remain downloadable, ensuring that existing users will continue to be able +to download them. + +Deprecation Period +------------------ + +This PEP uses the removal process documented above, which specifies +a deprecation period of 1 month for projects that have previously uploaded +egg distributions to PyPI. + +After the end of the deprecation period, support for uploading new egg +distributions will cease to exist on PyPI. + +Security Implications +===================== + +This PEP does not identify any positive or negative security implications +associated with removing upload support for egg distributions. + +How To Teach This +================= + +As part of the removal process, PyPI will send emails to all maintainers of +projects that have previously uploaded egg distributions in 2023. + +Additionally, PyPI will write a post on the +`PyPI blog `_ that publicly announces the deprecation +period's start and end. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0718.rst b/peps/pep-0718.rst new file mode 100644 index 000000000..b04475089 --- /dev/null +++ b/peps/pep-0718.rst @@ -0,0 +1,185 @@ +PEP: 718 +Title: Subscriptable functions +Author: James Hilton-Balfe +Sponsor: Guido van Rossum +Discussions-To: https://discuss.python.org/t/26463/ +Status: Draft +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 23-Jun-2023 +Python-Version: 3.13 +Post-History: `24-Jun-2023 `__ + +Abstract +-------- + +This PEP proposes making function objects subscriptable for typing purposes. Doing so +gives developers explicit control over the types produced by the type checker where +bi-directional inference (which allows for the types of parameters of anonymous +functions to be inferred) and other methods than specialisation are insufficient. + +Motivation +---------- + +Currently, it is not possible to infer the type parameters to generic functions in +certain situations: + +.. code-block:: python + + def make_list[T](*args: T) -> list[T]: ... + reveal_type(make_list()) # type checker cannot infer a meaningful type for T + +Making instances of ``FunctionType`` subscriptable would allow for this constructor to +be typed: + +.. code-block:: python + + reveal_type(make_list[int]()) # type is list[int] + +Currently you have to use an assignment to provide a precise type: + +.. code-block:: python + + x: list[int] = make_list() + reveal_type(x) # type is list[int] + +but this code is unnecessarily verbose taking up multiple lines for a simple function +call. + +Similarly, ``T`` in this example cannot currently be meaningfully inferred, so ``x`` is +untyped without an extra assignment: + +.. code-block:: python + + def factory[T](func: Callable[[T], Any]) -> Foo[T]: ... + + reveal_type(factory(lambda x: "Hello World" * x)) + +If function objects were subscriptable, however, a more specific type could be given: + +.. code-block:: python + + reveal_type(factory[int](lambda x: "Hello World" * x)) # type is Foo[int] + +Currently, with unspecialised literals, it is not possible to determine a type for +situations similar to: + +.. code-block:: python + + def foo[T](x: list[T]) -> T: ... + reveal_type(foo([])) # type checker cannot infer T (yet again) + +.. code-block:: python + + reveal_type(foo[int]([])) # type is int + +It is also useful to be able to specify in cases in which a certain type must be passed +to a function beforehand: + +.. code-block:: python + + words = ["hello", "world"] + foo[int](words) # Invalid: list[str] is incompatible with list[int] + +Allowing subscription makes functions and methods consistent with generic classes where +they weren't already. Whilst all of the proposed changes can be implemented using +callable generic classes, syntactic sugar would be highly welcome. + +Due to this, specialising the function and using it as a new factory is fine + +.. code-block:: python + + make_int_list = make_list[int] + reveal_type(make_int_list()) # type is list[int] + +This proposal also opens the door to +`monomorphisation `_ and +`reified types `_ + +Rationale +--------- + +Function objects in this PEP is used to refer to ``FunctionType``\ , ``MethodType``\ , +``BuiltinFunctionType``\ , ``BuiltinMethodType`` and ``MethodWrapperType``\ . + +For ``MethodType`` you should be able to write: + +.. code-block:: python + + class Foo: + def make_list[T](self, *args: T) -> list[T]: ... + + Foo().make_list[int]() + +and have it work similarly to a ``FunctionType``. + +For ``BuiltinFunctionType``, so builtin generic functions (e.g. ``max`` and ``min``) +work like ones defined in Python. Built-in functions should behave as much like +functions implemented in Python as possible. + +``BuiltinMethodType`` is the same type as ``BuiltinFunctionType``. + +``MethodWrapperType`` (e.g. the type of ``object().__str__``) is useful for +generic magic methods. + +Specification +------------- + +Function objects should implement ``__getitem__`` to allow for subscription at runtime +and return an instance of ``types.GenericAlias`` with ``__origin__`` set as the +callable and ``__args__`` as the types passed. + +Type checkers should support subscripting functions and understand that the parameters +passed to the function subscription should follow the same rules as a generic callable +class. + +Setting ``__orig_class__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Currently, ``__orig_class__`` is an attribute set in ``GenericAlias.__call__`` to the +instance of the ``GenericAlias`` that created the called class e.g. + +.. code-block:: python + + class Foo[T]: ... + + assert Foo[int]().__orig_class__ == Foo[int] + +Currently, ``__orig_class__`` is unconditionally set; however, to avoid potential +erasure on any created instances, this attribute should not be set if ``__origin__`` is +an instance of any function object. + +The following code snippet would fail at runtime without this change as +``__orig_class__`` would be ``bar[str]`` and not ``Foo[int]``. + +.. code-block:: python + + def bar[U](): + return Foo[int]() + + assert bar[str]().__orig_class__ is Foo[int] + +Backwards Compatibility +----------------------- +Currently these classes are not subclassable and so there are no backwards +compatibility concerns with regards to classes already implementing +``__getitem__``. + +Reference Implementation +------------------------ + +The runtime changes proposed can be found here +https://github.com/Gobot1234/cpython/tree/function-subscript + +Acknowledgements +---------------- + +Thank you to Alex Waygood and Jelle Zijlstra for their feedback on this PEP and Guido +for some motivating examples. + +Copyright +--------- + +This document is placed in the public domain or under the CC0-1.0-Universal license, +whichever is more permissive. diff --git a/peps/pep-0719.rst b/peps/pep-0719.rst new file mode 100644 index 000000000..40bcec4b0 --- /dev/null +++ b/peps/pep-0719.rst @@ -0,0 +1,84 @@ +PEP: 719 +Title: Python 3.13 Release Schedule +Author: Thomas Wouters +Status: Active +Type: Informational +Topic: Release +Content-Type: text/x-rst +Created: 26-May-2023 +Python-Version: 3.13 + + +Abstract +======== + +This document describes the development and release schedule for +Python 3.13. The schedule primarily concerns itself with PEP-sized +items. + +.. Small features may be added up to the first beta + release. Bugs may be fixed until the final release, + which is planned for October 2024. + + +Release Manager and Crew +======================== + +- 3.13 Release Manager: Thomas Wouters +- Windows installers: Steve Dower +- Mac installers: Ned Deily +- Documentation: Julien Palard + + +Release Schedule +================ + +3.13.0 schedule +--------------- + +Note: the dates below use a 17-month development period that results +in a 12-month release cadence between feature versions, as defined by +:pep:`602`. + +Actual: + +- 3.13 development begins: Monday, 2023-05-22 + +Expected: + +- 3.13.0 alpha 1: Tuesday, 2023-10-17 +- 3.13.0 alpha 2: Tuesday, 2023-11-21 +- 3.13.0 alpha 3: Tuesday, 2023-12-19 +- 3.13.0 alpha 4: Tuesday, 2024-01-16 +- 3.13.0 alpha 5: Tuesday, 2024-02-13 +- 3.13.0 alpha 6: Tuesday, 2024-03-12 +- 3.13.0 alpha 7: Tuesday, 2024-04-09 +- 3.13.0 beta 1: Tuesday, 2024-05-07 + (No new features beyond this point.) +- 3.13.0 beta 2: Tuesday, 2024-05-28 +- 3.13.0 beta 3: Tuesday, 2024-06-18 +- 3.13.0 beta 4: Tuesday, 2024-07-16 +- 3.13.0 candidate 1: Tuesday, 2024-07-30 +- 3.13.0 candidate 2: Tuesday, 2024-09-03 +- 3.13.0 final: Tuesday, 2024-10-01 + +Subsequent bugfix releases every two months. + + +3.13 Lifespan +------------- + +3.13 will receive bugfix updates approximately every 2 months for +approximately 24 months. Around the time of the release of 3.15.0 final, the +final 3.13 bugfix update will be released. After that, it is expected that +security updates (source only) will be released until 5 years after the +release of 3.13.0 final, so until approximately October 2029. + + +Copyright +========= + +This document is placed in the public domain or under the CC0-1.0-Universal +license, whichever is more permissive. + + diff --git a/peps/pep-0720.rst b/peps/pep-0720.rst new file mode 100644 index 000000000..14e0708c7 --- /dev/null +++ b/peps/pep-0720.rst @@ -0,0 +1,930 @@ +PEP: 720 +Title: Cross-compiling Python packages +Author: Filipe LaĂ­ns +PEP-Delegate: +Status: Draft +Type: Informational +Content-Type: text/x-rst +Created: 01-Jul-2023 +Python-Version: 3.12 + + +Abstract +======== + +This PEP attempts to document the status of cross-compilation of downstream +projects. + +It should give an overview of the approaches currently used by distributors +(Linux distros, WASM environment providers, etc.) to cross-compile downstream +projects (3rd party extensions, etc.). + + +Motivation +========== + +We write this PEP to express the challenges in cross-compilation and act as a +supporting document in future improvement proposals. + + +Analysis +======== + + +Introduction +------------ + +There are a couple different approaches being used to tackle this, with +different levels of interaction required from the user, but they all require a +significant amount of effort. This is due to the lack of standardized +cross-compilation infrastructure on the Python packaging ecosystem, which itself +stems from the complexity of cross-builds, making it a huge undertaking. + + +Upstream support +---------------- + +Some major projects like CPython, setuptools, etc. provide some support to help +with cross-compilation, but it's unofficial and at a best-effort basis. For +example, the ``sysconfig`` module allows overwriting the data module name via +the ``_PYTHON_SYSCONFIGDATA_NAME`` environment variable, something that is +required for cross-builds, and setuptools `accepts patches`__ [1]_ to tweak/fix +its logic to be compatible with popular `"environment faking" +`_ workflows [2]_. + +The lack of first-party support in upstream projects leads to cross-compilation +being fragile and requiring a significant effort from users, but at the same +time, the lack of standardization makes it harder for upstreams to improve +support as there's no clarity on how this feature should be provided. + +.. [1] At the time of writing (Jun 2023), setuptools' compiler interface code, + the component that most of affects cross-compilation, is developed on the + `pypa/distutils`__ repository, which gets periodically synced to the + setuptools repository. + +.. [2] We specifically mention *popular* workflows, because this is not + standardized. Though, many of the most popular implementations + (crossenv_, conda-forge_'s build system, etc.) work similarly, and this + is what we are referring to here. For clarity, the implementations we are + referring to here could be described as *crossenv-style*. + +.. __: https://github.com/pypa/distutils/pulls?q=cross +.. __: https://github.com/pypa/distutils + +Projects with decent cross-build support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It seems relevant to point out that there are a few modern Python package +build-backends with, at least, decent cross-compilation support, those being +scikit-build__ and meson-python__. Both these projects integrate external mature +build-systems into Python packaging — CMake__ and Meson__, respectively — so +cross-build support is inherited from them. + +.. __: https://github.com/scikit-build/scikit-build +.. __: https://github.com/mesonbuild/meson-python +.. __: https://cmake.org/ +.. __: https://mesonbuild.com/ + + +Downstream approaches +--------------------- + +Cross-compilation approaches fall in a spectrum that goes from, by design, +requiring extensive user interaction to (ideally) almost none. Usually, they'll +be based on one of two main strategies, using a `cross-build environment`_, +or `faking the target environment`_. + +.. _approach-cross-environment: + +Cross-build environment +~~~~~~~~~~~~~~~~~~~~~~~ + +This consists of running the Python interpreter normally and utilizing the +cross-build provided by the projects' build-system. However, as we saw above, +upstream support is lacking, so this approach only works for a small-ish set of +projects. When this fails, the usual strategy is to patch the build-system code +to build use the correct toolchain, system details, etc. [3]_. + +Since this approach often requires package-specific patching, it requires a lot +of user interaction. + +.. admonition:: Examples + :class: note + + `python-for-android`_, `kivy-ios`_, etc. + +.. [3] The scope of the build-system patching varies between users and usually + depends on the their goal — some (eg. Linux distributions) may patch the + build-system to support cross-builds, while others might hardcode + compiler paths and system information in the build-system, to simply make + the build work. + +Faking the target environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Aiming to drop the requirement for user input, a popular approach is trying to +fake the target environment. It generally consists of monkeypatching the Python +interpreter to get it to mimic the interpreter on the target system, which +constitutes of changing many of the ``sys`` module attributes, the ``sysconfig`` +data, etc. Using this strategy, build-backends do not need to have any +cross-build support, and should just work without any code changes. + +Unfortunately, though, it isn't possible to truly fake the target environment. +There are many reasons for this, one of the main ones being that it breaks code +that actually needs to introspect the running interpreter. As a result, +monkeypatching Python to look like target is very tricky — to achieve the less +amount of breakage, we can only patch certain aspects of the interpreter. +Consequently, build-backends may need some code changes, but these are generally +much smaller than the previous approach. This is an inherent limitation of the +technique, meaning this strategy still requires some user interaction. + +Nonetheless, this strategy still works out-of-the-box with significantly more +projects than the approach above, and requires much less effort in these cases. +It is successful in decreasing the amount of user interaction needed, even +though it doesn't succeed in being generic. + +.. admonition:: Examples + :class: note + + `crossenv`_, `conda-forge`_, etc. + + +Environment introspection +------------------------- + +As explained above, most build system code is written with the assumption that +the target system is the same as where the build is occurring, so introspection +is usually used to guide the build. + +In this section, we try to document most of the ways this is accomplished. It +should give a decent overview of of environment details that are required by +build systems. + +.. list-table:: + :header-rows: 1 + + * - Snippet + - Description + - Variance + + * - .. code-block:: python + + >>> importlib.machinery.EXTENSION_SUFFIXES + [ + '.cpython-311-x86_64-linux-gnu.so', + '.abi3.so', + '.so', + ] + + - Extension (native module) suffixes supported by this interpreter. + - This is implementation-defined, but it **usually** differs based on the + implementation, system architecture, build configuration, Python + language version, and implementation version — if one exists. + + * - .. code-block:: python + + >>> importlib.machinery.SOURCE_SUFFIXES + ['.py'] + + - Source (pure-Python) suffixes supported by this interpreter. + - This is implementation-defined, but it **usually** doesn't differ + (outside exotic implementations or systems). + + * - .. code-block:: python + + >>> importlib.machinery.all_suffixes() + [ + '.py', + '.pyc', + '.cpython-311-x86_64-linux-gnu.so', + '.abi3.so', + '.so', + ] + + - All module file suffixes supported by this interpreter. It *should* be the + union of all ``importlib.machinery.*_SUFFIXES`` attributes. + - This is implementation-defined, but it **usually** differs based on the + implementation, system architecture, build configuration, Python + language version, and implementation version — if one exists. See the + entries above for more information. + + * - .. code-block:: python + + >>> sys.abiflags + '' + + - ABI flags, as specified in :pep:`3149`. + - Differs based on the build configuration. + + * - .. code-block:: python + + >>> sys.api_version + 1013 + + - C API version. + - Differs based on the Python installation. + + * - .. code-block:: python + + >>> sys.base_prefix + /usr + + - Prefix of the installation-wide directories where platform independent + files are installed. + - Differs based on the platform, and installation. + + * - .. code-block:: python + + >>> sys.base_exec_prefix + /usr + + - Prefix of the installation-wide directories where platform dependent + files are installed. + - Differs based on the platform, and installation. + + * - .. code-block:: python + + >>> sys.byteorder + 'little' + + - Native byte order. + - Differs based on the platform. + + * - .. code-block:: python + + >>> sys.builtin_module_names + ('_abc', '_ast', '_codecs', ...) + + - Names of all modules that are compiled into the Python interpreter. + - Differs based on the platform, system architecture, and build + configuration. + + * - .. code-block:: python + + >>> sys.exec_prefix + /usr + + - Prefix of the site-specific directories where platform independent files + are installed. Because it concerns the site-specific directories, in + standard virtual environment implementation, it will be a + virtual-environment-specific path. + - Differs based on the platform, installation, and environment. + + * - .. code-block:: python + + >>> sys.executable + '/usr/bin/python' + + - Path of the Python interpreter being used. + - Differs based on the installation. + + * - .. code-block:: python + + >>> with open(sys.executable, 'rb') as f: + ... header = f.read(4) + ... if is_elf := (header == b'\x7fELF'): + ... elf_class = int(f.read(1)) + ... size = {1: 52, 2: 64}.get(elf_class) + ... elf_header = f.read(size - 5) + + - Whether the Python interpreter is an ELF file, and the ELF header. This + approach is something used to identify the target architecture of the + installation (example__). + - Differs based on the installation. + + * - .. code-block:: python + + >>> sys.float_info + sys.float_info( + max=1.7976931348623157e+308, + max_exp=1024, + max_10_exp=308, + min=2.2250738585072014e-308, + min_exp=-1021, + min_10_exp=-307, + dig=15, + mant_dig=53, + epsilon=2.220446049250313e-16, + radix=2, + rounds=1, + ) + + - Low level information about the float type, as defined by ``float.h``. + - Differs based on the architecture, and platform. + + * - .. code-block:: python + + >>> sys.getandroidapilevel() + 21 + + - Integer representing the Android API level. + - Differs based on the platform. + + * - .. code-block:: python + + >>> sys.getwindowsversion() + sys.getwindowsversion( + major=10, + minor=0, + build=19045, + platform=2, + service_pack='', + ) + + - Windows version of the system. + - Differs based on the platform. + + * - .. code-block:: python + + >>> sys.hexversion + 0x30b03f0 + + - Python version encoded as an integer. + - Differs based on the Python language version. + + * - .. code-block:: python + + >>> sys.implementation + namespace( + name='cpython', + cache_tag='cpython-311', + version=sys.version_info( + major=3, + minor=11, + micro=3, + releaselevel='final', + serial=0, + ), + hexversion=0x30b03f0, + _multiarch='x86_64-linux-gnu', + ) + + - Interpreter implementation details. + - Differs based on the interpreter implementation, Python language + version, and implementation version — if one exists. It may also include + architecture-dependent information, so it may also differ based on the + system architecture. + + * - .. code-block:: python + + >>> sys.int_info + sys.int_info( + bits_per_digit=30, + sizeof_digit=4, + default_max_str_digits=4300, + str_digits_check_threshold=640, + ) + + - Low level information about Python's internal integer representation. + - Differs based on the architecture, platform, implementation, build, and + runtime flags. + + * - .. code-block:: python + + >>> sys.maxsize + 0x7fffffffffffffff + + - Maximum value a variable of type ``Py_ssize_t`` can take. + - Differs based on the architecture, platform, and implementation. + + * - .. code-block:: python + + >>> sys.maxunicode + 0x10ffff + + - Value of the largest Unicode code point. + - Differs based on the implementation, and on Python versions older than + 3.3, the build. + + * - .. code-block:: python + + >>> sys.platform + linux + + - Platform identifier. + - Differs based on the platform. + + * - .. code-block:: python + + >>> sys.prefix + /usr + + - Prefix of the site-specific directories where platform dependent files + are installed. Because it concerns the site-specific directories, in + standard virtual environment implementation, it will be a + virtual-environment-specific path. + - Differs based on the platform, installation, and environment. + + * - .. code-block:: python + + >>> sys.platlibdir + lib + + - Platform-specific library directory. + - Differs based on the platform, and vendor. + + * - .. code-block:: python + + >>> sys.version_info + sys.version_info( + major=3, + minor=11, + micro=3, + releaselevel='final', + serial=0, + ) + + - Python language version implemented by the interpreter. + - Differs if the target Python version is not the same [4]_. + + * - .. code-block:: python + + >>> sys.thread_info + sys.thread_info( + name='pthread', + lock='semaphore', + version='NPTL 2.37', + ) + + - Information about the thread implementation. + - Differs based on the platform, and implementation. + + * - .. code-block:: python + + >>> sys.winver + 3.8-32 + + - Version number used to form Windows registry keys. + - Differs based on the platform, and implementation. + + * - .. code-block:: python + + >>> sysconfig.get_config_vars() + { ... } + >>> sysconfig.get_config_var(...) + ... + + - Python distribution configuration variables. It includes a set of + variables [5]_ — like ``prefix``, ``exec_prefix``, etc. — based on the + running context [6]_, and may include some extra variables based on the + Python implementation and system. + + In CPython and most other implementations that use the same + build-system, the "extra" variables mention above are: on POSIX, all + variables from the ``Makefile`` used to build the interpreter, and on + Windows, it usually only includes a small subset of the those [7]_ — + like ``EXT_SUFFIX``, ``BINDIR``, etc. + + - This is implementation-defined, but it **usually** differs between + non-identical builds. Please refer to the + `sysconfig configuration variables`_ table for a overview of the different + configuration variable that are usually present. + +.. [4] Ideally, you want to perform cross-builds with the same Python version + and implementation, however, this is often not the case. It should not + be very problematic as long as the major and minor versions don't + change. + +.. [5] The set of config variables that will always be present mostly consists + of variables needed to calculate the installation scheme paths. + +.. [6] The context we refer here consists of the "path initialization", which is + a process that happens in the interpreter startup and is responsible for + figuring out which environment it is being run — eg. global environment, + virtual environment, etc. — and setting ``sys.prefix`` and other + attributes accordingly. + +.. [7] This is because Windows builds may not use the ``Makefile``, and instead + `use the Visual Studio build system`__. A subset of the most relevant + ``Makefile`` variables is provided to make user code that uses them + simpler. + +.. __: https://github.com/pypa/packaging/blob/2f80de7fd2a8bc199dadf5cf3f5f302a17084792/src/packaging/_manylinux.py#L43-L50 +.. __: https://docs.python.org/3/using/configure.html#debug-build + + +CPython (and similar) +~~~~~~~~~~~~~~~~~~~~~ + + +.. list-table:: ``sysconfig`` configuration variables + :name: sysconfig configuration variables + :header-rows: 1 + :widths: 20 20 30 30 + + * - Name + - Example Value + - Description + - Variance + + * - ``SOABI`` + - ``cpython-311-x86_64-linux-gnu`` + - ABI string — defined by :pep:`3149`. + - Differs based on the implementation, system architecture, Python + language version, and implementation version — if one exists. + + * - ``SHLIB_SUFFIX`` + - ``.so`` + - Shared library suffix. + - Differs based on the platform. + + * - ``EXT_SUFFIX`` + - ``.cpython-311-x86_64-linux-gnu.so`` + - Interpreter-specific Python extension (native module) suffix — generally + defined as ``.{SOABI}.{SHLIB_SUFFIX}``. + - Differs based on the implementation, system architecture, Python + language version, and implementation version — if one exists. + + * - ``LDLIBRARY`` + - ``libpython3.11.so`` + - Shared ``libpython`` library name — if available. If unavailable [8]_, + the variable will be empty, if available, the library should be located + in ``LIBDIR``. + - Differs based on the implementation, system architecture, build + configuration, Python language version, and implementation version — if + one exists. + + * - ``PY3LIBRARY`` + - ``libpython3.so`` + - Shared Python 3 only (major version bound only) [9]_ ``libpython`` + library name — if available. If unavailable [8]_, the variable will be + empty, if available, the library should be located in ``LIBDIR``. + - Differs based on the implementation, system architecture, build + configuration, Python language version, and implementation version — if + one exists. + + * - ``LIBRARY`` + - ``libpython3.11.a`` + - Static ``libpython`` library name — if available. If unavailable [8]_, + the variable will be empty, if available, the library should be located + in ``LIBDIR``. + - Differs based on the implementation, system architecture, build + configuration, Python language version, and implementation version — if + one exists. + + * - ``Py_DEBUG`` + - ``0`` + - Whether this is a `debug build`__. + - Differs based on the build configuration. + + * - ``WITH_PYMALLOC`` + - ``1`` + - Whether this build has pymalloc__ support. + - Differs based on the build configuration. + + * - ``Py_TRACE_REFS`` + - ``0`` + - Whether reference tracing (debug build only) is enabled. + - Differs based on the build configuration. + + * - ``Py_UNICODE_SIZE`` + - + - Size of the ``Py_UNICODE`` object, in bytes. This variable is only + present in CPython versions older than 3.3, and was commonly used to + detect if the build uses UCS2 or UCS4 for unicode objects — before + :pep:`393`. + - Differs based on the build configuration. + + * - ``Py_ENABLE_SHARED`` + - ``1`` + - Whether a shared ``libpython`` is available. + - Differs based on the build configuration. + + * - ``PY_ENABLE_SHARED`` + - ``1`` + - Whether a shared ``libpython`` is available. + - Differs based on the build configuration. + + * - ``CC`` + - ``gcc`` + - The C compiler used to build the Python distribution. + - Differs based on the build configuration. + + * - ``CXX`` + - ``g++`` + - The C compiler used to build the Python distribution. + - Differs based on the build configuration. + + * - ``CFLAGS`` + - ``-DNDEBUG -g -fwrapv ...`` + - The C compiler flags used to build the Python distribution. + - Differs based on the build configuration. + + * - ``py_version`` + - ``3.11.3`` + - Full form of the Python version. + - Differs based on the Python language version. + + * - ``py_version_short`` + - ``3.11`` + - Custom form of the Python version, containing only the major and minor + numbers. + - Differs based on the Python language version. + + * - ``py_version_nodot`` + - ``311`` + - Custom form of the Python version, containing only the major and minor + numbers, and no dots. + - Differs based on the Python language version. + + * - ``prefix`` + - ``/usr`` + - Same as ``sys.prefix``, please refer to the entry in table above. + - Differs based on the platform, installation, and environment. + + * - ``base`` + - ``/usr`` + - Same as ``sys.prefix``, please refer to the entry in table above. + - Differs based on the platform, installation, and environment. + + * - ``exec_prefix`` + - ``/usr`` + - Same as ``sys.exec_prefix``, please refer to the entry in table above. + - Differs based on the platform, installation, and environment. + + * - ``platbase`` + - ``/usr`` + - Same as ``sys.exec_prefix``, please refer to the entry in table above. + - Differs based on the platform, installation, and environment. + + * - ``installed_base`` + - ``/usr`` + - Same as ``sys.base_prefix``, please refer to the entry in table above. + - Differs based on the platform, and installation. + + * - ``installed_platbase`` + - ``/usr`` + - Same as ``sys.base_exec_prefix``, please refer to the entry in table + above. + - Differs based on the platform, and installation. + + * - ``platlibdir`` + - ``lib`` + - Same as ``sys.platlibdir``, please refer to the entry in table above. + - Differs based on the platform, and vendor. + + * - ``SIZEOF_*`` + - ``4`` + - Size of a certain C type (``double``, ``float``, etc.). + - Differs based on the system architecture, and build details. + + +.. [8] Due to Python bring compiled without shared or static ``libpython`` + support, respectively. + +.. [9] This is the ``libpython`` library that users of the `stable ABI`__ should + link against, if they need to link against ``libpython``. + +.. __: https://docs.python.org/3/c-api/intro.html#debugging-builds +.. __: https://docs.python.org/3/c-api/memory.html#pymalloc +.. __: https://docs.python.org/3/c-api/stable.html#stable-application-binary-interface + + +Relevant Information +-------------------- + +There are some bits of information required by build systems — eg. platform +particularities — scattered across many places, and it often is difficult to +identify code with assumptions based on them. In this section, we try to +document the most relevant cases. + + +When should extensions be linked against ``libpython``? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Short answer + Yes, on Windows. No on POSIX platforms, except Android, Cygwin, and other + Windows-based POSIX-like platforms. + +When building extensions for dynamic loading, depending on the target platform, +they may need to be linked against ``libpython``. + +On Windows, extensions need to link against ``libpython``, because all symbols +must be resolvable at link time. POSIX-like platforms based on Windows — like +Cygwin, MinGW, or MSYS — will also require linking against ``libpython``. + +On most POSIX platforms, it is not necessary to link against ``libpython``, as +the symbols will already be available in the due to the interpreter — or, when +embedding, the executable/library in question — already linking to +``libpython``. Not linking an extension module against ``libpython`` will allow +it to be loaded by static Python builds, so when possible, it is desirable to do +so (see GH-65735__). + +This might not be the case on all POSIX platforms, so make sure you check. One +example is Android, where only the main executable and ``LD_PRELOAD`` entries +are considered to be ``RTLD_GLOBAL`` (meaning dependencies are ``RTLD_LOCAL``) +[10]_, which causes the ``libpython`` symbols be unavailable when loading the +extension. + +.. [10] Refer to `dlopen's man page`__ for more information. + +.. __: https://github.com/python/cpython/issues/65735 +.. __: https://man.archlinux.org/man/dlopen.3 + + +What are ``prefix``, ``exec_prefix``, ``base_prefix``, and ``base_exec_prefix``? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These are ``sys`` attributes `set in the Python initialization`__ that describe +the running environment. They refer to the prefix of directories where +installation/environment files are installed, according to the table below. + +==================== ====================================== ================= +Name Target files Environment Scope +==================== ====================================== ================= +``prefix`` platform independent (eg. pure Python) site-specific +``exec_prefix`` platform dependent (eg. native code) site-specific +``base_prefix`` platform independent (eg. pure Python) installation-wide +``base_exec_prefix`` platform dependent (eg. native code) installation-wide +==================== ====================================== ================= + +Because the site-specific prefixes will be different inside virtual +environments, checking ``sys.prexix != sys.base_prefix`` is commonly used to +check if we are in a virtual environment. + +.. __: https://github.com/python/cpython/blob/6a70edf24ca217c5ed4a556d0df5748fc775c762/Modules/getpath.py + +Case studies +============ + + +crossenv +-------- + +:Description: Virtual Environments for Cross-Compiling Python Extension Modules. +:URL: https://github.com/benfogle/crossenv + +``crossenv`` is a tool to create a virtual environment with a monkeypatched +Python installation that tries to emulate the target machine in certain +scenarios. More about this approach can be found in the +`Faking the target environment`_ section. + + +conda-forge +----------- + +:Description: A community-led collection of recipes, build infrastructure and distributions for the conda package manager. +:URL: https://conda-forge.org/ + +XXX: Jaime will write a quick summary once the PEP draft is public. + +XXX +Uses a modified crossenv. + + +Yocto Project +------------- + +:Description: The Yocto Project is an open source collaboration project that helps developers create custom Linux-based systems regardless of the hardware architecture. +:URL: https://www.yoctoproject.org/ + +XXX: Sent email to the mailing list. + +TODO + + +Buildroot +--------- + +:Description: Buildroot is a simple, efficient and easy-to-use tool to generate embedded Linux systems through cross-compilation. +:URL: https://buildroot.org/ + +TODO + + +Pyodide +------- + +:Description: Pyodide is a Python distribution for the browser and Node.js based on WebAssembly. +:URL: https://pyodide.org/en/stable/ + +XXX: Hood should review/expand this section. + +``Pyodide`` is a provides a Python distribution compiled to WebAssembly__ +using the Emscripten__ toolchain. + +It patches several aspects of the CPython installation and some external +components. A custom package manager — micropip__ — supporting both Pure and +wasm32/Emscripten wheels, is also provided as a part of the distribution. On top +of this, a repo with a `selected set of 3rd party packages`__ is also provided +and enabled by default. + +.. __: https://webassembly.org/ +.. __: https://emscripten.org/ +.. __: https://micropip.pyodide.org/ +.. __: https://pyodide.org/en/stable/usage/packages-in-pyodide.html + + +Beeware +------- + +:Description: BeeWare allows you to write your app in Python and release it on multiple platforms. +:URL: https://beeware.org/ + +TODO + + +python-for-android +------------------ + +:Description: Turn your Python application into an Android APK. +:URL: https://github.com/kivy/python-for-android + +resource https://github.com/Android-for-Python/Android-for-Python-Users + +``python-for-android`` is a tool to package Python apps on Android. It creates a +Python distribution with your app and its dependencies. + +Pure-Python dependencies are handled automatically and in a generic way, but +native dependencies need recipes__. A set of recipes for +`popular dependencies`__ is provided, but users need to provide their own +recipes for any other native dependencies. + +.. __: https://python-for-android.readthedocs.io/en/latest/recipes/ +.. __: https://github.com/kivy/python-for-android/tree/develop/pythonforandroid/recipes + + +kivy-ios +-------- + +:Description: Toolchain for compiling Python / Kivy / other libraries for iOS. +:URL: https://github.com/kivy/kivy-ios + +``kivy-ios`` is a tool to package Python apps on iOS. It provides a toolchain to +build a Python distribution with your app and its dependencies, as well as a CLI +to create and manage Xcode projects that integrate with the toolchain. + +It uses the same approach as `python-for-android`_ (also maintained by the +`Kivy project`__) for app dependencies — pure-Python dependencies are handled +automatically, but native dependencies need recipes__, and the project provides +recipes for `popular dependencies`__. + +.. __: https://kivy.org +.. __: https://python-for-android.readthedocs.io/en/latest/recipes/ +.. __: https://github.com/kivy/kivy-ios/tree/master/kivy_ios/recipes + + +AidLearning +----------- + +:Description: AI, Android, Linux, ARM: AI application development platform based on Android+Linux integrated ecology. +:URL: https://github.com/aidlearning/AidLearning-FrameWork + +TODO + + +QPython +------- + +:Description: QPython is the Python engine for android. +:URL: https://github.com/qpython-android/qpython + +TODO + + +pyqtdeploy +---------- + +:Description: pyqtdeploy is a tool for deploying PyQt applications. +:URL: https://www.riverbankcomputing.com/software/pyqtdeploy/ + +contact https://www.riverbankcomputing.com/pipermail/pyqt/2023-May/thread.html +contacted Phil, the maintainer + +TODO + + +Chaquopy +-------- + +:Description: Chaquopy provides everything you need to include Python components in an Android app. +:URL: https://chaquo.com/chaquopy/ + +TODO + + +EDK II +------ + +:Description: EDK II is a modern, feature-rich, cross-platform firmware development environment for the UEFI and PI specifications. +:URL: https://github.com/tianocore/edk2-libc/tree/master/AppPkg/Applications/Python + +TODO + + +ActivePython +------------ + +:Description: Commercial-grade, quality-assured Python distribution focusing on easy installation and cross-platform compatibility on Windows, Linux, Mac OS X, Solaris, HP-UX and AIX. +:URL: https://www.activestate.com/products/python/ + +TODO + + +Termux +------ + +:Description: Termux is an Android terminal emulator and Linux environment app that works directly with no rooting or setup required. +:URL: https://termux.dev/en/ + +TODO diff --git a/peps/pep-0721.rst b/peps/pep-0721.rst new file mode 100644 index 000000000..5c2375f61 --- /dev/null +++ b/peps/pep-0721.rst @@ -0,0 +1,205 @@ +PEP: 721 +Title: Using tarfile.data_filter for source distribution extraction +Author: Petr Viktorin +PEP-Delegate: Paul Moore +Status: Final +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Requires: 706 +Created: 12-Jul-2023 +Python-Version: 3.12 +Post-History: `04-Jul-2023 `__, +Resolution: https://discuss.python.org/t/28928/13 + +.. canonical-pypa-spec:: :ref:`packaging:sdist-archive-features` + +Abstract +======== + +Extracting a source distribution archive should normally use the ``data`` +filter added in :pep:`706`. +We clarify details, and specify the behaviour for tools that cannot use the +filter directly. + + +Motivation +========== + +The *source distribution* ``sdist`` is defined as a tar archive. + +The ``tar`` format is designed to capture all metadata of Unix-like files. +Some of these are dangerous, unnecessary for source code, and/or +platform-dependent. +As explained in :pep:`706`, when extracting a tarball, one should always either +limit the allowed features, or explicitly give the tarball total control. + + +Rationale +========= + +For source distributions, the ``data`` filter introduced in :pep:`706` +is enough. It allows slightly more features than ``git`` and ``zip`` (both +commonly used in packaging workflows). + +However, not all tools can use the ``data`` filter, +so this PEP specifies an explicit set of expectations. +The aim is that the current behaviour of ``pip download`` +and ``setuptools.archive_util.unpack_tarfile`` is valid, +except cases deemed too dangerous to allow. +Another consideration is ease of implementation for non-Python tools. + + +Unpatched versions of Python +---------------------------- + +Tools are allowed to ignore this PEP when running on Python without tarfile +filters. + +The feature has been backported to all versions of Python supported by +``python.org``. Vendoring it in third-party libraries is tricky, +and we should not force all tools to do so. +This shifts the responsibility to keep up with security updates from the tools +to the users. + + +Permissions +----------- + +Common tools (``git``, ``zip``) don't preserve Unix permissions (mode bits). +Telling users to not rely on them in *sdists*, and allowing tools to handle +them relatively freely, seems fair. + +The only exception is the *executable* permission. +We recommend, but not require, that tools preserve it. +Given that scripts are generally platform-specific, it seems fitting to +say that keeping them executable is tool-specific behaviour. + +Note that while ``git`` preserves executability, ``zip`` (and thus ``wheel``) +doesn't do it natively. (It is possible to encode it in “external attributes”, +but Python's ``ZipFile.extract`` does not honour that.) + + +Specification +============= + +The following will be added to `the PyPA source distribution format spec `_ +under a new heading, “*Source distribution archive features*”: + +Because extracting tar files as-is is dangerous, and the results are +platform-specific, archive features of source distributions are limited. + +Unpacking with the data filter +------------------------------ + +When extracting a source distribution, tools MUST either use +``tarfile.data_filter`` (e.g. ``TarFile.extractall(..., filter='data')``), OR +follow the *Unpacking without the data filter* section below. + +As an exception, on Python interpreters without ``hasattr(tarfile, 'data_filter')`` +(:pep:`706`), tools that normally use that filter (directly on indirectly) +MAY warn the user and ignore this specification. +The trade-off between usability (e.g. fully trusting the archive) and +security (e.g. refusing to unpack) is left up to the tool in this case. + + +Unpacking without the data filter +--------------------------------- + +Tools that do not use the ``data`` filter directly (e.g. for backwards +compatibility, allowing additional features, or not using Python) MUST follow +this section. +(At the time of this writing, the ``data`` filter also follows this section, +but it may get out of sync in the future.) + +The following files are invalid in an ``sdist`` archive. +Upon encountering such an entry, tools SHOULD notify the user, +MUST NOT unpack the entry, and MAY abort with a failure: + +- Files that would be placed outside the destination directory. +- Links (symbolic or hard) pointing outside the destination directory. +- Device files (including pipes). + +The following are also invalid. Tools MAY treat them as above, +but are NOT REQUIRED to do so: + +- Files with a ``..`` component in the filename or link target. +- Links pointing to a file that is not part of the archive. + +Tools MAY unpack links (symbolic or hard) as regular files, +using content from the archive. + +When extracting ``sdist`` archives: + +- Leading slashes in file names MUST be dropped. + (This is nowadays standard behaviour for ``tar`` unpacking.) +- For each ``mode`` (Unix permission) bit, tools MUST either: + + - use the platform's default for a new file/directory (respectively), + - set the bit according to the archive, or + - use the bit from ``rw-r--r--`` (``0o644``) for non-executable files or + ``rwxr-xr-x`` (``0o755``) for executable files and directories. + +- High ``mode`` bits (setuid, setgid, sticky) MUST be cleared. +- It is RECOMMENDED to preserve the user *executable* bit. + + +Further hints +------------- + +Tool authors are encouraged to consider how *hints for further +verification* in ``tarfile`` documentation apply for their tool. + + +Backwards Compatibility +======================= + +The existing behaviour is unspecified, and treated differently by different +tools. +This PEP makes the expectations explicit. + +There is no known case of backwards incompatibility, but some project out there +probably does rely on details that aren't guaranteed. +This PEP bans the most dangerous of those features, and the rest is +made tool-specific. + + +Security Implications +===================== + +The recommended ``data`` filter is believed safe against common exploits, +and is a single place to amend if flaws are found in the future. + +The explicit specification includes protections from the ``data`` filter. + + +How to Teach This +================= + +The PEP is aimed at authors of packaging tools, who should be fine with +a PEP and an updated packaging spec. + + +Reference Implementation +======================== + +TBD + + +Rejected Ideas +============== + +None yet. + + +Open Issues +=========== + +None yet. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0722.rst b/peps/pep-0722.rst new file mode 100644 index 000000000..56599d99a --- /dev/null +++ b/peps/pep-0722.rst @@ -0,0 +1,719 @@ +PEP: 722 +Title: Dependency specification for single-file scripts +Author: Paul Moore +PEP-Delegate: Brett Cannon +Discussions-To: https://discuss.python.org/t/29905 +Status: Draft +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 19-Jul-2023 +Post-History: `19-Jul-2023 `__ + + +Abstract +======== + +This PEP specifies a format for including 3rd-party dependencies in a +single-file Python script. + + +Motivation +========== + +Not all Python code is structured as a "project", in the sense of having its own +directory complete with a ``pyproject.toml`` file, and being built into an +installable distribution package. Python is also routinely used as a scripting +language, with Python scripts as a (better) alternative to shell scripts, batch +files, etc. When used to create scripts, Python code is typically stored as a +single file, often in a directory dedicated to such "utility scripts", which +might be in a mix of languages with Python being only one possibility among +many. Such scripts may be shared, often by something as simple as email, or a +link to a URL such as a Github gist. But they are typically *not* "distributed" +or "installed" as part of a normal workflow. + +One problem when using Python as a scripting language in this way is how to run +the script in an environment that contains whatever third party dependencies are +required by the script. There is currently no standard tool that addresses this +issue, and this PEP does *not* attempt to define one. However, any tool that +*does* address this issue will need to know what 3rd party dependencies a script +requires. By defining a standard format for storing such data, existing tools, +as well as any future tools, will be able to obtain that information without +requiring users to include tool-specific metadata in their scripts. + + +Rationale +========= + +Because a key requirement is writing single-file scripts, and simple sharing by +giving someone a copy of the script, the PEP defines a mechanism for embedding +dependency data *within the script itself*, and not in an external file. + +We define the concept of a *dependency block* that contains information about +what 3rd party packages a script depends on. + +In order to identify dependency blocks, the script can simply be read as a text +file. This is deliberate, as Python syntax changes over time, so attempting to +parse the script as Python code would require choosing a specific version of +Python syntax. Also, it is likely that at least some tools will not be written +in Python, and expecting them to implement a Python parser is too much of a +burden. + +However, to avoid needing changes to core Python, the format is designed to +appear as comments to the Python parser. It is possible to write code where a +dependency block is *not* interpreted as a comment (for example, by embedding it +in a Python multi-line string), but such uses are discouraged and can easily be +avoided assuming you are not deliberately trying to create a pathological +example. + +A `review `_ of how other languages allow scripts to specify +their dependencies shows that a "structured comment" like this is a +commonly-used approach. + +Specification +============= + +The content of this section will be published in the Python Packaging user +guide, PyPA Specifications section, as a document with the title "Embedding +Metadata in Script Files". + +Any Python script may contain a *dependency block*. The dependency block is +identified by reading the script *as a text file* (i.e., the file is not parsed +as Python source code), looking for the first line of the form:: + + # Script Dependencies: + +The hash character must be at the start of the line with no preceding whitespace. +The text "Script Dependencies" is recognised regardless of case, and the spaces +represent arbitrary whitespace (although at least one space must be present). The +following regular expression recognises the dependency block header line:: + + (?i)^#\s+script\s+dependencies:\s*$ + +Tools reading the dependency block MAY respect the standard Python encoding +declaration. If they choose not to do so, they MUST process the file as UTF-8. + +After the header line, all lines in the file up to the first line that doesn't +start with a ``#`` sign are considered *dependency lines* and are treated as +follows: + +1. The initial ``#`` sign is stripped. +2. If the line contains the character sequence " # " (SPACE HASH SPACE), then + those characters and any subsequent characters are discarded. This allows + dependency blocks to contain inline comments. +3. Whitespace at the start and end of the remaining text is discarded. +4. If the line is now empty, it is ignored. +5. The content of the line MUST now be a valid :pep:`508` dependency specifier. + +The requirement for spaces before and after the ``#`` in an inline comment is +necessary to distinguish them from part of a :pep:`508` URL specifier (which +can contain a hash, but without surrounding whitespace). + +Consumers MUST validate that at a minimum, all dependencies start with a +``name`` as defined in :pep:`508`, and they MAY validate that all dependencies +conform fully to :pep:`508`. They MUST fail with an error if they find an +invalid specifier. + +Example +------- + +The following is an example of a script with an embedded dependency block:: + + # In order to run, this script needs the following 3rd party libraries + # + # Script Dependencies: + # requests + # rich # Needed for the output + # + # # Not needed - just to show that fragments in URLs do not + # # get treated as comments + # pip @ https://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686 + + import requests + from rich.pretty import pprint + + resp = requests.get("https://peps.python.org/api/peps.json") + data = resp.json() + pprint([(k, v["title"]) for k, v in data.items()][:10]) + + +Backwards Compatibility +======================= + +As dependency blocks take the form of a structured comment, they can be added +without altering the meaning of existing code. + +It is possible that a comment may already exist which matches the form of a +dependency block. While the identifying header text, "Script Dependencies" is +chosen to minimise this risk, it is still possible. + +In the rare case where an existing comment would be interpreted incorrectly as a +dependency block, this can be addressed by adding an actual dependency block +(which can be empty if the script has no dependencies) earlier in the code. + + +Security Implications +===================== + +If a script containing a dependency block is run using a tool that automatically +installs dependencies, this could cause arbitrary code to be downloaded and +installed in the user's environment. + +The risk here is part of the functionality of the tool being used to run the +script, and as such should already be addressed by the tool itself. The only +additional risk introduced by this PEP is if an untrusted script with a +dependency block is run, when a potentially malicious dependency might be +installed. This risk is addressed by the normal good practice of reviewing code +before running it. + + +How to Teach This +================= + +The format is intended to be close to how a developer might already specify +script dependencies in an explanatory comment. The required structure is +deliberately minimal, so that formatting rules are easy to learn. + +Users will need to know how to write Python dependency specifiers. This is +covered by :pep:`508`, but for simple examples (which is expected to be the norm +for inexperienced users) the syntax is either just a package name, or a name and +a version restriction, which is fairly well-understood syntax. + +Users will also know how to *run* a script using a tool that interprets +dependency data. This is not covered by this PEP, as it is the responsibility of +such a tool to document how it should be used. + +Note that the core Python interpreter does *not* interpret dependency blocks. +This may be a point of confusion for beginners, who try to run ``python +some_script.py`` and do not understand why it fails. This is no different than +the current status quo, though, where running a script without its dependencies +present will give an error. + +In general, it is assumed that if a beginner is given a script with dependencies +(regardless of whether they are specified in a dependency block), the person +supplying the script should explain how to run that script, and if that involves +using a script runner tool, that should be noted. + + +Recommendations +=============== + +This section is non-normative and simply describes "good practices" when using +dependency blocks. + +While it is permitted for tools to do minimal validation of requirements, in +practice they should do as much "sanity check" validation as possible, even if +they cannot do a full check for :pep:`508` syntax. This helps to ensure that +dependency blocks that are not correctly terminated are reported early. A good +compromise between the minimal approach of checking just that the requirement +starts with a name, and full :pep:`508` validation, is to check for a bare name, +or a name followed by optional whitespace, and then one of ``[`` (extra), ``@`` +(urlspec), ``;`` (marker) or one of ``(~`` (version). + +Scripts should, in general, place the dependency block at the top of the file, +either immediately after any shebang line, or straight after the script +docstring. In particular, the dependency block should always be placed before +any executable code in the file. This makes it easy for the human reader to +locate it. + + +Reference Implementation +======================== + +Code to implement this proposal in Python is fairly straightforward, so the +reference implementation can be included here. + +.. code:: python + + import re + import tokenize + from packaging.requirements import Requirement + + DEPENDENCY_BLOCK_MARKER = r"(?i)^#\s+script\s+dependencies:\s*$" + + def read_dependency_block(filename): + # Use the tokenize module to handle any encoding declaration. + with tokenize.open(filename) as f: + # Skip lines until we reach a dependency block (OR EOF). + for line in f: + if re.match(DEPENDENCY_BLOCK_MARKER, line): + break + # Read dependency lines until we hit a line that doesn't + # start with #, or we are at EOF. + for line in f: + if not line.startswith("#"): + break + # Remove comments. An inline comment is introduced by + # a hash, which must be preceded and followed by a + # space. + line = line[1:].split(" # ", maxsplit=1)[0] + line = line.strip() + # Ignore empty lines + if not line: + continue + # Try to convert to a requirement. This will raise + # an error if the line is not a PEP 508 requirement + yield Requirement(line) + + +A format similar to the one proposed here is already supported `in pipx +`__ and in `pip-run +`__. + + +Rejected Ideas +============== + +Why not include other metadata? +------------------------------- + +The core use case addressed by this proposal is that of identifying what +dependencies a standalone script needs in order to run successfully. This is a +common real-world issue that is currently solved by script runner tools, using +implementation-specific ways of storing the data. Standardising the storage +format improves interoperability by not typing the script to a particular +runner. + +While it is arguable that other forms of metadata could be useful in a +standalone script, the need is largely theoretical at this point. In practical +terms, scripts either don't use other metadata, or they store it in existing, +widely used (and therefore de facto standard) formats. For example, scripts +needing README style text typically use the standard Python module docstring, +and scripts wanting to declare a version use the common convention of having a +``__version__`` variable. + +One case which was raised during the discussion on this PEP, was the ability to +declare a minimum Python version that a script needed to run, by analogy with +the ``Requires-Python`` core metadata item for packages. Unlike packages, +scripts are normally only run by one user or in one environment, in contexts +where multiple versions of Python are uncommon. The need for this metadata is +therefore much less critical in the case of scripts. As further evidence of +this, the two key script runners currently available, ``pipx`` and ``pip-run`` +do not offer a means of including this data in a script. + +Creating a standard "metadata container" format would unify the various +approaches, but in practical terms there is no real need for unification, and +the disruption would either delay adoption, or more likely simply mean script +authors would ignore the standard. + +This proposal therefore chooses to focus just on the one use case where there is +a clear need for something, and no existing standard or common practice. + + +Why not use a marker per line? +------------------------------ + +Rather than using a comment block with a header, another possibility would be to +use a marker on each line, something like:: + + # Script-Dependency: requests + # Script-Dependency: click + +While this makes it easier to parse lines individually, it has a number of +issues. The first is simply that it's rather verbose, and less readable. This is +clearly affected by the chosen keyword, but all of the suggested options were +(in the author's opinion) less readable than the block comment form. + +More importantly, this form *by design* makes it impossible to require that the +dependency specifiers are all together in a single block. As a result, it's not +possible for a human reader, without a careful check of the whole file, to be +sure that they have identified all of the dependencies. See the question below, +"Why not allow multiple dependency blocks and merge them?", for further +discussion of this problem. + +Finally, as the reference implementation demonstrates, parsing the "comment +block" form isn't, in practice, significantly more difficult than parsing this +form. + + +Why not use a distinct form of comment for the dependency block? +---------------------------------------------------------------- + +A previous version of this proposal used ``##`` to identify dependency blocks. +Unfortunately, however, the flake8 linter implements a rule requiring that +comments must have a space after the initial ``#`` sign. While the PEP author +considers that rule misguided, it is on by default and as a result would cause +checks to fail when faced with a dependency block. + +Furthermore, the ``black`` formatter, although it allows the ``##`` form, does +add a space after the ``#`` for most other forms of comment. This means that if +we chose an alternative like ``#%``, automatic reformatting would corrupt the +dependency block. Forms including a space, like ``# #`` are possible, but less +natural for the average user (omitting the space is an obvious mistake to make). + +While it is possible that linters and formatters could be changed to recognise +the new standard, the benefit of having a dedicated prefix did not seem +sufficient to justify the transition cost, or the risk that users might be using +older tools. + + +Why not allow multiple dependency blocks and merge them? +-------------------------------------------------------- + +Because it's too easy for the human reader to miss the fact that there's a +second dependency block. This could simply result in the script runner +unexpectedly downloading extra packages, or it could even be a way to smuggle +malicious packages onto a user's machine (by "hiding" a second dependency block +in the body of the script). + +While the principle of "don't run untrusted code" applies here, the benefits +aren't sufficient to be worth the risk. + + +Why not use a more standard data format (e.g., TOML)? +----------------------------------------------------- + +First of all, the only practical choice for an alternative format is TOML. +Python packaging has standardised on TOML for structured data, and using a +different format, such as YAML or JSON, would add complexity and confusion for +no real benefit. + +So the question is essentially, "why not use TOML?" + +The key idea behind the "dependency block" format is to define something that +reads naturally as a comment in the script. Dependency data is useful both for +tools and for the human reader, so having a human readable format is beneficial. +On the other hand, TOML of necessity has a syntax of its own, which distracts +from the underlying data. + +It is important to remember that developers who *write* scripts in Python are +often *not* experienced in Python, or Python packaging. They are often systems +administrators, or data analysts, who may simply be using Python as a "better +batch file". For such users, the TOML format is extremely likely to be +unfamiliar, and the syntax will be obscure to them, and not particularly +intuitive. Such developers may well be copying dependency specifiers from +sources such as Stack Overflow, without really understanding them. Having to +embed such a requirement into a TOML structure is an additional complexity -- +and it is important to remember that the goal here is to make using 3rd party +libraries *easy* for such users. + +Furthermore, TOML, by its nature, is a flexible format intended to support very +general data structures. There are *many* ways of writing a simple list of +strings in it, and it will not be clear to inexperienced users which form to use. + +Another potential issue is that using a generalised TOML parser can `in some cases +`__ +result in a measurable performance overhead. Startup time is often quoted as an +issue when running small scripts, so this may be a problem for script runners that +are aiming for high performance. + +And finally, there will be tools that expect to *write* dependency data into +scripts -- for example, an IDE with a feature that automatically adds an import +and a dependency specifier when you reference a library function. While +libraries exist that allow editing TOML data, they are not always good at +preserving the user's layout. Even if libraries exist which do an effective job +at this, expecting all tools to use such a library is a significant imposition +on code supporting this PEP. + +By choosing a simple, line-based format with no quoting rules, dependency data +is easy to read (for humans and tools) and easy to write. The format doesn't +have the flexibility of something like TOML, but the use case simply doesn't +demand that sort of flexibility. + + +Why not use (possibly restricted) Python syntax? +------------------------------------------------ + +This would typically involve storing the dependencies as a (runtime) list +variable with a conventional name, such as:: + + __requires__ = [ + "requests", + "click", + ] + +Other suggestions include a static multi-line string, or including the +dependencies in the script's docstring. + +The most significant problem with this proposal is that it requires all +consumers of the dependency data to implement a Python parser. Even if the +syntax is restricted, the *rest* of the script will use the full Python syntax, +and trying to define a syntax which can be successfully parsed in isolation from +the surrounding code is likely to be extremely difficult and error-prone. + +Furthermore, Python's syntax changes in every release. If extracting dependency +data needs a Python parser, the parser will need to know which version of Python +the script is written for, and the overhead for a generic tool of having a +parser that can handle *multiple* versions of Python is unsustainable. + +Even if the above issues could be addressed, the format would give the +impression that the data could be altered at runtime. However, this is not the +case in general, and code that tries to do so will encounter unexpected and +confusing behaviour. + +And finally, there is no evidence that having dependency data available at +runtime is of any practical use. Should such a use be found, it is simple enough +to get the data by parsing the source - ``read_dependency_block(__file__)``. + +It is worth noting, though, that the ``pip-run`` utility does implement (an +extended form of) this approach. `Further discussion `_ of +the ``pip-run`` design is available on the project's issue tracker. + + +Why not embed a ``pyproject.toml`` file in the script? +------------------------------------------------------ + +First of all, ``pyproject.toml`` is a TOML based format, so all of the previous +concerns around TOML as a format apply. However, ``pyproject.toml`` is a +standard used by Python packaging, and re-using an existing standard is a +reasonable suggestion that deserves to be addressed on its own merits. + +The first issue is that the suggestion rarely implies that *all* of +``pyproject.toml`` is to be supported for scripts. A script is not intended to +be "built" into any sort of distributable artifact like a wheel (see below for +more on this point), so the ``[build-system]`` section of ``pyproject.toml`` +makes little sense, for example. And while the tool-specific sections of +``pyproject.toml`` might be useful for scripts, it's not at all clear that a +tool like `ruff `__ would want to support per-file +configuration in this way, leading to confusion when users *expect* it to work, +but it doesn't. Furthermore, this sort of tool-specific configuration is just as +useful for individual files in a larger project, so we have to consider what it +would mean to embed a ``pyproject.toml`` into a single file in a larger project +that has its own ``pyproject.toml``. + +In addition, ``pyproject.toml`` is currently focused on projects that are to be +built into wheels. There is `an ongoing discussion `_ +about how to use ``pyproject.toml`` for projects that are not intended to be +built as wheels, and until that question is resolved (which will likely require +some PEPs of its own) it seems premature to be discussing embedding +``pyproject.toml`` into scripts, which are *definitely* not intended to be built +and distributed in that manner. + +The conclusion, therefore (which has been stated explicitly in some, but not +all, cases) is that this proposal is intended to mean that we would embed *part +of* ``pyproject.toml``. Typically this is the ``[project]`` section from +:pep:`621`, or even just the ``dependencies`` item from that section. + +At this point, the first issue is that by framing the proposal as "embedding +``pyproject.toml``", we would be encouraging the sort of confusion discussed in +the previous paragraphs - developers will expect the full capabilities of +``pyproject.toml``, and be confused when there are differences and limitations. +It would be better, therefore, to consider this suggestion as simply being a +proposal to use an embedded TOML format, but specifically re-using the +*structure* of a particular part of ``pyproject.toml``. The problem then becomes +how we describe that structure, *without* causing confusion for people familiar +with ``pyproject.toml``. If we describe it with reference to ``pyproject.toml``, +the link is still there. But if we describe it in isolation, people will be +confused by the "similar but different" nature of the structure. + +It is also important to remember that a key part of the target audience for this +proposal is developers who are simply using Python as a "better batch file" +solution. These developers will generally not be familiar with Python packaging +and its conventions, and are often the people most critical of the "complexity" +and "difficulty" of packaging solutions. As a result, proposals based on those +existing solutions are likely to be unwelcome to that audience, and could easily +result in people simply continuing to use existing adhoc solutions, and ignoring +the standard that was intended to make their lives easier. + +Why not infer the requirements from import statements? +------------------------------------------------------ + +The idea would be to automatically recognize ``import`` statements in the source +file and turn them into a list of requirements. + +However, this is infeasible for several reasons. First, the points above about +the necessity to keep the syntax easily parsable, for all Python versions, also +by tools written in other languages, apply equally here. + +Second, PyPI and other package repositories conforming to the Simple Repository +API do not provide a mechanism to resolve package names from the module names +that are imported (see also `this related discussion `_). + +Third, even if repositories did offer this information, the same import name may +correspond to several packages on PyPI. One might object that disambiguating +which package is wanted would only be needed if there are several projects +providing the same import name. However, this would make it easy for anyone to +unintentionally or malevolently break working scripts, by uploading a package to +PyPI providing an import name that is the same as an existing project. The +alternative where, among the candidates, the first package to have been +registered on the index is chosen, would be confusing in case a popular package +is developed with the same import name as an existing obscure package, and even +harmful if the existing package is malware intentionally uploaded with a +sufficiently generic import name that has a high probability of being reused. + +A related idea would be to attach the requirements as comments to the import +statements instead of gathering them in a block, with a syntax such as:: + + import numpy as np # requires: numpy + import rich # requires: rich + +This still suffers from parsing difficulties. Also, where to place the comment +in the case of multiline imports is ambiguous and may look ugly:: + + from PyQt5.QtWidgets import ( + QCheckBox, QComboBox, QDialog, QDialogButtonBox, + QGridLayout, QLabel, QSpinBox, QTextEdit + ) # requires: PyQt5 + +Furthermore, this syntax cannot behave as might be intuitively expected +in all situations. Consider:: + + import platform + if platform.system() == "Windows": + import pywin32 # requires: pywin32 + +Here, the user's intent is that the package is only required on Windows, but +this cannot be understood by the script runner (the correct way to write +it would be ``requires: pywin32 ; sys_platform == 'win32'``). + +(Thanks to Jean Abou-Samra for the clear discussion of this point) + + +Why not simply manage the environment at runtime? +------------------------------------------------- + +Another approach to running scripts with dependencies is simply to manage those +dependencies at runtime. This can be done by using a library that makes packages +available. There are many options for implementing such a library, for example +by installing them directly into the user's environment or by manipulating +``sys.path`` to make them available from a local cache. + +These approaches are not incompatible with this PEP. An API such as + +.. code:: python + + env_mgr.install("rich") + env_mgr.install("click") + + import rich + import click + + ... + +is certainly feasible. However, such a library could be written without the need +for any new standards, and as far as the PEP author is aware, this has not +happened. This suggests that an approach like this is not as attractive as it +first seems. There is also the bootstrapping issue of making the ``env_mgr`` +library available in the first place. And finally, this approach doesn't +actually offer any interoperability benefits, as it does not use a standard form +for the dependency list, and so other tools cannot access the data. + +In any case, such a library could still benefit from this proposal, as it could +include an API to read the packages to install from the script dependency block. +This would give the same functionality while allowing interoperability with +other tools that support this specification. + +.. code:: python + + # Script Dependencies: + # rich + # click + env_mgr.install_dependencies(__file__) + + import rich + import click + + ... + + +Why not just set up a Python project with a ``pyproject.toml``? +--------------------------------------------------------------- + +Again, a key issue here is that the target audience for this proposal is people +writing scripts which aren't intended for distribution. Sometimes scripts will +be "shared", but this is far more informal than "distribution" - it typically +involves sending a script via an email with some written instructions on how to +run it, or passing someone a link to a gist. + +Expecting such users to learn the complexities of Python packaging is a +significant step up in complexity, and would almost certainly give the +impression that "Python is too hard for scripts". + +In addition, if the expectation here is that the ``pyproject.toml`` will somehow +be designed for running scripts in place, that's a new feature of the standard +that doesn't currently exist. At a minimum, this isn't a reasonable suggestion +until the `current discussion on Discourse `_ about +using ``pyproject.toml`` for projects that won't be distributed as wheels is +resolved. And even then, it doesn't address the "sending someone a script in a +gist or email" use case. + +Why not use a requirements file for dependencies? +------------------------------------------------- + +Putting your requirements in a requirements file, doesn't require a PEP. You can +do that right now, and in fact it's quite likely that many adhoc solutions do +this. However, without a standard, there's no way of knowing how to locate a +script's dependency data. And furthermore, the requirements file format is +pip-specific, so tools relying on it are depending on a pip implementation +detail. + +So in order to make a standard, two things would be required: + +1. A standardised replacement for the requirements file format. +2. A standard for how to locate the requirements file for a given script. + +The first item is a significant undertaking. It has been discussed on a number +of occasions, but so far no-one has attempted to actually do it. The most likely +approach would be for standards to be developed for individual use cases +currently addressed with requirements files. One option here would be for this +PEP to simply define a new file format which is simply a text file containing +:pep:`508` requirements, one per line. That would just leave the question of how +to locate that file. + +The "obvious" solution here would be to do something like name the file the same +as the script, but with a ``.reqs`` extension (or something similar). However, +this still requires *two* files, where currently only a single file is needed, +and as such, does not match the "better batch file" model (shell scripts and +batch files are typically self-contained). It requires the developer to remember +to keep the two files together, and this may not always be possible. For +example, system administration policies may require that *all* files in a +certain directory are executable (the Linux filesystem standards require this of +``/usr/bin``, for example). And some methods of sharing a script (for example, +publishing it on a text file sharing service like Github's gist, or a corporate +intranet) may not allow for deriving the location of an associated requirements +file from the script's location (tools like ``pipx`` support running a script +directly from a URL, so "download and unpack a zip of the script and its +dependencies" may not be an appropriate requirement). + +Essentially, though, the issue here is that there is an explicitly stated +requirement that the format supports storing dependency data *in the script file +itself*. Solutions that don't do that are simply ignoring that requirement. + +Should scripts be able to specify a package index? +-------------------------------------------------- + +Dependency metadata is about *what* package the code depends on, and not *where* +that package comes from. There is no difference here between metadata for +scripts, and metadata for distribution packages (as defined in +``pyproject.toml``). In both cases, dependencies are given in "abstract" form, +without specifying how they are obtained. + +Some tools that use the dependency information may, of course, need to locate +concrete dependency artifacts - for example if they expect to create an +environment containing those dependencies. But the way they choose to do that +will be closely linked to the tool's UI in general, and this PEP does not try to +dictate the UI for tools. + +There is more discussion of this point, and in particular of the UI choices made +by the ``pip-run`` tool, in `the previously mentioned pip-run issue `_. + +What about local dependencies? +------------------------------ + +These can be handled without needing special metadata and tooling, simply by +adding the location of the dependencies to ``sys.path``. This PEP simply isn't +needed for this case. If, on the other hand, the "local dependencies" are actual +distributions which are published locally, they can be specified as usual with a +:pep:`508` requirement, and the local package index specified when running a +tool by using the tool's UI for that. + +Open Issues +=========== + +None at this point. + + +References +========== + +.. _pip-run issue: https://github.com/jaraco/pip-run/issues/44 +.. _language survey: https://dbohdan.com/scripts-with-dependencies +.. _pyproject without wheels: https://discuss.python.org/t/projects-that-arent-meant-to-generate-a-wheel-and-pyproject-toml/29684 +.. _import-names: https://discuss.python.org/t/record-the-top-level-names-of-a-wheel-in-metadata/29494 + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0723.rst b/peps/pep-0723.rst new file mode 100644 index 000000000..90170af41 --- /dev/null +++ b/peps/pep-0723.rst @@ -0,0 +1,788 @@ +PEP: 723 +Title: Embedding pyproject.toml in single-file scripts +Author: Ofek Lev +Sponsor: Adam Turner +PEP-Delegate: Brett Cannon +Discussions-To: https://discuss.python.org/t/31151 +Status: Draft +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 04-Aug-2023 +Post-History: `04-Aug-2023 `__, + `06-Aug-2023 `__, + `23-Aug-2023 `__, +Replaces: 722 + + +Abstract +======== + +This PEP specifies a metadata format that can be embedded in single-file Python +scripts to assist launchers, IDEs and other external tools which may need to +interact with such scripts. + + +Motivation +========== + +Python is routinely used as a scripting language, with Python scripts as a +(better) alternative to shell scripts, batch files, etc. When Python code is +structured as a script, it is usually stored as a single file and does not +expect the availability of any other local code that may be used for imports. +As such, it is possible to share with others over arbitrary text-based means +such as email, a URL to the script, or even a chat window. Code that is +structured like this may live as a single file forever, never becoming a +full-fledged project with its own directory and ``pyproject.toml`` file. + +An issue that users encounter with this approach is that there is no standard +mechanism to define metadata for tools whose job it is to execute such scripts. +For example, a tool that runs a script may need to know which dependencies are +required or the supported version(s) of Python. + +There is currently no standard tool that addresses this issue, and this PEP +does *not* attempt to define one. However, any tool that *does* address this +issue will need to know what the runtime requirements of scripts are. By +defining a standard format for storing such metadata, existing tools, as well +as any future tools, will be able to obtain that information without requiring +users to include tool-specific metadata in their scripts. + + +Rationale +========= + +This PEP defines a mechanism for embedding metadata *within the script itself*, +and not in an external file. + +We choose to follow the latest developments of other modern packaging +ecosystems (namely `Go`__ and provisionally `Rust`__) by embedding the existing +file used to describe projects, in our case ``pyproject.toml``. + +__ https://github.com/erning/gorun +__ https://rust-lang.github.io/rfcs/3424-cargo-script.html + +The format is intended to bridge the gap between different types of users +of Python. Users will benefit from seamless interoperability with tools that +already work with TOML. + +One of the central themes we discovered from the recent +`packaging survey `__ is that users have +begun getting frustrated with the lack of unification regarding both tooling +and specs. Adding yet another metadata format (like :pep:`722` syntax for a +list of dependencies), even for a currently unsatisfied use case, would +further fragment the community. + +The following are some of the use cases that this PEP wishes to support: + +* A user facing CLI that is capable of executing scripts. If we take Hatch as + an example, the interface would be simply + ``hatch run /path/to/script.py [args]`` and Hatch will manage the + environment for that script. Such tools could be used as shebang lines on + non-Windows systems e.g. ``#!/usr/bin/env hatch run``. +* A script that desires to transition to a directory-type project. A user may + be rapidly prototyping locally or in a remote REPL environment and then + decide to transition to a more formal project layout if their idea works + out. This intermediate script stage would be very useful to have fully + reproducible bug reports. By using the same format, the user can simply copy + and paste the metadata into a ``pyproject.toml`` file and continue working + without having to learn a new format. More likely, even, is that tooling will + eventually support this transformation with a single command. +* Users that wish to avoid manual dependency management. For example, package + managers that have commands to add/remove dependencies or dependency update + automation in CI that triggers based on new versions or in response to + CVEs [1]_. + + +Specification +============= + +This PEP defines a metadata comment block format loosely inspired [2]_ by +`reStructuredText Directives`__. + +__ https://docutils.sourceforge.io/docs/ref/rst/directives.html + +Any Python script may have top-level comment blocks that start with the line +``# /// TYPE`` where ``TYPE`` determines how to process the content, and ends +with the line ``# ///``. Every line between these two lines MUST be a comment +starting with ``#``. If there are characters after the ``#`` then the first +character MUST be a space. The embedded content is formed by taking away the +first two characters of each line if the second character is a space, otherwise +just the first character (which means the line consists of only a single +``#``). + +When there are multiple comment blocks of the same ``TYPE`` defined, tools MUST +produce an error. + +Tools reading embedded metadata MAY respect the standard Python encoding +declaration. If they choose not to do so, they MUST process the file as UTF-8. + +This is the canonical regular expression that MAY be used to parse the +metadata: + +.. code:: text + + (?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$ + +In circumstances where there is a discrepancy between the text specification +and the regular expression, the text specification takes precedence. + +Tools MUST NOT read from metadata blocks with types that have not been +standardized by this PEP or future ones. + +pyproject type +-------------- + +The first type of metadata block is named ``pyproject`` which represents +content similar to [3]_ what one would see in a ``pyproject.toml`` file. + +This document MAY include the ``[run]`` and ``[tool]`` tables. + +The :pep:`tool table <518#tool-table>` MAY be used by any tool, script runner +or otherwise, to configure behavior. + +The ``[run]`` table MAY include the following optional fields: + +* ``dependencies``: A list of strings that specifies the runtime dependencies + of the script. Each entry MUST be a valid :pep:`508` dependency. +* ``requires-python``: A string that specifies the Python version(s) with which + the script is compatible. The value of this field MUST be a valid + :pep:`version specifier <440#version-specifiers>`. + +Any future PEPs that define additional fields for the ``[run]`` table when used +in a ``pyproject.toml`` file MUST include the aforementioned fields exactly as +specified. The fields defined by this PEP are equally as applicable to +full-fledged projects as they are to single-file scripts. + +Script runners MUST error if the specified ``dependencies`` cannot be provided. +Script runners SHOULD error if no version of Python that satisfies the specified +``requires-python`` can be provided. + +Example +------- + +The following is an example of a script with an embedded ``pyproject.toml``: + +.. code:: python + + # /// pyproject + # [run] + # requires-python = ">=3.11" + # dependencies = [ + # "requests<3", + # "rich", + # ] + # /// + + import requests + from rich.pretty import pprint + + resp = requests.get("https://peps.python.org/api/peps.json") + data = resp.json() + pprint([(k, v["title"]) for k, v in data.items()][:10]) + +The following [4]_ is an example of a proposed syntax for single-file Rust +projects that embeds their equivalent of ``pyproject.toml``, which is called +``Cargo.toml``: + +.. code:: rust + + #!/usr/bin/env cargo + + //! ```cargo + //! [dependencies] + //! regex = "1.8.0" + //! ``` + + fn main() { + let re = Regex::new(r"^\d{4}-\d{2}-\d{2}$").unwrap(); + println!("Did our date match? {}", re.is_match("2014-01-01")); + } + +Reference Implementation +======================== + +The following is an example of how to read the metadata on Python 3.11 or +higher. + +.. code:: python + + import re + import tomllib + + REGEX = r'(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$' + + def read(script: str) -> dict | None: + name = 'pyproject' + matches = list( + filter(lambda m: m.group('type') == name, re.finditer(REGEX, script)) + ) + if len(matches) > 1: + raise ValueError(f'Multiple {name} blocks found') + elif len(matches) == 1: + return tomllib.loads(matches[0]) + else: + return None + +Often tools will edit dependencies like package managers or dependency update +automation in CI. The following is a crude example of modifying the content +using the ``tomlkit`` library__. + +__ https://tomlkit.readthedocs.io/en/latest/ + +.. code:: python + + import re + + import tomlkit + + REGEX = r'(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$' + + def add(script: str, dependency: str) -> str: + match = re.search(REGEX, script) + content = ''.join( + line[2:] if line.startswith('# ') else line[1:] + for line in match.group('content').splitlines(keepends=True) + ) + + config = tomlkit.parse(content) + config['project']['dependencies'].append(dependency) + new_content = ''.join( + f'# {line}' if line.strip() else f'#{line}' + for line in tomlkit.dumps(config).splitlines(keepends=True) + ) + + start, end = match.span('content') + return script[:start] + new_content + script[end:] + +Note that this example used a library that preserves TOML formatting. This is +not a requirement for editing by any means but rather is a "nice to have" +feature. + + +Backwards Compatibility +======================= + +At the time of writing, the ``# /// pyproject`` block comment starter does not +appear `on GitHub`__. Therefore, there is little risk of existing scripts being +broken by this PEP. + +__ https://github.com/search?q=%22%23+%2F%2F%2F+pyproject%22&type=code + + +Security Implications +===================== + +If a script containing embedded metadata is ran using a tool that automatically +installs dependencies, this could cause arbitrary code to be downloaded and +installed in the user's environment. + +The risk here is part of the functionality of the tool being used to run the +script, and as such should already be addressed by the tool itself. The only +additional risk introduced by this PEP is if an untrusted script with a +embedded metadata is run, when a potentially malicious dependency or transitive +dependency might be installed. + +This risk is addressed by the normal good practice of reviewing code +before running it. Additionally, tools may be able to provide +`locking functionality <723-tool-configuration_>`__ to ameliorate this risk. + + +How to Teach This +================= + +To embed metadata in a script, define a comment block that starts with the +line ``# /// pyproject`` and ends with the line ``# ///``. Every line between +those two lines must be a comment and the full content is derived by removing +the first two characters. The ``pyproject`` type indicates that the content +is TOML and resembles a ``pyproject.toml`` file. + +.. code:: python + + # /// pyproject + # [run] + # dependencies = [ + # "requests<3", + # "rich", + # ] + # requires-python = ">=3.11" + # /// + +The two allowed tables are ``[run]`` and ``[tool]``. The ``[run]`` table may +contain the following fields: + +.. list-table:: + + * - Field + - Description + - Tool behavior + + * - ``dependencies`` + - A list of strings that specifies the runtime dependencies of the script. + Each entry must be a valid :pep:`508` dependency. + - Tools will error if the specified dependencies cannot be provided. + + * - ``requires-python`` + - A string that specifies the Python version(s) + with which the script is compatible. + The value of this field must be a valid + :pep:`version specifier <440#version-specifiers>`. + - Tools might error if no version of Python that satisfies + the constraint can be executed. + +It is up to individual tools whether or not their behavior is altered based on +the embedded metadata. For example, every script runner may not be able to +provide an environment for specific Python versions as defined by the +``requires-python`` field. + +The :pep:`tool table <518#tool-table>` may be used by any tool, script runner +or otherwise, to configure behavior. + + +Recommendations +=============== + +Tools that support managing different versions of Python should attempt to use +the highest available version of Python that is compatible with the script's +``requires-python`` metadata, if defined. + + +Tooling buy-in +============== + +The following is a list of tools that have expressed support for this PEP or +have committed to implementing support should it be accepted: + +* `Pantsbuild and Pex `__: expressed + support for any way to define dependencies and also features that this PEP + considers as valid use cases such as building packages from scripts and + embedding tool configuration +* `Mypy `__ and + `Ruff `__: strongly expressed support + for embedding tool configuration as it would solve existing pain points for + users +* `Hatch `__: (author of this PEP) + expressed support for all aspects of this PEP, and will be one of the first + tools to support running scripts with specifically configured Python versions + + +Rejected Ideas +============== + +.. _723-comment-block: + +Why not use a comment block resembling requirements.txt? +-------------------------------------------------------- + +This PEP considers there to be different types of users for whom Python code +would live as single-file scripts: + +* Non-programmers who are just using Python as a scripting language to achieve + a specific task. These users are unlikely to be familiar with concepts of + operating systems like shebang lines or the ``PATH`` environment variable. + Some examples: + + * The average person, perhaps at a workplace, who wants to write a script to + automate something for efficiency or to reduce tedium + * Someone doing data science or machine learning in industry or academia who + wants to write a script to analyze some data or for research purposes. + These users are special in that, although they have limited programming + knowledge, they learn from sources like StackOverflow and blogs that have a + programming bent and are increasingly likely to be part of communities that + share knowledge and code. Therefore, a non-trivial number of these users + will have some familiarity with things like Git(Hub), Jupyter, HuggingFace, + etc. +* Non-programmers who manage operating systems e.g. a sysadmin. These users are + able to set up ``PATH``, for example, but are unlikely to be familiar with + Python concepts like virtual environments. These users often operate in + isolation and have limited need to gain exposure to tools intended for + sharing like Git. +* Programmers who manage operating systems/infrastructure e.g. SREs. These + users are not very likely to be familiar with Python concepts like virtual + environments, but are likely to be familiar with Git and most often use it + to version control everything required to manage infrastructure like Python + scripts and Kubernetes config. +* Programmers who write scripts primarily for themselves. These users over time + accumulate a great number of scripts in various languages that they use to + automate their workflow and often store them in a single directory, that is + potentially version controlled for persistence. Non-Windows users may set + up each Python script with a shebang line pointing to the desired Python + executable or script runner. + +This PEP argues that reusing our TOML-based metadata format is the best for +each category of user and that the requirements-like block comment is only +approachable for those who have familiarity with ``requirements.txt``, which +represents a small subset of users. + +* For the average person automating a task or the data scientist, they are + already starting with zero context and are unlikely to be familiar with + TOML nor ``requirements.txt``. These users will very likely rely on + snippets found online via a search engine or utilize AI in the form + of a chat bot or direct code completion software. Searching for Python + metadata formatting will lead them to the TOML-based format that already + exists which they can reuse. The author tested GitHub Copilot with this + PEP and it already supports auto-completion of ``dependencies``. In contrast, + a new format may take years of being trained on the Internet for models to + learn. + + Additionally, these users are most susceptible to formatting quirks and + syntax errors. TOML is a well-defined format with existing online + validators that features assignment that is compatible with Python + expressions and has no strict indenting rules. The block comment format + on the other hand could be easily malformed by forgetting the colon, for + example, and debugging why it's not working with a search engine would be + a difficult task for such a user. +* For the sysadmin types, they are equally unlikely as the previously described + users to be familiar with TOML or ``requirements.txt``. For either format + they would have to read documentation. They would likely be more comfortable + with TOML since they are used to structured data formats and there would be + less perceived magic in their systems. + + Additionally, for maintenance of their systems ``/// pyproject`` would be + much easier to search for from a shell than a block comment with potentially + numerous extensions over time. +* For the SRE types, they are likely to be familiar with TOML already from + other projects that they might have to work with like configuring the + `GitLab Runner`__ or `Cloud Native Buildpacks`__. + + __ https://docs.gitlab.com/runner/configuration/advanced-configuration.html + __ https://buildpacks.io/docs/reference/config/ + + These users are responsible for the security of their systems and most likely + have security scanners set up to automatically open PRs to update versions + of dependencies. Such automated tools like Dependabot would have a much + easier time using existing TOML libraries than writing their own custom + parser for a block comment format. +* For the programmer types, they are more likely to be familiar with TOML + than they have ever seen a ``requirements.txt`` file, unless they are a + Python programmer who has had previous experience with writing applications. + In the case of experience with the requirements format, it necessarily means + that they are at least somewhat familiar with the ecosystem and therefore + it is safe to assume they know what TOML is. + + Another benefit of this PEP to these users is that their IDEs like Visual + Studio Code would be able to provide TOML syntax highlighting much more + easily than each writing custom logic for this feature. + +Additionally, since the original block comment alternative format (double +``#``) went against the recommendation of :pep:`8` and as a result linters +and IDE auto-formatters that respected the recommendation would +`fail by default `__, the final +proposal uses standard comments starting with a single ``#`` character without +any obvious start nor end sequence. + +The concept of regular comments that do not appear to be intended for machines +(i.e. `encoding declarations`__) affecting behavior would not be customary to +users of Python and goes directly against the "explicit is better than +implicit" foundational principle. + +__ https://docs.python.org/3/reference/lexical_analysis.html#encoding-declarations + +Users typing what to them looks like prose could alter runtime behavior. This +PEP takes the view that the possibility of that happening, even when a tool +has been set up as such (maybe by a sysadmin), is unfriendly to users. + +Finally, and critically, the alternatives to this PEP like :pep:`722` do not +satisfy the use cases enumerated herein, such as setting the supported Python +versions, the eventual building of scripts into packages, and the ability to +have machines edit metadata on behalf of users. It is very likely that the +requests for such features persist and conceivable that another PEP in the +future would allow for the embedding of such metadata. At that point there +would be multiple ways to achieve the same thing which goes against our +foundational principle of "there should be one - and preferably only one - +obvious way to do it". + +Why not use a multi-line string? +-------------------------------- + +A previous version of this PEP proposed that the metadata be stored as follows: + +.. code:: python + + __pyproject__ = """ + ... + """ + +The most significant problem with this proposal is that the embedded TOML would +be limited in the following ways: + +* It would not be possible to use multi-line double-quoted strings in the TOML + as that would conflict with the Python string containing the document. Many + TOML writers do not preserve style and may potentially produce output that + would be malformed. +* The way in which character escaping works in Python strings is not quite the + way it works in TOML strings. It would be possible to preserve a one-to-one + character mapping by enforcing raw strings, but this ``r`` prefix requirement + may be potentially confusing to users. + +Why not reuse core metadata fields? +----------------------------------- + +A previous version of this PEP proposed to reuse the existing +`metadata standard `_ that is used to describe projects. + +There are two significant problems with this proposal: + +* The ``name`` and ``version`` fields are required and changing that would + require its own PEP +* Reusing the data is `fundamentally a misuse of it`__ + + __ https://snarky.ca/differentiating-between-writing-down-dependencies-to-use-packages-and-for-packages-themselves/ + +Why not limit to specific metadata fields? +------------------------------------------ + +By limiting the metadata to just ``dependencies``, we would prevent the known +use case of tools that support managing Python installations, which would +allows users to target specific versions of Python for new syntax or standard +library functionality. + +.. _723-tool-configuration: + +Why not limit tool configuration? +--------------------------------- + +By not allowing the ``[tool]`` table, we would prevent known functionality +that would benefit users. For example: + +* A script runner may support injecting of dependency resolution data for an + embedded lock file (this is what Go's ``gorun`` can do). +* A script runner may support configuration instructing to run scripts in + containers for situations in which there is no cross-platform support for a + dependency or if the setup is too complex for the average user like when + requiring Nvidia drivers. Situations like this would allow users to proceed + with what they want to do whereas otherwise they may stop at that point + altogether. +* Tools may wish to experiment with features to ease development burden for + users such as the building of single-file scripts into packages. We received + `feedback `__ stating that there are + already tools that exist in the wild that build wheels and source + distributions from single files. + + The author of the Rust RFC for embedding metadata + `mentioned to us `__ that they are + actively looking into that as well based on user feedback saying that there + is unnecessary friction with managing small projects. + + There has been `a commitment `__ to + support this by at least one major build system. + +Why not limit tool behavior? +---------------------------- + +A previous version of this PEP proposed that non-script running tools SHOULD +NOT modify their behavior when the script is not the sole input to the tool. +For example, if a linter is invoked with the path to a directory, it SHOULD +behave the same as if zero files had embedded metadata. + +This was done as a precaution to avoid tool behavior confusion and generating +various feature requests for tools to support this PEP. However, during +discussion we received `feedback `__ +from maintainers of tools that this would be undesirable and potentially +confusing to users. Additionally, this may allow for a universally easier +way to configure tools in certain circumstances and solve existing issues. + +Why not just set up a Python project with a ``pyproject.toml``? +--------------------------------------------------------------- + +Again, a key issue here is that the target audience for this proposal is people +writing scripts which aren't intended for distribution. Sometimes scripts will +be "shared", but this is far more informal than "distribution" - it typically +involves sending a script via an email with some written instructions on how to +run it, or passing someone a link to a GitHub gist. + +Expecting such users to learn the complexities of Python packaging is a +significant step up in complexity, and would almost certainly give the +impression that "Python is too hard for scripts". + +In addition, if the expectation here is that the ``pyproject.toml`` will +somehow be designed for running scripts in place, that's a new feature of the +standard that doesn't currently exist. At a minimum, this isn't a reasonable +suggestion until the `current discussion on Discourse +`_ about using ``pyproject.toml`` for projects that +won't be distributed as wheels is resolved. And even then, it doesn't address +the "sending someone a script in a gist or email" use case. + +Why not infer the requirements from import statements? +------------------------------------------------------ + +The idea would be to automatically recognize ``import`` statements in the source +file and turn them into a list of requirements. + +However, this is infeasible for several reasons. First, the points above about +the necessity to keep the syntax easily parsable, for all Python versions, also +by tools written in other languages, apply equally here. + +Second, PyPI and other package repositories conforming to the Simple Repository +API do not provide a mechanism to resolve package names from the module names +that are imported (see also `this related discussion`__). + +__ https://discuss.python.org/t/record-the-top-level-names-of-a-wheel-in-metadata/29494 + +Third, even if repositories did offer this information, the same import name may +correspond to several packages on PyPI. One might object that disambiguating +which package is wanted would only be needed if there are several projects +providing the same import name. However, this would make it easy for anyone to +unintentionally or malevolently break working scripts, by uploading a package to +PyPI providing an import name that is the same as an existing project. The +alternative where, among the candidates, the first package to have been +registered on the index is chosen, would be confusing in case a popular package +is developed with the same import name as an existing obscure package, and even +harmful if the existing package is malware intentionally uploaded with a +sufficiently generic import name that has a high probability of being reused. + +A related idea would be to attach the requirements as comments to the import +statements instead of gathering them in a block, with a syntax such as:: + + import numpy as np # requires: numpy + import rich # requires: rich + +This still suffers from parsing difficulties. Also, where to place the comment +in the case of multiline imports is ambiguous and may look ugly:: + + from PyQt5.QtWidgets import ( + QCheckBox, QComboBox, QDialog, QDialogButtonBox, + QGridLayout, QLabel, QSpinBox, QTextEdit + ) # requires: PyQt5 + +Furthermore, this syntax cannot behave as might be intuitively expected +in all situations. Consider:: + + import platform + if platform.system() == "Windows": + import pywin32 # requires: pywin32 + +Here, the user's intent is that the package is only required on Windows, but +this cannot be understood by the script runner (the correct way to write +it would be ``requires: pywin32 ; sys_platform == 'win32'``). + +(Thanks to Jean Abou-Samra for the clear discussion of this point) + +Why not use a requirements file for dependencies? +------------------------------------------------- + +Putting your requirements in a requirements file, doesn't require a PEP. You +can do that right now, and in fact it's quite likely that many adhoc solutions +do this. However, without a standard, there's no way of knowing how to locate a +script's dependency data. And furthermore, the requirements file format is +pip-specific, so tools relying on it are depending on a pip implementation +detail. + +So in order to make a standard, two things would be required: + +1. A standardised replacement for the requirements file format. +2. A standard for how to locate the requirements file for a given script. + +The first item is a significant undertaking. It has been discussed on a number +of occasions, but so far no-one has attempted to actually do it. The most +likely approach would be for standards to be developed for individual use cases +currently addressed with requirements files. One option here would be for this +PEP to simply define a new file format which is simply a text file containing +:pep:`508` requirements, one per line. That would just leave the question of +how to locate that file. + +The "obvious" solution here would be to do something like name the file the +same as the script, but with a ``.reqs`` extension (or something similar). +However, this still requires *two* files, where currently only a single file is +needed, and as such, does not match the "better batch file" model (shell +scripts and batch files are typically self-contained). It requires the +developer to remember to keep the two files together, and this may not always +be possible. For example, system administration policies may require that *all* +files in a certain directory are executable (the Linux filesystem standards +require this of ``/usr/bin``, for example). And some methods of sharing a +script (for example, publishing it on a text file sharing service like Github's +gist, or a corporate intranet) may not allow for deriving the location of an +associated requirements file from the script's location (tools like ``pipx`` +support running a script directly from a URL, so "download and unpack a zip of +the script and its dependencies" may not be an appropriate requirement). + +Essentially, though, the issue here is that there is an explicitly stated +requirement that the format supports storing dependency data *in the script +file itself*. Solutions that don't do that are simply ignoring that +requirement. + +Why not use (possibly restricted) Python syntax? +------------------------------------------------ + +This would typically involve storing metadata as multiple special variables, +such as the following. + +.. code:: python + + __requires_python__ = ">=3.11" + __dependencies__ = [ + "requests", + "click", + ] + +The most significant problem with this proposal is that it requires all +consumers of the dependency data to implement a Python parser. Even if the +syntax is restricted, the *rest* of the script will use the full Python syntax, +and trying to define a syntax which can be successfully parsed in isolation +from the surrounding code is likely to be extremely difficult and error-prone. + +Furthermore, Python's syntax changes in every release. If extracting dependency +data needs a Python parser, the parser will need to know which version of +Python the script is written for, and the overhead for a generic tool of having +a parser that can handle *multiple* versions of Python is unsustainable. + +With this approach there is the potential to clutter scripts with many +variables as new extensions get added. Additionally, intuiting which metadata +fields correspond to which variable names would cause confusion for users. + +It is worth noting, though, that the ``pip-run`` utility does implement (an +extended form of) this approach. `Further discussion `_ of +the ``pip-run`` design is available on the project's issue tracker. + +What about local dependencies? +------------------------------ + +These can be handled without needing special metadata and tooling, simply by +adding the location of the dependencies to ``sys.path``. This PEP simply isn't +needed for this case. If, on the other hand, the "local dependencies" are +actual distributions which are published locally, they can be specified as +usual with a :pep:`508` requirement, and the local package index specified when +running a tool by using the tool's UI for that. + +Open Issues +=========== + +None at this point. + + +References +========== + +.. _pyproject metadata: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ +.. _pip-run issue: https://github.com/jaraco/pip-run/issues/44 +.. _pyproject without wheels: https://discuss.python.org/t/projects-that-arent-meant-to-generate-a-wheel-and-pyproject-toml/29684 + + +Footnotes +========= + +.. [1] A large number of users use scripts that are version controlled. For + example, `the SREs that were mentioned <723-comment-block_>`_ or + projects that require special maintenance like the + `AWS CLI `__ + or `Calibre `__. +.. [2] The syntax is taken directly from the final resolution of the + `Blocks extension`__ to `Python Markdown`__. + + __ https://github.com/facelessuser/pymdown-extensions/discussions/1973 + __ https://github.com/Python-Markdown/markdown +.. [3] A future PEP that officially introduces the ``[run]`` table to + ``pyproject.toml`` files will make this PEP not just similar but a strict + subset. +.. [4] One important thing to note is that the metadata is embedded in a + `doc-comment`__ (their equivalent of docstrings). `Other syntaxes`__ are + under consideration within the Rust project. + + __ https://doc.rust-lang.org/stable/book/ch14-02-publishing-to-crates-io.html#making-useful-documentation-comments + __ https://github.com/epage/cargo-script-mvs/blob/main/0000-cargo-script.md#embedded-manifest-format + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0724.rst b/peps/pep-0724.rst new file mode 100644 index 000000000..a6ae5a1d2 --- /dev/null +++ b/peps/pep-0724.rst @@ -0,0 +1,331 @@ +PEP: 724 +Title: Stricter Type Guards +Author: Rich Chiodo , + Eric Traut , + Erik De Bonte , +Sponsor: Jelle Zijlstra +Discussions-To: https://discuss.python.org/t/pep-724-stricter-type-guards/34124 +Status: Draft +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 28-Jul-2023 +Python-Version: 3.13 +Post-History: `30-Dec-2021 `__, + `19-Sep-2023 `__, + +Abstract +======== + +:pep:`647` introduced the concept of a user-defined type guard function which +returns ``True`` if the type of the expression passed to its first parameter +matches its return ``TypeGuard`` type. For example, a function that has a +return type of ``TypeGuard[str]`` is assumed to return ``True`` if and only if +the type of the expression passed to its first input parameter is a ``str``. +This allows type checkers to narrow types when a user-defined type guard +function returns ``True``. + +This PEP refines the ``TypeGuard`` mechanism introduced in :pep:`647`. It +allows type checkers to narrow types when a user-defined type guard function +returns ``False``. It also allows type checkers to apply additional (more +precise) type narrowing under certain circumstances when the type guard +function returns ``True``. + + +Motivation +========== + +User-defined type guard functions enable a type checker to narrow the type of +an expression when it is passed as an argument to the type guard function. The +``TypeGuard`` mechanism introduced in :pep:`647` is flexible, but this +flexibility imposes some limitations that developers have found inconvenient +for some uses. + +Limitation 1: Type checkers are not allowed to narrow a type in the case where +the type guard function returns ``False``. This means the type is not narrowed +in the negative ("else") clause. + +Limitation 2: Type checkers must use the ``TypeGuard`` return type if the type +guard function returns ``True`` regardless of whether additional narrowing can +be applied based on knowledge of the pre-narrowed type. + +The following code sample demonstrates both of these limitations. + +.. code-block:: python + + def is_iterable(val: object) -> TypeGuard[Iterable[Any]]: + return isinstance(val, Iterable) + + def func(val: int | list[int]): + if is_iterable(val): + # The type is narrowed to 'Iterable[Any]' as dictated by + # the TypeGuard return type + reveal_type(val) # Iterable[Any] + else: + # The type is not narrowed in the "False" case + reveal_type(val) # int | list[int] + + # If "isinstance" is used in place of the user-defined type guard + # function, the results differ because type checkers apply additional + # logic for "isinstance" + + if isinstance(val, Iterable): + # Type is narrowed to "list[int]" because this is + # a narrower (more precise) type than "Iterable[Any]" + reveal_type(val) # list[int] + else: + # Type is narrowed to "int" because the logic eliminates + # "list[int]" from the original union + reveal_type(val) # int + + +:pep:`647` imposed these limitations so it could support use cases where the +return ``TypeGuard`` type was not a subtype of the input type. Refer to +:pep:`647` for examples. + + +Specification +============= + +The use of a user-defined type guard function involves five types: + +* I = ``TypeGuard`` input type +* R = ``TypeGuard`` return type +* A = Type of argument passed to type guard function (pre-narrowed) +* NP = Narrowed type (positive) +* NN = Narrowed type (negative) + +.. code-block:: python + + def guard(x: I) -> TypeGuard[R]: ... + + def func1(val: A): + if guard(val): + reveal_type(val) # NP + else: + reveal_type(val) # NN + + +This PEP proposes some modifications to :pep:`647` to address the limitations +discussed above. These limitations are safe to eliminate only when a specific +condition is met. In particular, when the output type ``R`` of a user-defined +type guard function is consistent [#isconsistent]_ with the type of its first +input parameter (``I``), type checkers should apply stricter type guard +semantics. + + .. code-block:: python + + # Stricter type guard semantics are used in this case because + # "Kangaroo | Koala" is consistent with "Animal" + def is_marsupial(val: Animal) -> TypeGuard[Kangaroo | Koala]: + return isinstance(val, Kangaroo | Koala) + + # Stricter type guard semantics are not used in this case because + # "list[T]"" is not consistent with "list[T | None]" + def has_no_nones(val: list[T | None]) -> TypeGuard[list[T]]: + return None not in val + +When stricter type guard semantics are applied, the application of a +user-defined type guard function changes in two ways. + +* Type narrowing is applied in the negative ("else") case. + +.. code-block:: python + + def is_str(val: str | int) -> TypeGuard[str]: + return isinstance(val, str) + + def func(val: str | int): + if not is_str(val): + reveal_type(val) # int + +* Additional type narrowing is applied in the positive "if" case if applicable. + +.. code-block:: python + + def is_cardinal_direction(val: str) -> TypeGuard[Literal["N", "S", "E", "W"]]: + return val in ("N", "S", "E", "W") + + def func(direction: Literal["NW", "E"]): + if is_cardinal_direction(direction): + reveal_type(direction) # "Literal[E]" + else: + reveal_type(direction) # "Literal[NW]" + + +The type-theoretic rules for type narrowing are specificed in the following +table. + +============ ======================= =================== +\ Non-strict type guard Strict type guard +============ ======================= =================== +Applies when R not consistent with I R consistent with I +NP is .. :math:`R` :math:`A \land R` +NN is .. :math:`A` :math:`A \land \neg{R}` +============ ======================= =================== + +In practice, the theoretic types for strict type guards cannot be expressed +precisely in the Python type system. Type checkers should fall back on +practical approximations of these types. As a rule of thumb, a type checker +should use the same type narrowing logic -- and get results that are consistent +with -- its handling of "isinstance". This guidance allows for changes and +improvements if the type system is extended in the future. + + +Additional Examples +=================== + +``Any`` is consistent [#isconsistent]_ with any other type, which means +stricter semantics can be applied. + +.. code-block:: python + + # Stricter type guard semantics are used in this case because + # "str" is consistent with "Any" + def is_str(x: Any) -> TypeGuard[str]: + return isinstance(x, str) + + def test(x: float | str): + if is_str(x): + reveal_type(x) # str + else: + reveal_type(x) # float + + +Backwards Compatibility +======================= + +This PEP proposes to change the existing behavior of ``TypeGuard``. This has no +effect at runtime, but it does change the types evaluated by a type checker. + +.. code-block:: python + + def is_int(val: int | str) -> TypeGuard[int]: + return isinstance(val, int) + + def func(val: int | str): + if is_int(val): + reveal_type(val) # "int" + else: + reveal_type(val) # Previously "int | str", now "str" + + +This behavioral change results in different types evaluated by a type checker. +It could therefore produce new (or mask existing) type errors. + +Type checkers often improve narrowing logic or fix existing bugs in such logic, +so users of static typing will be used to this type of behavioral change. + +We also hypothesize that it is unlikely that existing typed Python code relies +on the current behavior of ``TypeGuard``. To validate our hypothesis, we +implemented the proposed change in pyright and ran this modified version on +roughly 25 typed code bases using `mypy primer`__ to see if there were any +differences in the output. As predicted, the behavioral change had minimal +impact. The only noteworthy change was that some ``# type: ignore`` comments +were no longer necessary, indicating that these code bases were already working +around the existing limitations of ``TypeGuard``. + +__ https://github.com/hauntsaninja/mypy_primer + +Breaking change +--------------- + +It is possible for a user-defined type guard function to rely on the old +behavior. Such type guard functions could break with the new behavior. + +.. code-block:: python + + def is_positive_int(val: int | str) -> TypeGuard[int]: + return isinstance(val, int) and val > 0 + + def func(val: int | str): + if is_positive_int(val): + reveal_type(val) # "int" + else: + # With the older behavior, the type of "val" is evaluated as + # "int | str"; with the new behavior, the type is narrowed to + # "str", which is perhaps not what was intended. + reveal_type(val) + +We think it is unlikley that such user-defined type guards exist in real-world +code. The mypy primer results didn't uncover any such cases. + + +How to Teach This +================= + +Users unfamiliar with ``TypeGuard`` are likely to expect the behavior outlined +in this PEP, therefore making ``TypeGuard`` easier to teach and explain. + + +Reference Implementation +======================== + +A reference `implementation`__ of this idea exists in pyright. + +__ https://github.com/microsoft/pyright/commit/9a5af798d726bd0612cebee7223676c39cf0b9b0 + +To enable the modified behavior, the configuration flag +``enableExperimentalFeatures`` must be set to true. This can be done on a +per-file basis by adding a comment: + +.. code-block:: python + + # pyright: enableExperimentalFeatures=true + + +Rejected Ideas +============== + +StrictTypeGuard +--------------- + +A new ``StrictTypeGuard`` construct was proposed. This alternative form would +be similar to a ``TypeGuard`` except it would apply stricter type guard +semantics. It would also enforce that the return type was consistent +[#isconsistent]_ with the input type. See this thread for details: +`StrictTypeGuard proposal`__ + +__ https://github.com/python/typing/discussions/1013#discussioncomment-1966238 + +This idea was rejected because it is unnecessary in most cases and added +unnecessary complexity. It would require the introduction of a new special +form, and developers would need to be educated about the subtle difference +between the two forms. + +TypeGuard with a second output type +----------------------------------- + +Another idea was proposed where ``TypeGuard`` could support a second optional +type argument that indicates the type that should be used for narrowing in the +negative ("else") case. + +.. code-block:: python + + def is_int(val: int | str) -> TypeGuard[int, str]: + return isinstance(val, int) + + +This idea was proposed `here`__. + +__ https://github.com/python/typing/issues/996 + +It was rejected because it was considered too complicated and addressed only +one of the two main limitations of ``TypeGuard``. Refer to this `thread`__ for +the full discussion. + +__ https://mail.python.org/archives/list/typing-sig@python.org/thread/EMUD2D424OI53DCWQ4H5L6SJD2IXBHUL + + +Footnotes +========= + +.. [#isconsistent] :pep:`PEP 483's discussion of is-consistent <483#summary-of-gradual-typing>` + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + diff --git a/peps/pep-0725.rst b/peps/pep-0725.rst new file mode 100644 index 000000000..172afbd86 --- /dev/null +++ b/peps/pep-0725.rst @@ -0,0 +1,587 @@ +PEP: 725 +Title: Specifying external dependencies in pyproject.toml +Author: Pradyun Gedam , + Ralf Gommers +Discussions-To: https://discuss.python.org/t/31888 +Status: Draft +Type: Standards Track +Topic: Packaging +Content-Type: text/x-rst +Created: 17-Aug-2023 +Post-History: `18-Aug-2023 `__ + + +Abstract +======== + +This PEP specifies how to write a project's external, or non-PyPI, build and +runtime dependencies in a ``pyproject.toml`` file for packaging-related tools +to consume. + + +Motivation +========== + +Python packages may have dependencies on build tools, libraries, command-line +tools, or other software that is not present on PyPI. Currently there is no way +to express those dependencies in standardized metadata +[#singular-vision-native-deps]_, [#pypacking-native-deps]_. Key motivators for +this PEP are to: + +- Enable tools to automatically map external dependencies to packages in other + packaging repositories, +- Make it possible to include needed dependencies in error messages emitting by + Python package installers and build frontends, +- Provide a canonical place for package authors to record this dependency + information. + +Packaging ecosystems like Linux distros, Conda, Homebrew, Spack, and Nix need +full sets of dependencies for Python packages, and have tools like pyp2rpm_ +(Fedora), Grayskull_ (Conda), and dh_python_ (Debian) which attempt to +automatically generate dependency metadata from the metadata in +upstream Python packages. External dependencies are currently handled manually, +because there is no metadata for this in ``pyproject.toml`` or any other +standard location. Enabling automating this conversion is a key benefit of +this PEP, making packaging Python easier and more reliable. In addition, the +authors envision other types of tools making use of this information, e.g., +dependency analysis tools like Repology_, Dependabot_ and libraries.io_. +Software bill of materials (SBOM) generation tools may also be able to use this +information, e.g. for flagging that external dependencies listed in +``pyproject.toml`` but not contained in wheel metadata are likely vendored +within the wheel. + +Packages with external dependencies are typically hard to build from source, +and error messages from build failures tend to be hard to decipher for end +users. Missing external dependencies on the end user's system are the most +likely cause of build failures. If installers can show the required external +dependencies as part of their error message, this may save users a lot of time. + +At the moment, information on external dependencies is only captured in +installation documentation of individual packages. It is hard to maintain for +package authors and tends to go out of date. It's also hard for users and +distro packagers to find it. Having a canonical place to record this dependency +information will improve this situation. + +This PEP is not trying to specify how the external dependencies should be used, +nor a mechanism to implement a name mapping from names of individual packages +that are canonical for Python projects published on PyPI to those of other +packaging ecosystems. Those topics should be addressed in separate PEPs. + + +Rationale +========= + +Types of external dependencies +------------------------------ + +Multiple types of external dependencies can be distinguished: + +- Concrete packages that can be identified by name and have a canonical + location in another language-specific package repository. E.g., Rust + packages on `crates.io `__, R packages on + `CRAN `__, JavaScript packages on the + `npm registry `__. +- Concrete packages that can be identified by name but do not have a clear + canonical location. This is typically the case for libraries and tools + written in C, C++, Fortran, CUDA and other low-level languages. E.g., + Boost, OpenSSL, Protobuf, Intel MKL, GCC. +- "Virtual" packages, which are names for concepts, types of tools or + interfaces. These typically have multiple implementations, which *are* + concrete packages. E.g., a C++ compiler, BLAS, LAPACK, OpenMP, MPI. + +Concrete packages are straightforward to understand, and are a concept present +in virtually every package management system. Virtual packages are a concept +also present in a number of packaging systems -- but not always, and the +details of their implementation varies. + +Cross compilation +----------------- + +Cross compilation is not yet (as of August 2023) well-supported by stdlib +modules and ``pyproject.toml`` metadata. It is however important when +translating external dependencies to those of other packaging systems (with +tools like ``pyp2rpm``). Introducing support for cross compilation immediately +in this PEP is much easier than extending ``[external]`` in the future, hence +the authors choose to include this now. + +Terminology +''''''''''' + +This PEP uses the following terminology: + +- *build machine*: the machine on which the package build process is being + executed +- *host machine*: the machine on which the produced artifact will be installed + and run +- *build dependency*: dependency for building the package that needs to be + present at build time and itself was built for the build machine's OS and + architecture +- *host dependency*: dependency for building the package that needs to be + present at build time and itself was built for the host machine's OS and + architecture + +Note that this terminology is not consistent across build and packaging tools, +so care must be taken when comparing build/host dependencies in +``pyproject.toml`` to dependencies from other package managers. + +Note that "target machine" or "target dependency" is not used in this PEP. That +is typically only relevant for cross-compiling compilers or other such advanced +scenarios [#gcc-cross-terminology]_, [#meson-cross]_ - this is out of scope for +this PEP. + +Finally, note that while "dependency" is the term most widely used for packages +needed at build time, the existing key in ``pyproject.toml`` for PyPI +build-time dependencies is ``build-requires``. Hence this PEP uses the keys +``build-requires`` and ``host-requires`` under ``[external]`` for consistency. + +Build and host dependencies +''''''''''''''''''''''''''' + +Clear separation of metadata associated with the definition of build and target +platforms, rather than assuming that build and target platform will always be +the same, is important [#pypackaging-native-cross]_. + +Build dependencies are typically run during the build process - they may be +compilers, code generators, or other such tools. In case the use of a build +dependency implies a runtime dependency, that runtime dependency does not have +to be declared explicitly. For example, when compiling Fortran code with +``gfortran`` into a Python extension module, the package likely incurs a +dependency on the ``libgfortran`` runtime library. The rationale for not +explicitly listing such runtime dependencies is two-fold: (1) it may depend on +compiler/linker flags or details of the build environment whether the +dependency is present, and (2) these runtime dependencies can be detected and +handled automatically by tools like ``auditwheel``. + +Host dependencies are typically not run during the build process, but only used +for linking against. This is not a rule though -- it may be possible or +necessary to run a host dependency under an emulator, or through a custom tool +like crossenv_. When host dependencies imply a runtime dependency, that runtime +dependency also does not have to be declared, just like for build dependencies. + +When host dependencies are declared and a tool is not cross-compilation aware +and has to do something with external dependencies, the tool MAY merge the +``host-requires`` list into ``build-requires``. This may for example happen if +an installer like ``pip`` starts reporting external dependencies as a likely +cause of a build failure when a package fails to build from an sdist. + +Specifying external dependencies +-------------------------------- + +Concrete package specification through PURL +''''''''''''''''''''''''''''''''''''''''''' + +The two types of concrete packages are supported by PURL_ (Package URL), which +implements a scheme for identifying packages that is meant to be portable +across packaging ecosystems. Its design is:: + + scheme:type/namespace/name@version?qualifiers#subpath + +The ``scheme`` component is a fixed string, ``pkg``, and of the other +components only ``type`` and ``name`` are required. As an example, a package +URL for the ``requests`` package on PyPI would be:: + + pkg:pypi/requests + +Adopting PURL to specify external dependencies in ``pyproject.toml`` solves a +number of problems at once - and there are already implementations of the +specification in Python and multiple languages. PURL is also already supported +by dependency-related tooling like SPDX (see +`External Repository Identifiers in the SPDX 2.3 spec `__), +the `Open Source Vulnerability format `__, +and the `Sonatype OSS Index `__; +not having to wait years before support in such tooling arrives is valuable. + +For concrete packages without a canonical package manager to refer to, either +``pkg:generic/pkg-name`` can be used, or a direct reference to the VCS system +that the package is maintained in (e.g., +``pkg:github/user-or-org-name/pkg-name``). Which of these is more appropriate +is situation-dependent. This PEP recommends using ``pkg:generic`` when the +package name is unambiguous and well-known (e.g., ``pkg:generic/git`` or +``pkg:generic/openblas``), and using the VCS as the PURL type otherwise. + +Virtual package specification +''''''''''''''''''''''''''''' + +There is no ready-made support for virtual packages in PURL or another +standard. There are a relatively limited number of such dependencies though, +and adoption a scheme similar to PURL but with the ``virtual:`` rather than +``pkg:`` scheme seems like it will be understandable and map well to Linux +distros with virtual packages and the likes of Conda and Spack. + +The two known virtual package types are ``compiler`` and ``interface``. + +Versioning +'''''''''' + +Support in PURL for version expressions and ranges beyond a fixed version is +still pending, see the Open Issues section. + +Dependency specifiers +''''''''''''''''''''' + +Regular Python dependency specifiers (as originally defined in :pep:`508`) may +be used behind PURLs. PURL qualifiers, which use ``?`` followed by a package +type-specific dependency specifier component, must not be used. The reason for +this is pragmatic: dependency specifiers are already used for other metadata in +``pyproject.toml``, any tooling that is used with ``pyproject.toml`` is likely +to already have a robust implementation to parse it. And we do not expect to +need the extra possibilities that PURL qualifiers provide (e.g. to specify a +Conan or Conda channel, or a RubyGems platform). + +Usage of core metadata fields +----------------------------- + +The `core metadata`_ specification contains one relevant field, namely +``Requires-External``. This has no well-defined semantics in core metadata 2.1; +this PEP chooses to reuse the field for external runtime dependencies. The core +metadata specification does not contain fields for any metadata in +``pyproject.toml``'s ``[build-system]`` table. Therefore the ``build-requires`` +and ``host-requires`` content also does not need to be reflected in core +metadata fields. The ``optional-dependencies`` content from ``[external]`` +would need to either reuse ``Provides-Extra`` or require a new +``Provides-External-Extra`` field. Neither seems desirable. + +Differences between sdist and wheel metadata +'''''''''''''''''''''''''''''''''''''''''''' + +A wheel may vendor its external dependencies. This happens in particular when +distributing wheels on PyPI or other Python package indexes - and tools like +auditwheel_, delvewheel_ and delocate_ automate this process. As a result, a +``Requires-External`` entry in an sdist may disappear from a wheel built from +that sdist. It is also possible that a ``Requires-External`` entry remains in a +wheel, either unchanged or with narrower constraints. ``auditwheel`` does not +vendor certain allow-listed dependencies, such as OpenGL, by default. In +addition, ``auditwheel`` and ``delvewheel`` allow a user to manually exclude +dependencies via a ``--exclude`` or ``--no-dll`` command-line flag. This is +used to avoid vendoring large shared libraries, for example those from CUDA. + +``Requires-External`` entries generated from external dependencies in +``pyproject.toml`` in a wheel are therefore allowed to be narrower than those +for the corresponding sdist. They must not be wider, i.e. constraints must not +allow a version of a dependency for a wheel that isn't allowed for an sdist, +nor contain new dependencies that are not listed in the sdist's metadata at +all. + + +Specification +============= + +If metadata is improperly specified then tools MUST raise an error to notify +the user about their mistake. + +Details +------- + +Note that ``pyproject.toml`` content is in the same format as in :pep:`621`. + +Table name +'''''''''' + +Tools MUST specify fields defined by this PEP in a table named ``[external]``. +No tools may add fields to this table which are not defined by this PEP or +subsequent PEPs. The lack of an ``[external]`` table means the package either +does not have any external dependencies, or the ones it does have are assumed +to be present on the system already. + +``build-requires``/``optional-build-requires`` +'''''''''''''''''''''''''''''''''''''''''''''' + +- Format: Array of PURL_ strings (``build-requires``) and a table + with values of arrays of PURL_ strings (``optional-build-requires``) +- `Core metadata`_: N/A + +The (optional) external build requirements needed to build the project. + +For ``build-requires``, it is a key whose value is an array of strings. Each +string represents a build requirement of the project and MUST be formatted as +either a valid PURL_ string or a ``virtual:`` string. + +For ``optional-build-requires``, it is a table where each key specifies an +extra set of build requirements and whose value is an array of strings. The +strings of the arrays MUST be valid PURL_ strings. + +``host-requires``/``optional-host-requires`` +'''''''''''''''''''''''''''''''''''''''''''' + +- Format: Array of PURL_ strings (``host-requires``) and a table + with values of arrays of PURL_ strings (``optional-host-requires``) +- `Core metadata`_: N/A + +The (optional) external host requirements needed to build the project. + +For ``host-requires``, it is a key whose value is an array of strings. Each +string represents a host requirement of the project and MUST be formatted as +either a valid PURL_ string or a ``virtual:`` string. + +For ``optional-host-requires``, it is a table where each key specifies an +extra set of host requirements and whose value is an array of strings. The +strings of the arrays MUST be valid PURL_ strings. + +``dependencies``/``optional-dependencies`` +'''''''''''''''''''''''''''''''''''''''''' + +- Format: Array of PURL_ strings (``dependencies``) and a table + with values of arrays of PURL_ strings (``optional-dependencies``) +- `Core metadata`_: ``Requires-External``, N/A + +The (optional) dependencies of the project. + +For ``dependencies``, it is a key whose value is an array of strings. Each +string represents a dependency of the project and MUST be formatted as either a +valid PURL_ string or a ``virtual:`` string. Each string maps directly to a +``Requires-External`` entry in the `core metadata`_. + +For ``optional-dependencies``, it is a table where each key specifies an extra +and whose value is an array of strings. The strings of the arrays MUST be valid +PURL_ strings. Optional dependencies do not map to a core metadata field. + +Examples +-------- + +These examples show what the ``[external]`` content for a number of packages is +expected to be. + +cryptography 39.0: + +.. code:: toml + + [external] + build-requires = [ + "virtual:compiler/rust", + ] + host-requires = [ + "pkg:generic/openssl", + ] + +SciPy 1.10: + +.. code:: toml + + [external] + build-requires = [ + "virtual:compiler/c", + "virtual:compiler/cpp", + "virtual:compiler/fortran", + "pkg:generic/ninja", + ] + host-requires = [ + "virtual:interface/blas", + "virtual:interface/lapack", # >=3.7.1 (can't express version ranges with PURL yet) + ] + + [external.optional-host-requires] + dependency_detection = [ + "pkg:generic/pkg-config", + "pkg:generic/cmake", + ] + +pygraphviz 1.10: + +.. code:: toml + + [external] + build-requires = [ + "virtual:compiler/c", + ] + host-requires = [ + "pkg:generic/graphviz", + ] + +NAVis 1.4.0: + +.. code:: toml + + [project.optional-dependencies] + r = ["rpy2"] + + [external] + build-requires = [ + "pkg:generic/XCB; platform_system=='Linux'", + ] + + [external.optional-dependencies] + nat = [ + "pkg:cran/nat", + "pkg:cran/nat.nblast", + ] + +Spyder 6.0: + +.. code:: toml + + [external] + dependencies = [ + "pkg:cargo/ripgrep", + "pkg:cargo/tree-sitter-cli", + "pkg:golang/github.com/junegunn/fzf", + ] + +jupyterlab-git 0.41.0: + +.. code:: toml + + [external] + dependencies = [ + "pkg:generic/git", + ] + + [external.optional-build-requires] + dev = [ + "pkg:generic/nodejs", + ] + +PyEnchant 3.2.2: + +.. code:: toml + + [external] + dependencies = [ + # libenchant is needed on all platforms but only vendored into wheels on + # Windows, so on Windows the build backend should remove this external + # dependency from wheel metadata. + "pkg:github/AbiWord/enchant", + ] + + +Backwards Compatibility +======================= + +There is no impact on backwards compatibility, as this PEP only adds new, +optional metadata. In the absence of such metadata, nothing changes for package +authors or packaging tooling. + + +Security Implications +===================== + +There are no direct security concerns as this PEP covers how to statically +define metadata for external dependencies. Any security issues would stem from +how tools consume the metadata and choose to act upon it. + + +How to Teach This +================= + +External dependencies and if and how those external dependencies are vendored +are topics that are typically not understood in detail by Python package +authors. We intend to start from how an external dependency is defined, the +different ways it can be depended on---from runtime-only with ``ctypes`` or a +``subprocess`` call to it being a build dependency that's linked against--- +before going into how to declare external dependencies in metadata. The +documentation should make explicit what is relevant for package authors, and +what for distro packagers. + +Material on this topic will be added to the most relevant packaging tutorials, +primarily the `Python Packaging User Guide`_. In addition, we expect that any +build backend that adds support for external dependencies metadata will include +information about that in its documentation, as will tools like ``auditwheel``. + + +Reference Implementation +======================== + +There is no reference implementation at this time. + + +Rejected Ideas +============== + +Specific syntax for external dependencies which are also packaged on PyPI +------------------------------------------------------------------------- + +There are non-Python packages which are packaged on PyPI, such as Ninja, +patchelf and CMake. What is typically desired is to use the system version of +those, and if it's not present on the system then install the PyPI package for +it. The authors believe that specific support for this scenario is not +necessary (or too complex to justify such support); a dependency provider for +external dependencies can treat PyPI as one possible source for obtaining the +package. + +Using library and header names as external dependencies +------------------------------------------------------- + +A previous draft PEP (`"External dependencies" (2015) `__) +proposed using specific library and header names as external dependencies. This +is too granular; using package names is a well-established pattern across +packaging ecosystems and should be preferred. + + +Open Issues +=========== + +Version specifiers for PURLs +---------------------------- + +Support in PURL for version expressions and ranges is still pending. The pull +request at `vers implementation for PURL`_ seems close to being merged, at +which point this PEP could adopt it. + +Syntax for virtual dependencies +------------------------------- + +The current syntax this PEP uses for virtual dependencies is +``virtual:type/name``, which is analogous to but not part of the PURL spec. +This open issue discusses supporting virtual dependencies within PURL: +`purl-spec#222 `__. + +Should a ``host-requires`` key be added under ``[build-system]``? +----------------------------------------------------------------- + +Adding ``host-requires`` for host dependencies that are on PyPI in order to +better support name mapping to other packaging systems with support for +cross-compiling may make sense. +`This issue `__ tracks this topic +and has arguments in favor and against adding ``host-requires`` under +``[build-system]`` as part of this PEP. + + +References +========== + +.. [#singular-vision-native-deps] The "define native requirements metadata" + part of the "Wanting a singular packaging vision" thread (2022, Discourse): + https://discuss.python.org/t/wanting-a-singular-packaging-tool-vision/21141/92 + +.. [#pypacking-native-deps] pypackaging-native: "Native dependencies" + https://pypackaging-native.github.io/key-issues/native-dependencies/ + +.. [#gcc-cross-terminology] GCC documentation - Configure Terms and History, + https://gcc.gnu.org/onlinedocs/gccint/Configure-Terms.html + +.. [#meson-cross] Meson documentation - Cross compilation + https://mesonbuild.com/Cross-compilation.html + +.. [#pypackaging-native-cross] pypackaging-native: "Cross compilation" + https://pypackaging-native.github.io/key-issues/cross_compilation/ + +.. [#pkgconfig-and-ctypes-findlibrary] The "``pkgconfig`` specification as an + alternative to ``ctypes.util.find_library``" thread (2023, Discourse): + https://discuss.python.org/t/pkgconfig-specification-as-an-alternative-to-ctypes-util-find-library/31379 + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + + +.. _PyPI: https://pypi.org +.. _core metadata: https://packaging.python.org/specifications/core-metadata/ +.. _setuptools: https://setuptools.readthedocs.io/ +.. _setuptools metadata: https://setuptools.readthedocs.io/en/latest/setuptools.html#metadata +.. _SPDX: https://spdx.dev/ +.. _PURL: https://github.com/package-url/purl-spec/ +.. _vers: https://github.com/package-url/purl-spec/blob/version-range-spec/VERSION-RANGE-SPEC.rst +.. _vers implementation for PURL: https://github.com/package-url/purl-spec/pull/139 +.. _pyp2rpm: https://github.com/fedora-python/pyp2rpm +.. _Grayskull: https://github.com/conda/grayskull +.. _dh_python: https://www.debian.org/doc/packaging-manuals/python-policy/index.html#dh-python +.. _Repology: https://repology.org/ +.. _Dependabot: https://github.com/dependabot +.. _libraries.io: https://libraries.io/ +.. _crossenv: https://github.com/benfogle/crossenv +.. _Python Packaging User Guide: https://packaging.python.org +.. _auditwheel: https://github.com/pypa/auditwheel +.. _delocate: https://github.com/matthew-brett/delocate +.. _delvewheel: https://github.com/adang1345/delvewheel diff --git a/peps/pep-0726.rst b/peps/pep-0726.rst new file mode 100644 index 000000000..44cd3d9fa --- /dev/null +++ b/peps/pep-0726.rst @@ -0,0 +1,226 @@ +PEP: 726 +Title: Module ``__setattr__`` and ``__delattr__`` +Author: Sergey B Kirpichev +Sponsor: Adam Turner +Discussions-To: https://discuss.python.org/t/32640/ +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 24-Aug-2023 +Python-Version: 3.13 +Post-History: `06-Apr-2023 `__, + `31-Aug-2023 `__, + + +Abstract +======== + +This PEP proposes supporting user-defined ``__setattr__`` +and ``__delattr__`` methods on modules to extend customization +of module attribute access beyond :pep:`562`. + +Motivation +========== + +There are several potential uses of a module ``__setattr__``: + +1. To prevent setting an attribute at all (i.e. make it read-only) +2. To validate the value to be assigned +3. To intercept setting an attribute and update some other state + +Proper support for read-only attributes would also require adding the +``__delattr__`` function to prevent their deletion. + +A typical workaround is assigning the ``__class__`` of a module object to a +custom subclass of :py:class:`python:types.ModuleType` (see [1]_). +Unfortunately, this also brings a noticeable speed regression +(~2-3x) for attribute *access*. It would be convenient to directly +support such customization, by recognizing ``__setattr__`` and ``__delattr__`` +methods defined in a module that would act like normal +:py:meth:`python:object.__setattr__` and :py:meth:`python:object.__delattr__` +methods, except that they will be defined on module *instances*. + +For example + +.. code:: python + + # mplib.py + + CONSTANT = 3.14 + prec = 53 + dps = 15 + + def dps_to_prec(n): + """Return the number of bits required to represent n decimals accurately.""" + return max(1, int(round((int(n)+1)*3.3219280948873626))) + + def prec_to_dps(n): + """Return the number of accurate decimals that can be represented with n bits.""" + return max(1, int(round(int(n)/3.3219280948873626)-1)) + + def validate(n): + n = int(n) + if n <= 0: + raise ValueError('non-negative integer expected') + return n + + def __setattr__(name, value): + if name == 'CONSTANT': + raise AttributeError('Read-only attribute!') + if name == 'dps': + value = validate(value) + globals()['dps'] = value + globals()['prec'] = dps_to_prec(value) + return + if name == 'prec': + value = validate(value) + globals()['prec'] = value + globals()['dps'] = prec_to_dps(value) + return + globals()[name] = value + + def __delattr__(name): + if name in ('CONSTANT', 'dps', 'prec'): + raise AttributeError('Read-only attribute!') + del globals()[name] + +.. code:: pycon + + >>> import mplib + >>> mplib.foo = 'spam' + >>> mplib.CONSTANT = 42 + Traceback (most recent call last): + ... + AttributeError: Read-only attribute! + >>> del mplib.foo + >>> del mplib.CONSTANT + Traceback (most recent call last): + ... + AttributeError: Read-only attribute! + >>> mplib.prec + 53 + >>> mplib.dps + 15 + >>> mplib.dps = 5 + >>> mplib.prec + 20 + >>> mplib.dps = 0 + Traceback (most recent call last): + ... + ValueError: non-negative integer expected + + +Specification +============= + +The ``__setattr__`` function at the module level should accept two +arguments, the name of an attribute and the value to be assigned, +and return :py:obj:`None` or raise an :exc:`AttributeError`. + +.. code:: python + + def __setattr__(name: str, value: typing.Any, /) -> None: ... + +The ``__delattr__`` function should accept one argument, +the name of an attribute, and return :py:obj:`None` or raise an +:py:exc:`AttributeError`: + +.. code:: python + + def __delattr__(name: str, /) -> None: ... + +The ``__setattr__`` and ``__delattr__`` functions are looked up in the +module ``__dict__``. If present, the appropriate function is called to +customize setting the attribute or its deletion, else the normal +mechanism (storing/deleting the value in the module dictionary) will work. + +Defining module ``__setattr__`` or ``__delattr__`` only affects lookups made +using the attribute access syntax---directly accessing the module globals +(whether by ``globals()`` within the module, or via a reference to the module's +globals dictionary) is unaffected. For example: + +.. code:: pycon + + >>> import mod + >>> mod.__dict__['foo'] = 'spam' # bypasses __setattr__, defined in mod.py + +or + +.. code:: python + + # mod.py + + def __setattr__(name, value): + ... + + foo = 'spam' # bypasses __setattr__ + globals()['bar'] = 'spam' # here too + + def f(): + global x + x = 123 + + f() # and here + +To use a module global and trigger ``__setattr__`` (or ``__delattr__``), +one can access it via ``sys.modules[__name__]`` within the module's code: + +.. code:: python + + # mod.py + + sys.modules[__name__].foo = 'spam' # bypasses __setattr__ + + def __setattr__(name, value): + ... + + sys.modules[__name__].bar = 'spam' # triggers __setattr__ + + +How to Teach This +================= + +The "Customizing module attribute access" [1]_ section of the documentation +will be expanded to include new functions. + + +Reference Implementation +======================== + +The reference implementation for this PEP can be found in `CPython PR #108261 +`__. + + +Backwards compatibility +======================= + +This PEP may break code that uses module level (global) names +``__setattr__`` and ``__delattr__``, but the language reference +explicitly reserves *all* undocumented dunder names, and allows +"breakage without warning" [2]_. + +The performance implications of this PEP are small, since additional +dictionary lookup is much cheaper than storing/deleting the value in +the dictionary. Also it is hard to imagine a module that expects the +user to set (and/or delete) attributes enough times to be a +performance concern. On another hand, proposed mechanism allows to +override setting/deleting of attributes without affecting speed of +attribute access, which is much more likely scenario to get a +performance penalty. + + +Footnotes +========= + +.. [1] Customizing module attribute access + (https://docs.python.org/3.11/reference/datamodel.html#customizing-module-attribute-access) + +.. [2] Reserved classes of identifiers + (https://docs.python.org/3.11/reference/lexical_analysis.html#reserved-classes-of-identifiers) + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0727.rst b/peps/pep-0727.rst new file mode 100644 index 000000000..d8f900415 --- /dev/null +++ b/peps/pep-0727.rst @@ -0,0 +1,518 @@ +PEP: 727 +Title: Documentation Metadata in Typing +Author: SebastiĂĄn RamĂ­rez +Sponsor: Jelle Zijlstra +Discussions-To: https://discuss.python.org/t/32566 +Status: Draft +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 28-Aug-2023 +Python-Version: 3.13 +Post-History: `30-Aug-2023 `__ + + +Abstract +======== + +This document proposes a way to complement docstrings to add additional documentation +to Python symbols using type annotations with :py:class:`~typing.Annotated` +(in class attributes, function and method parameters, return values, and variables). + + +Motivation +========== + +The current standard method of documenting code APIs in Python is using docstrings. +But there's no standard way to document parameters in docstrings. + +There are several pseudo-standards for the format in these docstrings, and new +pseudo-standards can appear easily: numpy, Google, Keras, reST, etc. + +All these formats are some specific syntax inside a string. Because of this, when +editing those docstrings, editors can't easily provide support for autocompletion, +inline errors for broken syntax, etc. + +Editors don't have a way to support all the possible micro languages in the docstrings +and show nice user interfaces when developers use libraries with those different +formats. They could even add support for some of the syntaxes, but probably not all, +or not completely. + +Because the docstring is in a different place in the code than the actual parameters +and it requires duplication of information (the parameter name) the information about +a parameter is easily in a place in the code quite far away from the declaration of +the actual parameter. This means it's easy to refactor a function, remove a parameter, +and forget to remove its docs. The same happens when adding a new parameter: it's easy +to forget to add the docstring for it. + +And because of this same duplication of information (the parameter name) editors and +other tools would need to have complex custom logic to check or ensure the +consistency of the parameters in the signature and in their docstring, or they +would simply not be able to support that. + +Additionally, it would be difficult to robustly parse varying existing docstring +conventions to programatically get the documentation for each individual parameter +or variable at runtime. This would be useful, for example, +for testing the contents of each parameter's documentation, to ensure consistency +across several similar functions, or to extract and expose that same parameter +documentation in some other way (e.g. an API, a CLI, etc). + +Some of these previous formats tried to account for the lack of type annotations +in older Python versions by including typing information in the docstrings, +but now that information doesn't need to be in docstrings as there is now an official +:pep:`syntax for type annotations <484>`. + + +Rationale +========= + +This proposal intends to address these shortcomings by extending and complementing the +information in docstrings, keeping backwards compatibility with existing docstrings, +and doing it in a way that leverages the Python language and structure, via type +annotations with ``Annotated``, and a new function in ``typing``. + +The reason why this would belong in the standard Python library instead of an +external package is because although the implementation would be quite trivial, +the actual power and benefit from it would come from being a standard, so that +editors and other tools could implement support for it. + +This doesn't deprecate current usage of docstrings, it's transparent to common +developers (library users), and it's only opt-in for library authors that would +like to adopt it. + + +Specification +============= + +The main proposal is to introduce a new function, ``typing.doc()``, +to be used when documenting Python objects. +This function MUST only be used within :py:class:`~typing.Annotated` annotations. +The function takes a single string argument, ``documentation``, +and returns an instance of ``typing.DocInfo``, +which stores the input string unchanged. + +Any tool processing ``typing.DocInfo`` objects SHOULD interpret the string as +a docstring, and therefore SHOULD normalize whitespace +as if ``inspect.cleandoc()`` were used. + +The string passed to ``typing.doc()`` SHOULD be of the form that would be a valid docstring. +This means that `f-strings`__ and string operations SHOULD NOT be used. +As this cannot be enforced by the Python runtime, +tools SHOULD NOT rely on this behaviour, +and SHOULD exit with an error if such a prohibited string is encountered. + +__ https://docs.python.org/3/reference/lexical_analysis.html#formatted-string-literals + +Examples +-------- + +Class attributes may be documented: + +.. code:: python + + from typing import Annotated, doc + + class User: + first_name: Annotated[str, doc("The user's first name")] + last_name: Annotated[str, doc("The user's last name")] + + ... + +As can function or method parameters: + +.. code:: python + + from typing import Annotated, doc + + def create_user( + first_name: Annotated[str, doc("The user's first name")], + last_name: Annotated[str, doc("The user's last name")], + cursor: DatabaseConnection | None = None, + ) -> Annotated[User, doc("The created user after saving in the database")]: + """Create a new user in the system. + + It needs the database connection to be already initialized. + """ + pass + + +Additional Scenarios +-------------------- + +The main scenarios that this proposal intends to cover are described above, and +for implementers to be conformant to this specification, they only need to support +those scenarios described above. + +Here are some additional edge case scenarios with their respective considerations, +but implementers are not required to support them. + + +Type Alias +'''''''''' + +When creating a type alias, like: + +.. code:: python + + Username = Annotated[str, doc("The name of a user in the system")] + + +The documentation would be considered to be carried by the parameter annotated +with ``Username``. + +So, in a function like: + +.. code:: python + + def hi(to: Username) -> None: ... + + +It would be equivalent to: + +.. code:: python + + def hi(to: Annotated[str, doc("The name of a user in the system")]) -> None: ... + +Nevertheless, implementers would not be required to support type aliases outside +of the final type annotation to be conformant with this specification, as it +could require more complex dereferencing logic. + + +Annotating Type Parameters +'''''''''''''''''''''''''' + +When annotating type parameters, as in: + +.. code:: python + + def hi( + to: list[Annotated[str, doc("The name of a user in a list")]], + ) -> None: ... + +The documentation in ``doc()`` would refer to what it is annotating, in this +case, each item in the list, not the list itself. + +There are currently no practical use cases for documenting type parameters, +so implementers are not required to support this scenario to be considered +conformant, but it's included for completeness. + + +Annotating Unions +''''''''''''''''' + +If used in one of the parameters of a union, as in: + +.. code:: python + + def hi( + to: str | Annotated[list[str], doc("List of user names")], + ) -> None: ... + +Again, the documentation in ``doc()`` would refer to what it is annotating, +in this case, this documents the list itself, not its items. + +In particular, the documentation would not refer to a single string passed as a +parameter, only to a list. + +There are currently no practical use cases for documenting unions, so implementers +are not required to support this scenario to be considered conformant, but it's +included for completeness. + + +Nested ``Annotated`` +'''''''''''''''''''' + +Continuing with the same idea above, if ``Annotated`` was used nested and used +multiple times in the same parameter, ``doc()`` would refer to the type it +is annotating. + +So, in an example like: + +.. code:: python + + def hi( + to: Annotated[ + Annotated[str, doc("A user name")] + | Annotated[list, doc("A list of user names")], + doc("Who to say hi to"), + ], + ) -> None: ... + + +The documentation for the whole parameter ``to`` would be considered to be +"``Who to say hi to``". + +The documentation for the case where that parameter ``to`` is specifically a ``str`` +would be considered to be "``A user name``". + +The documentation for the case where that parameter ``to`` is specifically a +``list`` would be considered to be "``A list of user names``". + +Implementers would only be required to support the top level use case, where the +documentation for ``to`` is considered to be "``Who to say hi to``". +They could optionally support having conditional documentation for when the type +of the parameter passed is of one type or another, but they are not required to do so. + + +Duplication +''''''''''' + +If ``doc()`` is used multiple times in a single ``Annotated``, it would be +considered invalid usage from the developer, for example: + +.. code:: python + + def hi( + to: Annotated[str, doc("A user name"), doc("The current user name")], + ) -> None: ... + + +Implementers can consider this invalid and are not required to support this to be +considered conformant. + +Nevertheless, as it might be difficult to enforce it on developers, implementers +can opt to support one of the ``doc()`` declarations. + +In that case, the suggestion would be to support the last one, just because +this would support overriding, for example, in: + +.. code:: python + + User = Annotated[str, doc("A user name")] + + CurrentUser = Annotated[User, doc("The current user name")] + + +Internally, in Python, ``CurrentUser`` here is equivalent to: + +.. code:: python + + CurrentUser = Annotated[str, + doc("A user name"), + doc("The current user name")] + + +For an implementation that supports the last ``doc()`` appearance, the above +example would be equivalent to: + +.. code:: python + + def hi(to: Annotated[str, doc("The current user name")]) -> None: ... + + +.. you need to fill these in: + + Backwards Compatibility + ======================= + + [Describe potential impact and severity on pre-existing code.] + + + Security Implications + ===================== + + [How could a malicious user take advantage of this new feature?] + + + How to Teach This + ================= + + [How to teach users, new and experienced, how to apply the PEP to their work.] + + +Reference Implementation +======================== + +``typing.doc`` and ``typing.DocInfo`` are implemented as follows: + +.. code:: python + + def doc(documentation: str, /) -> DocInfo: + return DocInfo(documentation) + + class DocInfo: + def __init__(self, documentation: str, /): + self.documentation = documentation + + +These have been implemented in the `typing_extensions`__ package. + +__ https://pypi.org/project/typing-extensions/ + + +Rejected Ideas +============== + + +Standardize Current Docstrings +------------------------------ + +A possible alternative would be to support and try to push as a standard one of the +existing docstring formats. But that would only solve the standardization. + +It wouldn't solve any of the other problems, like getting editor support +(syntax checks) for library authors, the distance and duplication of information +between a parameter definition and its documentation in the docstring, etc. + + +Extra Metadata and Decorator +---------------------------- + +An earlier version of this proposal included several parameters to indicate whether +an object is discouraged from use, what exceptions it may raise, etc. +To allow also deprecating functions and classes, it was also expected +that ``doc()`` could be used as a decorator. But this functionality is covered +by ``typing.deprecated()`` in :pep:`702`, so it was dropped from this proposal. + +A way to declare additional information could still be useful in the future, +but taking early feedback on this document, all that was postponed to future +proposals. + +This also shifts the focus from an all-encompassing function ``doc()`` +with multiple parameters to multiple composable functions, having ``doc()`` +handle one single use case: additional documentation in ``Annotated``. + +This design change also allows better interoperability with other proposals +like ``typing.deprecated()``, as in the future it could be considered to +allow having ``typing.deprecated()`` also in ``Annotated`` to deprecate +individual parameters, coexisting with ``doc()``. + + +Open Issues +=========== + + +Verbosity +--------- + +The main argument against this would be the increased verbosity. + +Nevertheless, this verbosity would not affect end users as they would not see the +internal code using ``typing.doc()``. + +And the cost of dealing with the additional verbosity would only be carried +by those library maintainers that decide to opt-in into this feature. + +Any authors that decide not to adopt it, are free to continue using docstrings +with any particular format they decide, no docstrings at all, etc. + +This argument could be analogous to the argument against type annotations +in general, as they do indeed increase verbosity, in exchange for their +features. But again, as with type annotations, this would be optional and only +to be used by those that are willing to take the extra verbosity in exchange +for the benefits. + + +Documentation is not Typing +--------------------------- + +It could also be argued that documentation is not really part of typing, or that +it should live in a different module. Or that this information should not be part +of the signature but live in another place (like the docstring). + +Nevertheless, type annotations in Python could already be considered, by default, +mainly documentation: they carry additional information about variables, +parameters, return types, and by default they don't have any runtime behavior. + +It could be argued that this proposal extends the type of information that +type annotations carry, the same way as :pep:`702` extends them to include +deprecation information. + +And as described above, including this in ``typing_extensions`` to support older +versions of Python would have a very simple and practical benefit. + + +Multiple Standards +------------------ + +Another argument against this would be that it would create another standard, +and that there are already several pseudo-standards for docstrings. It could +seem better to formalize one of the currently existing standards. + +Nevertheless, as stated above, none of those standards cover the general +drawbacks of a doctsring-based approach that this proposal solves naturally. + +None of the editors have full docstring editing support (even when they have +rendering support). Again, this is solved by this proposal just by using +standard Python syntax and structures instead of a docstring microsyntax. + +The effort required to implement support for this proposal by tools would +be minimal compared to that required for alternative docstring-based +pseudo-standards, as for this proposal, editors would only need to +access an already existing value in their ASTs, instead of writing a parser +for a new string microsyntax. + +In the same way, it can be seen that, in many cases, a new standard that +takes advantage of new features and solves several problems from previous +methods can be worth having. As is the case with the new ``pyproject.toml``, +``dataclass_transform``, the new typing pipe/union (``|``) operator, and other cases. + + +Adoption +-------- + +As this is a new standard proposal, it would only make sense if it had +interest from the community. + +Fortunately there's already interest from several mainstream libraries +from several developers and teams, including FastAPI, Typer, SQLModel, +Asyncer (from the author of this proposal), Pydantic, Strawberry, and others, +from other teams. + +There's also interest and support from documentation tools, like +`mkdocstrings `__, which added +support even for an earlier version of this proposal. + +All the CPython core developers contacted for early feedback (at least 4) have +shown interest and support for this proposal. + +Editor developers (VS Code and PyCharm) have shown some interest, while showing +concerns about the verbosity of the proposal, although not about the +implementation (which is what would affect them the most). And they have shown +they would consider adding support for this if it were to become an +official standard. In that case, they would only need to add support for +rendering, as support for editing, which is normally non-existing for +other standards, is already there, as they already support editing standard +Python syntax. + + +Bike Shedding +------------- + +I think ``doc()`` is a good name for the main function. But it might make sense +to consider changing the names for the other parts. + +The returned class containing info currently named ``DocInfo`` could instead +be named just ``Doc``. Although it could make verbal conversations more +confusing as it's the same word as the name of the function. + +The parameter received by ``doc()`` currently named ``documentation`` could +instead be named also ``doc``, but it would make it more ambiguous in +discussions to distinguish when talking about the function and the parameter, +although it would simplify the amount of terms, but as these terms refer to +different things closely related, it could make sense to have different names. + +The parameter received by ``doc()`` currently named ``documentation`` could +instead be named ``value``, but the word "documentation" might convey +the meaning better. + +The parameter received by ``doc()`` currently named ``documentation`` could be a +position-only parameter, in which case the name wouldn't matter much. But then +there wouldn't be a way to make it match with the ``DocInfo`` attribute. + +The ``DocInfo`` class has a single attribute ``documentation``, this name matches +the parameter passed to ``doc()``. It could be named something different, +like ``doc``, but this would mean a mismatch between the ``doc()`` parameter +``documentation`` and the equivalent attribute ``doc``, and it would mean that in +one case (in the function), the term ``doc`` refers to a function, and in the +other case (the resulting class) the term ``doc`` refers to a string value. + +This shows the logic to select the current terms, but it could all be +discussed further. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/pep-0754.txt b/peps/pep-0754.rst similarity index 98% rename from pep-0754.txt rename to peps/pep-0754.rst index 9282f1988..d10565fc5 100644 --- a/pep-0754.txt +++ b/peps/pep-0754.rst @@ -2,7 +2,7 @@ PEP: 754 Title: IEEE 754 Floating Point Special Values Version: $Revision$ Last-Modified: $Date$ -Author: Gregory R. Warnes (Pfizer, Inc.) +Author: Gregory R. Warnes Status: Rejected Type: Standards Track Content-Type: text/x-rst diff --git a/pep-0801.rst b/peps/pep-0801.rst similarity index 70% rename from pep-0801.rst rename to peps/pep-0801.rst index cd4e6bf0b..243e1c22a 100644 --- a/pep-0801.rst +++ b/peps/pep-0801.rst @@ -1,7 +1,7 @@ PEP: 801 Title: Reserved Author: Barry Warsaw -Status: Draft +Status: Active Type: Informational Content-Type: text/x-rst Created: 21-Jun-2018 @@ -10,8 +10,9 @@ Created: 21-Jun-2018 Abstract ======== -This PEP is reserved for future use. Contact the author or -`the PEP editors `_ for details. +This PEP is reserved for future use, because +`We are the 801 `_. +Contact the author for details. Copyright diff --git a/pep-3000.txt b/peps/pep-3000.rst similarity index 93% rename from pep-3000.txt rename to peps/pep-3000.rst index 2827689f4..af3ee070a 100644 --- a/pep-3000.txt +++ b/peps/pep-3000.rst @@ -35,11 +35,11 @@ files. PEP Numbering ============= -Python 3000 PEPs are numbered starting at PEP 3000. PEPs 3000-3099 +Python 3000 PEPs are numbered starting at :pep:`3000`. PEPs 3000-3099 are meta-PEPs -- these can be either process or informational PEPs. -PEPs 3100-3999 are feature PEPs. PEP 3000 itself (this PEP) is +PEPs 3100-3999 are feature PEPs. :pep:`3000` itself (this PEP) is special; it is the meta-PEP for Python 3000 meta-PEPs (IOW it describe -the process to define processes). PEP 3100 is also special; it's a +the process to define processes). :pep:`3100` is also special; it's a laundry list of features that were selected for (hopeful) inclusion in Python 3000 before we started the Python 3000 process for real. PEP 3099, finally, is a list of features that will *not* change. @@ -48,7 +48,7 @@ Python 3000 before we started the Python 3000 process for real. PEP Timeline ======== -See PEP 361 [#pep361]_, which contains the release schedule for Python +See :pep:`361`, which contains the release schedule for Python 2.6 and 3.0. These versions will be released in lockstep. Note: standard library development is expected to ramp up after 3.0a1 @@ -149,9 +149,6 @@ References .. [2] Joel on Software: Things You Should Never Do, Part I http://www.joelonsoftware.com/articles/fog0000000069.html -.. [#pep361] PEP 361 (Python 2.6 and 3.0 Release Schedule) - http://www.python.org/dev/peps/pep-0361 - Copyright ========= diff --git a/pep-3001.txt b/peps/pep-3001.rst similarity index 96% rename from pep-3001.txt rename to peps/pep-3001.rst index d4fb893c5..08df2138f 100644 --- a/pep-3001.txt +++ b/peps/pep-3001.rst @@ -29,8 +29,8 @@ but are too widely used to be deprecated or removed. Python 3000 is the big occasion to get rid of them. There will have to be a document listing all removed modules, together -with information on possible substitutes or alternatives. This infor- -mation will also have to be provided by the python3warn.py porting +with information on possible substitutes or alternatives. This +information will also have to be provided by the python3warn.py porting helper script mentioned in PEP XXX. diff --git a/pep-3002.txt b/peps/pep-3002.rst similarity index 100% rename from pep-3002.txt rename to peps/pep-3002.rst diff --git a/pep-3003.txt b/peps/pep-3003.rst similarity index 98% rename from pep-3003.txt rename to peps/pep-3003.rst index 2c5c87b45..d26fcc2fa 100644 --- a/pep-3003.txt +++ b/peps/pep-3003.rst @@ -122,7 +122,7 @@ Allowed to Change * Backports of 3.x features to 2.x The moratorium only affects features that would be new in 3.x. * Import semantics - For example, PEP 382. After all, import semantics vary between + For example, :pep:`382`. After all, import semantics vary between Python implementations anyway. diff --git a/pep-3099.txt b/peps/pep-3099.rst similarity index 98% rename from pep-3099.txt rename to peps/pep-3099.rst index d0a3af2e1..f76d4a24a 100644 --- a/pep-3099.txt +++ b/peps/pep-3099.rst @@ -149,8 +149,8 @@ Core language * The ``else`` clause in ``while`` and ``for`` loops will not change semantics, or be removed. - Thread: "for/except/else syntax" - https://mail.python.org/pipermail/python-ideas/2009-October/006083.html + Thread: "for/except/else syntax" + https://mail.python.org/pipermail/python-ideas/2009-October/006083.html Builtins diff --git a/pep-3100.txt b/peps/pep-3100.rst similarity index 80% rename from pep-3100.txt rename to peps/pep-3100.rst index 6cc33f3ae..f758a8ccf 100644 --- a/pep-3100.txt +++ b/peps/pep-3100.rst @@ -1,7 +1,5 @@ PEP: 3100 Title: Miscellaneous Python 3.0 Plans -Version: $Revision$ -Last-Modified: $Date$ Author: Brett Cannon Status: Final Type: Process @@ -13,7 +11,7 @@ Post-History: Abstract ======== -This PEP, previously known as PEP 3000, describes smaller scale changes +This PEP, previously known as :pep:`3000`, describes smaller scale changes and new features for which no separate PEP is written yet, all targeted for Python 3000. @@ -29,7 +27,7 @@ decisions for which changes are listed in this document are made by Guido van Rossum, who has chosen them as goals for Python 3.0. Guido's pronouncements on things that will not change in Python 3.0 -are recorded in PEP 3099. [#pep3099]_ +are recorded in :pep:`3099`. General goals @@ -43,10 +41,10 @@ obvious way of doing something is enough. [1]_ Influencing PEPs ================ -* PEP 238 (Changing the Division Operator) [#pep238]_ -* PEP 328 (Imports: Multi-Line and Absolute/Relative) [#pep328]_ -* PEP 343 (The "with" Statement) [#pep343]_ -* PEP 352 (Required Superclass for Exceptions) [#pep352]_ +* :pep:`238` (Changing the Division Operator) +* :pep:`328` (Imports: Multi-Line and Absolute/Relative) +* :pep:`343` (The "with" Statement) +* :pep:`352` (Required Superclass for Exceptions) Style changes @@ -62,11 +60,11 @@ Style changes Core language ============= -* True division becomes default behavior [#pep238]_ [done] +* True division becomes default behavior :pep:`238` [done] * ``exec`` as a statement is not worth it -- make it a function [done] -* Add optional declarations for static typing [#pep3107]_ [10]_ [done] +* Add optional declarations for static typing :pep:`3107` [10]_ [done] * Support only new-style classes; classic classes will be gone [1]_ [done] -* Replace ``print`` by a function [14]_ [#pep3105]_ [done] +* Replace ``print`` by a function [14]_ :pep:`3105` [done] * The ``softspace`` attribute of files goes away. [done] * Use ``except E1, E2, E3 as err:`` if you want the error variable. [3]_ [done] * ``None`` becomes a keyword [4]_; also ``True`` and ``False`` [done] @@ -74,14 +72,14 @@ Core language * ``as`` becomes a keyword [5]_ (starting in 2.6 already) [done] * Have list comprehensions be syntactic sugar for passing an equivalent generator expression to ``list()``; as a consequence the - loop variable will no longer be exposed [#pep289]_ [done] + loop variable will no longer be exposed :pep:`289` [done] * Comparisons other than ``==`` and ``!=`` between disparate types will raise an exception unless explicitly supported by the type [6]_ [done] * floats will not be acceptable as arguments in place of ints for operations where floats are inadvertently accepted (PyArg_ParseTuple() i & l formats) * Remove from ... import * at function scope. [done] This means that functions can always be optimized and support for unoptimized functions can go away. -* Imports [#pep328]_ +* Imports :pep:`328` + Imports will be absolute by default. [done] + Relative imports must be explicitly specified. [done] + Indirection entries in ``sys.modules`` (i.e., a value of ``None`` for @@ -96,7 +94,7 @@ Core language - List comprehensions will require parentheses around the iterables. This will make list comprehensions more similar to generator comprehensions. [x for x in 1, 2] will need to be: [x for x in (1, 2)] [done] - - Lambdas may have to be parenthesized [#pep308]_ [NO] + - Lambdas may have to be parenthesized :pep:`308` [NO] * In order to get rid of the confusion between __builtin__ and __builtins__, it was decided to rename __builtin__ (the module) to builtins, and to leave @@ -112,7 +110,7 @@ Core language * The ``__nonzero__`` special method will be renamed to ``__bool__`` and have to return a bool. The typeobject slot will be called ``tp_bool`` [23]_ [done] -* Dict comprehensions, as first proposed in [#pep274]_ [done] +* Dict comprehensions, as first proposed in :pep:`274` [done] {K(x): V(x) for x in S if P(x)} means dict((K(x), V(x)) for x in S if P(x)). To be removed: @@ -123,7 +121,7 @@ To be removed: * ``x``: use ``repr(x)`` [2]_ [done] * The ``<>`` operator: use ``!=`` instead [3]_ [done] * The __mod__ and __divmod__ special methods on float. [they should stay] [21]_ -* Drop unbound methods [7]_ [25]_ [done] +* Drop unbound methods [7]_ [26]_ [done] * METH_OLDARGS [done] * WITH_CYCLE_GC [done] * __getslice__, __setslice__, __delslice__ [#sequence-types]_; @@ -147,7 +145,7 @@ Atomic Types * Remove distinction between int and long types; 'long' built-in type and literals with 'L' or 'l' suffix disappear [1]_ [done] * Make all strings be Unicode, and have a separate bytes() type [1]_ - The new string type will be called 'str'. See PEP 3137. [done] + The new string type will be called 'str'. See :pep:`3137`. [done] * Return iterable views instead of lists where appropriate for atomic type methods (e.g. ``dict.keys()``, ``dict.values()``, ``dict.items()``, etc.); iter* methods will be removed. [done] @@ -176,7 +174,7 @@ Built-in Namespace * Introduce ``trunc()``, which would call the ``__trunc__()`` method on its argument; suggested use is for objects like float where calling ``__int__()`` has data loss, but an integral representation is still desired? [8]_ [done] -* Exception hierarchy changes [#pep352]_ [done] +* Exception hierarchy changes :pep:`352` [done] * Add a ``bin()`` function for a binary representation of integers [done] To be removed: @@ -202,7 +200,7 @@ Standard library * Move test code to where it belongs, there will be no more test() functions in the standard library * Convert all tests to use either doctest or unittest. -* For the procedures of standard library improvement, see PEP 3001 [#pep3001]_ +* For the procedures of standard library improvement, see :pep:`3001` To be removed: @@ -212,7 +210,7 @@ To be removed: - ``macfs`` [to do] - ``new``, ``reconvert``, ``stringold``, ``xmllib``, ``pcre``, ``pypcre``, ``strop`` [all done] - + see PEP 4 [#pep4]_ + + see :pep:`4` - ``buildtools``, ``mimetools``, ``multifile``, @@ -222,7 +220,7 @@ To be removed: ``sha``, ``statcache``, ``sv``, ``TERMIOS``, ``timing`` [done] - ``cfmfile``, ``gopherlib``, ``md5``, ``MimeWriter``, ``mimify`` [done] - ``cl``, ``sets``, ``xreadlines``, ``rotor``, ``whrandom`` [done] - + Everything in lib-old [#pep4]_ [done] + + Everything in lib-old :pep:`4` [done] - ``Para``, ``addpack``, ``cmp``, ``cmpcache``, ``codehack``, ``dircmp``, ``dump``, ``find``, ``fmt``, ``grep``, ``lockfile``, ``newdir``, ``ni``, ``packmail``, ``poly``, @@ -234,7 +232,7 @@ To be removed: not thread-safe; use ``sys.exc_info()`` or an attribute of the exception [2]_ [11]_ [#sys-module]_ [done] * ``sys.exc_clear``: Python 3's except statements provide the same - functionality [24]_ [#pep3110]_ [#sys-module]_ [done] + functionality [24]_ :pep:`3110` [#sys-module]_ [done] * ``array.read``, ``array.write`` [#array-module]_ * ``operator.isCallable`` : ``callable()`` built-in is being removed [#operator-module]_ [#remove-operator-funcs]_ [done] @@ -246,13 +244,13 @@ To be removed: in favor of always using threading.py.) * UserXyz classes, in favour of XyzMixins. -* Remove the unreliable empty() and full() methods from Queue.py? -* Remove jumpahead() from the random API? +* Remove the unreliable empty() and full() methods from Queue.py? [25]_ +* Remove jumpahead() from the random API? [25]_ * Make the primitive for random be something generating random bytes - rather than random floats? -* Get rid of Cookie.SerialCookie and Cookie.SmartCookie? + rather than random floats? [25]_ +* Get rid of Cookie.SerialCookie and Cookie.SmartCookie? [25]_ * Modify the heapq.heapreplace() API to compare the new value to the top - of the heap? + of the heap? [25]_ Outstanding Issues ================== @@ -268,13 +266,13 @@ References ========== .. [1] PyCon 2003 State of the Union: - http://www.python.org/doc/essays/ppt/pycon2003/pycon2003.ppt + https://legacy.python.org/doc/essays/ppt/pycon2003/pycon2003.ppt .. [2] Python Regrets: - http://www.python.org/doc/essays/ppt/regrets/PythonRegrets.pdf + https://legacy.python.org/doc/essays/ppt/regrets/PythonRegrets.pdf .. [3] Python Wiki: - http://www.python.org/moin/Python3.0 + https://wiki.python.org/moin/Python3.0 .. [4] python-dev email ("Constancy of None") https://mail.python.org/pipermail/python-dev/2004-July/046294.html @@ -293,10 +291,10 @@ References https://mail.python.org/pipermail/python-dev/2005-February/051674.html .. [9] Guido's blog ("The fate of reduce() in Python 3000") - http://www.artima.com/weblogs/viewpost.jsp?thread=98196 + https://www.artima.com/weblogs/viewpost.jsp?thread=98196 .. [10] Guido's blog ("Python Optional Typechecking Redux") - http://www.artima.com/weblogs/viewpost.jsp?thread=89161 + https://www.artima.com/weblogs/viewpost.jsp?thread=89161 .. [11] python-dev email ("anonymous blocks") https://mail.python.org/pipermail/python-dev/2005-April/053060.html @@ -350,61 +348,19 @@ References https://mail.python.org/pipermail/python-dev/2008-February/076818.html .. [#sys-module] Python docs (sys -- System-specific parameters and functions) - http://docs.python.org/library/sys.html + https://docs.python.org/release/2.6/library/sys.html .. [#operator-module] Python docs (operator -- Standard operators as functions) - http://docs.python.org/library/operator.html + https://docs.python.org/release/2.6/library/operator.html .. [#array-module] Python docs (array -- Efficient arrays of numeric values) - http://docs.python.org/library/array.html + https://docs.python.org/release/2.6/library/array.html .. [#file-object] Python docs (File objects) - http://docs.python.org/library/stdtypes.html + https://docs.python.org/release/2.6/library/stdtypes.html .. [#sequence-types] Python docs (Additional methods for emulation of sequence types) - http://docs.python.org/reference/datamodel.html#additional-methods-for-emulation-of-sequence-types - -.. [#pep4] PEP 4 ("Deprecation of Standard Modules") - http://www.python.org/dev/peps/pep-0004 - -.. [#pep238] PEP 238 (Changing the Division Operator) - http://www.python.org/dev/peps/pep-0238 - -.. [#pep274] PEP 274 (Dict Comprehensions) - http://www.python.org/dev/peps/pep-0274 - -.. [#pep289] PEP 289 ("Generator Expressions") - http://www.python.org/dev/peps/pep-0289 - -.. [#pep299] PEP 299 ("Special __main__() function in modules") - http://www.python.org/dev/peps/pep-0299 - -.. [#pep308] PEP 308 ("Conditional Expressions") - http://www.python.org/dev/peps/pep-0308 - -.. [#pep328] PEP 328 (Imports: Multi-Line and Absolute/Relative) - http://www.python.org/dev/peps/pep-0328 - -.. [#pep343] PEP 343 (The "with" Statement) - http://www.python.org/dev/peps/pep-0343 - -.. [#pep352] PEP 352 (Required Superclass for Exceptions) - http://www.python.org/dev/peps/pep-0352 - -.. [#pep3001] PEP 3001 (Process for reviewing and improving standard - library modules) http://www.python.org/dev/peps/pep-3001 - -.. [#pep3099] PEP 3099 (Things that will Not Change in Python 3000) - http://www.python.org/dev/peps/pep-3099 - -.. [#pep3105] PEP 3105 (Make print a function) - http://www.python.org/dev/peps/pep-3105 - -.. [#pep3107] PEP 3107 (Function Annotations) - http://www.python.org/dev/peps/pep-3107 - -.. [#pep3110] PEP 3110 (Catching Exceptions in Python 3000) - http://www.python.org/dev/peps/pep-3110/#semantic-changes + https://docs.python.org/release/2.6/reference/datamodel.html#additional-methods-for-emulation-of-sequence-types .. [#builtin] Approach to resolving __builtin__ vs __builtins__ https://mail.python.org/pipermail/python-3000/2007-March/006161.html @@ -413,22 +369,12 @@ References https://mail.python.org/pipermail/python-dev/2007-November/075388.html .. [#exitfunc-patch] Patch to remove sys.exitfunc - https://bugs.python.org/issue1680961 + https://github.com/python/cpython/issues/44715 .. [#remove-operator-funcs] Remove deprecated functions from operator - https://bugs.python.org/issue1516309 + https://github.com/python/cpython/issues/43602 Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-3101.txt b/peps/pep-3101.rst similarity index 99% rename from pep-3101.txt rename to peps/pep-3101.rst index 41c17516e..9267d17cb 100644 --- a/pep-3101.txt +++ b/peps/pep-3101.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 16-Apr-2006 Python-Version: 3.0 -Post-History: 28-Apr-2006, 6-May-2006, 10-Jun-2007, 14-Aug-2007, 14-Sep-2008 +Post-History: 28-Apr-2006, 06-May-2006, 10-Jun-2007, 14-Aug-2007, 14-Sep-2008 Abstract @@ -310,10 +310,10 @@ The available integer presentation types are:: Unicode character before printing. 'd' - Decimal Integer. Outputs the number in base 10. 'o' - Octal format. Outputs the number in base 8. - 'x' - Hex format. Outputs the number in base 16, using lower- - case letters for the digits above 9. - 'X' - Hex format. Outputs the number in base 16, using upper- - case letters for the digits above 9. + 'x' - Hex format. Outputs the number in base 16, using + lower-case letters for the digits above 9. + 'X' - Hex format. Outputs the number in base 16, using + upper-case letters for the digits above 9. 'n' - Number. This is the same as 'd', except that it uses the current locale setting to insert the appropriate number separator characters. diff --git a/pep-3102.txt b/peps/pep-3102.rst similarity index 99% rename from pep-3102.txt rename to peps/pep-3102.rst index 03df2a612..85fb8d98d 100644 --- a/pep-3102.txt +++ b/peps/pep-3102.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 22-Apr-2006 Python-Version: 3.0 -Post-History: 28-Apr-2006, May-19-2006 +Post-History: 28-Apr-2006, 19-May-2006 Abstract diff --git a/pep-3103.txt b/peps/pep-3103.rst similarity index 98% rename from pep-3103.txt rename to peps/pep-3103.rst index 400cfd66e..007b909a3 100644 --- a/pep-3103.txt +++ b/peps/pep-3103.rst @@ -1,8 +1,6 @@ PEP: 3103 Title: A Switch/Case Statement -Version: $Revision$ -Last-Modified: $Date$ -Author: guido@python.org (Guido van Rossum) +Author: Guido van Rossum Status: Rejected Type: Standards Track Content-Type: text/x-rst @@ -27,7 +25,7 @@ the smorgasbord of proposals, discussing alternatives and explaining my choices where I can. I'll also indicate how strongly I feel about alternatives I discuss. -This PEP should be seen as an alternative to PEP 275. My views are +This PEP should be seen as an alternative to :pep:`275`. My views are somewhat different from that PEP's author, but I'm grateful for the work done in that PEP. @@ -90,7 +88,7 @@ Semantics are discussed in the next top-level section. Alternative 1 ------------- -This is the preferred form in PEP 275:: +This is the preferred form in :pep:`275`:: switch EXPR: case EXPR: @@ -234,7 +232,7 @@ like this:: case [*]EXPR, [*]EXPR, ...: -The `*` notation is similar to the use of prefix `*` already in use for +The ``*`` notation is similar to the use of prefix ``*`` already in use for variable-length parameter lists and for passing computed argument lists, and often proposed for value-unpacking (e.g. ``a, b, *c = X`` as an alternative to ``(a, b), c = X[:2], X[2:]``). @@ -322,7 +320,7 @@ We need to further separate school I into school Ia and school Ib: - School Ib has a more complex position: it agrees with school II that optimization is important, and is willing to concede the compiler - certain liberties to allow this. (For example, PEP 275 Solution 1.) + certain liberties to allow this. (For example, :pep:`275` Solution 1.) In particular, hash() of the switch and case expressions may or may not be called (so it should be side-effect-free); and the case expressions may not be evaluated each time as expected by the @@ -624,14 +622,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3104.txt b/peps/pep-3104.rst similarity index 96% rename from pep-3104.txt rename to peps/pep-3104.rst index aca837da9..920b929a5 100644 --- a/pep-3104.txt +++ b/peps/pep-3104.rst @@ -1,7 +1,5 @@ PEP: 3104 Title: Access to Names in Outer Scopes -Version: $Revision$ -Last-Modified: $Date$ Author: Ka-Ping Yee Status: Final Type: Standards Track @@ -53,7 +51,7 @@ different contexts. Here's an example:: print factorial(5) Python 2.1 moved closer to static nested scoping by making visible -the names bound in all enclosing scopes (see PEP 227). This change +the names bound in all enclosing scopes (see :pep:`227`). This change makes the above code example work as expected. However, because any assignment to a name implicitly declares that name to be local, it is impossible to rebind a name in an outer scope (except when a @@ -119,15 +117,15 @@ much strength, practical flexibility, and pedagogical power from its support and graceful integration of multiple programming paradigms. A proposal for scoping syntax appeared on Python-Dev as far back as -1994 [1]_, long before PEP 227's support for nested scopes was +1994 [1]_, long before :pep:`227`'s support for nested scopes was adopted. At the time, Guido's response was: This is dangerously close to introducing CSNS [classic static nested scopes]. *If* you were to do so, your proposed semantics - of scoped seem allright. I still think there is not enough need + of scoped seem alright. I still think there is not enough need for CSNS to warrant this kind of construct ... -After PEP 227, the "outer name rebinding discussion" has reappeared +After :pep:`227`, the "outer name rebinding discussion" has reappeared on Python-Dev enough times that it has become a familiar event, having recurred in its present form since at least 2003 [2]_. Although none of the language changes proposed in these discussions @@ -265,7 +263,7 @@ statement ``x = 3`` both declares ``x`` a local variable and binds it to 3, the statement ``x := 3`` would change the existing binding of ``x`` without declaring it local. -This is a simple solution, but according to PEP 3099 it has been +This is a simple solution, but according to :pep:`3099` it has been rejected (perhaps because it would be too easy to miss or to confuse with ``=``). @@ -482,8 +480,8 @@ References .. [14] Explicit Lexical Scoping (pre-PEP?) (Guido van Rossum) https://mail.python.org/pipermail/python-dev/2006-July/066991.html -.. [15] Explicit Lexical Scoping (pre-PEP?) (Guido van Rossum) - https://mail.python.org/pipermail/python-dev/2006-July/066995.html +[15] Explicit Lexical Scoping (pre-PEP?) (Guido van Rossum) +\ https://mail.python.org/pipermail/python-dev/2006-July/066995.html .. [16] Lexical scoping in Python 3k (Guido van Rossum) https://mail.python.org/pipermail/python-dev/2006-July/066968.html @@ -519,10 +517,10 @@ References https://mail.python.org/pipermail/python-3000/2006-November/004237.html .. [27] Global variable (version 2006-11-01T01:23:16) - http://en.wikipedia.org/wiki/Global_variable + https://en.wikipedia.org/w/index.php?title=Global_variable&oldid=85001451 .. [28] Ruby 2.0 block local variable - http://redhanded.hobix.com/inspect/ruby20BlockLocalVariable.html + https://web.archive.org/web/20070105131417/http://redhanded.hobix.com/inspect/ruby20BlockLocalVariable.html .. [29] Issue 4199: combining assignment with global & nonlocal (Guido van Rossum) https://mail.python.org/pipermail/python-dev/2013-June/127142.html @@ -547,13 +545,3 @@ Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3105.txt b/peps/pep-3105.rst similarity index 100% rename from pep-3105.txt rename to peps/pep-3105.rst diff --git a/pep-3106.txt b/peps/pep-3106.rst similarity index 99% rename from pep-3106.txt rename to peps/pep-3106.rst index c88098b95..2d50df934 100644 --- a/pep-3106.txt +++ b/peps/pep-3106.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 19-Dec-2006 +Python-Version: 3.0 Post-History: diff --git a/pep-3107.txt b/peps/pep-3107.rst similarity index 98% rename from pep-3107.txt rename to peps/pep-3107.rst index 2e4ffafa6..d29554ab6 100644 --- a/pep-3107.txt +++ b/peps/pep-3107.rst @@ -25,7 +25,7 @@ Rationale Because Python's 2.x series lacks a standard way of annotating a function's parameters and return values, a variety of tools and libraries have appeared to fill this gap. Some -utilise the decorators introduced in "PEP 318", while others parse a +utilise the decorators introduced in :pep:`318`, while others parse a function's docstring, looking for annotations there. This PEP aims to provide a single, standard way of specifying this @@ -225,8 +225,8 @@ to support annotations. Relation to Other PEPs ====================== -Function Signature Objects [#pep-362]_ --------------------------------------- +Function Signature Objects (PEP 362) +------------------------------------ Function Signature Objects should expose the function's annotations. The ``Parameter`` object may change or other changes may be warranted. @@ -304,9 +304,6 @@ References and Footnotes .. [#lambda] https://mail.python.org/pipermail/python-3000/2006-May/001613.html -.. [#pep-362] - http://www.python.org/dev/peps/pep-0362/ - .. [#interop0] https://mail.python.org/pipermail/python-3000/2006-August/002895.html diff --git a/pep-3108.txt b/peps/pep-3108.rst similarity index 95% rename from pep-3108.txt rename to peps/pep-3108.rst index 547b7d6b1..02d913067 100644 --- a/pep-3108.txt +++ b/peps/pep-3108.rst @@ -26,7 +26,7 @@ inception that not all modules follow. Python 3.0 presents a chance to remove modules that do not have long term usefulness. This chance also allows for the renaming of -modules so that they follow the Python style guide [#pep-0008]_. This +modules so that they follow the :pep:`Python style guide <8>`. This PEP lists modules that should not be included in Python 3.0 or which need to be renamed. @@ -58,9 +58,9 @@ period. Previously deprecated [done] ---------------------------- -PEP 4 lists all modules that have been deprecated in the stdlib -[#pep-0004]_. The specified motivations mirror those listed in -PEP 4. All modules listed +:pep:`4` lists all modules that have been deprecated in the stdlib. +The specified motivations mirror those listed in +:pep:`4`. All modules listed in the PEP at the time of the first alpha release of Python 3.0 will be removed. @@ -544,7 +544,7 @@ for what the module is meant for. * commands [done] - + subprocess module replaces it [#pep-0324]_. + + subprocess module replaces it (:pep:`324`). + Remove getstatus(), move rest to subprocess. * compiler [done] @@ -595,7 +595,7 @@ for what the module is meant for. * popen2 [done] - + subprocess module replaces it [#pep-0324]_. + + subprocess module replaces it (:pep:`324`). * sgmllib [done] @@ -667,7 +667,7 @@ Modules to Rename ================= Many modules existed in -the stdlib before PEP 8 came into existence [#pep-0008]_. This has +the stdlib before :pep:`8` came into existence. This has led to some naming inconsistencies and namespace bloat that should be addressed. @@ -675,10 +675,10 @@ addressed. PEP 8 violations [done] ------------------------ -PEP 8 specifies that modules "should have short, all-lowercase names" -where "underscores can be used ... if it improves readability" -[#pep-0008]_. The use of underscores is discouraged in package names. -The following modules violate PEP 8 and are not somehow being renamed +:pep:`8` specifies that modules "should have short, all-lowercase names" +where "underscores can be used ... if it improves readability". +The use of underscores is discouraged in package names. +The following modules violate :pep:`8` and are not somehow being renamed by being moved to a package. ================== ================================================== @@ -1032,10 +1032,10 @@ Open Issues Renaming of modules maintained outside of the stdlib ---------------------------------------------------- -xml.etree.ElementTree not only does not meet PEP 8 naming standards -but it also has an exposed C implementation [#pep-0008]_. It is an -externally maintained package, though [#pep-0360]_. A request will be -made for the maintainer to change the name so that it matches PEP 8 +xml.etree.ElementTree not only does not meet :pep:`8` naming standards +but it also has an exposed C implementation. It is an +externally maintained package, though :pep:`360`. A request will be +made for the maintainer to change the name so that it matches :pep:`8` and hides the C implementation. @@ -1128,18 +1128,6 @@ been chosen as the guiding force for PEPs to create. References ========== -.. [#pep-0004] PEP 4: Deprecation of Standard Modules - (http://www.python.org/dev/peps/pep-0004/) - -.. [#pep-0008] PEP 8: Style Guide for Python Code - (http://www.python.org/dev/peps/pep-0008/) - -.. [#pep-0324] PEP 324: subprocess -- New process module - (http://www.python.org/dev/peps/pep-0324/) - -.. [#pep-0360] PEP 360: Externally Maintained Packages - (http://www.python.org/dev/peps/pep-0360/) - .. [#module-index] Python Documentation: Global Module Index (http://docs.python.org/modindex.html) diff --git a/pep-3109.txt b/peps/pep-3109.rst similarity index 92% rename from pep-3109.txt rename to peps/pep-3109.rst index ac8872506..53fb911c1 100644 --- a/pep-3109.txt +++ b/peps/pep-3109.rst @@ -22,8 +22,8 @@ language. Rationale ========= -One of Python's guiding maxims is "there should be one -- and -preferably only one -- obvious way to do it" [#zen]_. Python 2.x's +One of Python's guiding maxims is :pep:`"there should be one -- and +preferably only one -- obvious way to do it" <20>`. Python 2.x's ``raise`` statement violates this principle, permitting multiple ways of expressing the same thought. For example, these statements are equivalent: :: @@ -37,7 +37,7 @@ tracebacks to be attached to an exception [#grammar]_: :: raise E, V, T -where T is a traceback. As specified in PEP 344 [#pep344]_, +where T is a traceback. As specified in :pep:`344`, exception objects in Python 3.x will possess a ``__traceback__`` attribute, admitting this translation of the three-expression ``raise`` statement: :: @@ -59,7 +59,7 @@ four forms to two: 2. ``raise EXCEPTION`` is used to raise a new exception. This form has two sub-variants: ``EXCEPTION`` may be an exception class or an instance of an exception class; valid exception classes are - BaseException and its subclasses [#pep352]_. If ``EXCEPTION`` + BaseException and its subclasses (:pep:`352`). If ``EXCEPTION`` is a subclass, it will be called with no arguments to obtain an exception instance. @@ -99,8 +99,8 @@ Changes to Builtin Types Because of its relation to exception raising, the signature for the ``throw()`` method on generator objects will change, dropping the -optional second and third parameters. The signature thus changes -from [#throw-sig]_ :: +optional second and third parameters. The signature thus changes (:pep:`342`) +from :: generator.throw(E, [V, [T]]) @@ -232,7 +232,7 @@ The following translations will be performed: raise CompileError from msg Using the ``raise ... from ...`` syntax introduced in - PEP 344. + :pep:`344`. Implementation @@ -244,21 +244,9 @@ This PEP was implemented in revision 57783 [#r57783]_. References ========== -.. [#zen] - http://www.python.org/dev/peps/pep-0020/ - .. [#grammar] http://docs.python.org/reference/simple_stmts.html#raise -.. [#throw-sig] - http://www.python.org/dev/peps/pep-0342/ - -.. [#pep344] - http://www.python.org/dev/peps/pep-0344/ - -.. [#pep352] - http://www.python.org/dev/peps/pep-0352/ - .. [#amk-line-noise] https://mail.python.org/pipermail/python-dev/2005-August/055187.html diff --git a/pep-3110.txt b/peps/pep-3110.rst similarity index 91% rename from pep-3110.txt rename to peps/pep-3110.rst index c6d7d3ff5..6129e51a6 100644 --- a/pep-3110.txt +++ b/peps/pep-3110.rst @@ -41,7 +41,7 @@ Rationale except (, ): -2. As specified in PEP 352 [#pep352]_, the ability to treat exceptions +2. As specified in :pep:`352`, the ability to treat exceptions as tuples will be removed, meaning this code will no longer work :: except os.error, (errno, errstr): @@ -49,7 +49,7 @@ Rationale Because the automatic unpacking will no longer be possible, it is desirable to remove the ability to use tuples as ``except`` targets. -3. As specified in PEP 344 [#pep344]_, exception instances in Python 3 +3. As specified in :pep:`344`, exception instances in Python 3 will possess a ``__traceback__`` attribute. The Open Issues section of that PEP includes a paragraph on garbage collection difficulties caused by this attribute, namely a "exception -> traceback -> @@ -59,13 +59,13 @@ Rationale Python 3 whereby the target name is deleted at the end of the ``except`` suite. -4. In the spirit of "there should be one -- and preferably only one - -- obvious way to do it" [#zen]_, it is desirable to consolidate +4. In the spirit of :pep:`"there should be one -- and preferably only one + -- obvious way to do it" <20>`, it is desirable to consolidate duplicate functionality. To this end, the ``exc_value``, ``exc_type`` and ``exc_traceback`` attributes of the ``sys`` module [#sys-module]_ will be removed in favor of ``sys.exc_info()``, which provides the same information. These - attributes are already listed in PEP 3100 [#pep3100]_ as targeted + attributes are already listed in :pep:`3100` as targeted for removal. @@ -105,7 +105,7 @@ hard-to-catch bugs -- cannot legally occur in 3.x code. Semantic Changes ================ -In order to resolve the garbage collection issue related to PEP 344, +In order to resolve the garbage collection issue related to :pep:`344`, ``except`` statements in Python 3 will generate additional bytecode to delete the target, thus eliminating the reference cycle. The source-to-source translation, as suggested by Phillip J. Eby @@ -219,7 +219,7 @@ Replacing or Dropping "sys.exc_info()" The idea of dropping ``sys.exc_info()`` or replacing it with a ``sys.exception`` attribute or a ``sys.get_exception()`` function has been raised several times on python-3000 ([#drop-excinfo]_, -[#replace-excinfo]_) and mentioned in PEP 344's "Open Issues" section. +[#replace-excinfo]_) and mentioned in :pep:`344`'s "Open Issues" section. While a ``2to3`` fixer to replace calls to ``sys.exc_info()`` and some attribute accesses would be trivial, it would be far more @@ -240,21 +240,9 @@ implemented in revision 55446 [#r55446]_. References ========== -.. [#pep352] - http://www.python.org/dev/peps/pep-0352/ - -.. [#zen] - http://www.python.org/dev/peps/pep-0020/ - .. [#sys-module] http://docs.python.org/library/sys.html -.. [#pep3100] - http://www.python.org/dev/peps/pep-3100/ - -.. [#pep344] - http://www.python.org/dev/peps/pep-0344/ - .. [#firstproposal] https://mail.python.org/pipermail/python-dev/2006-March/062449.html diff --git a/pep-3111.txt b/peps/pep-3111.rst similarity index 94% rename from pep-3111.txt rename to peps/pep-3111.rst index 4b9ab7d33..947014ee9 100644 --- a/pep-3111.txt +++ b/peps/pep-3111.rst @@ -2,7 +2,7 @@ PEP: 3111 Title: Simple input built-in in Python 3000 Version: $Revision$ Last-Modified: $Date$ -Author: Andre Roberge +Author: Andre Roberge Status: Final Type: Standards Track Content-Type: text/x-rst @@ -20,7 +20,8 @@ and two simple means of interactive input through the input() and raw_input() built-in functions. Python 3.0 will introduce various incompatible changes with previous -Python versions\ [1]_. Among the proposed changes, print will become a built-in +Python versions (:pep:`3100`). +Among the proposed changes, print will become a built-in function, print(), while input() and raw_input() would be removed completely from the built-in namespace, requiring importing some module to provide even the most basic input capability. @@ -42,7 +43,8 @@ and to obtain information from the user (interactive input). Any computer language intended to be used in an educational setting should provide straightforward methods for both output and interactive input. -The current proposals for Python 3.0 [1]_ include a simple output pathway +The :pep:`current proposals for Python 3.0 <3100>` +include a simple output pathway via a built-in function named print(), but a more complicated method for input [e.g. via sys.stdin.readline()], one that requires importing an external module. Current versions of Python (pre-3.0) include raw_input() as a @@ -139,9 +141,6 @@ The rationale for accepting the renaming can be found here [4]_. References ========== -.. [1] PEP 3100, Miscellaneous Python 3.0 Plans, Kuchling, Cannon - http://www.python.org/dev/peps/pep-3100/ - .. [2] The fate of raw_input() in Python 3000 https://mail.python.org/pipermail/edu-sig/2006-September/006967.html diff --git a/pep-3112.txt b/peps/pep-3112.rst similarity index 98% rename from pep-3112.txt rename to peps/pep-3112.rst index 7c51b5949..584d31a3e 100644 --- a/pep-3112.txt +++ b/peps/pep-3112.rst @@ -16,7 +16,7 @@ Abstract ======== This PEP proposes a literal syntax for the ``bytes`` objects -introduced in PEP 358. The purpose is to provide a convenient way to +introduced in :pep:`358`. The purpose is to provide a convenient way to spell ASCII strings and arbitrary binary data. diff --git a/pep-3113.txt b/peps/pep-3113.rst similarity index 95% rename from pep-3113.txt rename to peps/pep-3113.rst index 689b22020..45a48d70c 100644 --- a/pep-3113.txt +++ b/peps/pep-3113.rst @@ -111,13 +111,13 @@ When looking at the various types of parameters that a Python function can have, one will notice that tuple parameters tend to be an exception rather than the rule. -Consider PEP 3102 (keyword-only arguments) and PEP 3107 (function -annotations) [#pep-3102]_ [#pep-3107]_. Both PEPs have been accepted and +Consider :pep:`3102` (keyword-only arguments) and :pep:`3107` (function +annotations). Both PEPs have been accepted and introduce new functionality within a function's signature. And yet for both PEPs the new feature cannot be applied to tuple parameters as -a whole. PEP 3102 has no support for tuple parameters at all (which +a whole. :pep:`3102` has no support for tuple parameters at all (which makes sense as there is no way to reference a tuple parameter by -name). PEP 3107 allows annotations for each item within the tuple +name). :pep:`3107` allows annotations for each item within the tuple (e.g., ``(x:int, y:int)``), but not the whole tuple (e.g., ``(x, y):int``). @@ -260,12 +260,6 @@ References .. [#MSIL] Microsoft Intermediate Language (http://msdn.microsoft.com/library/en-us/cpguide/html/cpconmicrosoftintermediatelanguagemsil.asp?frame=true) -.. [#pep-3102] PEP 3102 (Keyword-Only Arguments) - (http://www.python.org/dev/peps/pep-3102/) - -.. [#pep-3107] PEP 3107 (Function Annotations) - (http://www.python.org/dev/peps/pep-3107/) - Copyright ========= diff --git a/pep-3114.txt b/peps/pep-3114.rst similarity index 96% rename from pep-3114.txt rename to peps/pep-3114.rst index d34527b1a..1fe8c1c27 100644 --- a/pep-3114.txt +++ b/peps/pep-3114.rst @@ -1,7 +1,5 @@ PEP: 3114 Title: Renaming iterator.next() to iterator.__next__() -Version: $Revision$ -Last-Modified: $Date$ Author: Ka-Ping Yee Status: Final Type: Standards Track @@ -116,7 +114,7 @@ Previous Proposals This proposal is not a new idea. The idea proposed here was supported by the BDFL on python-dev [1]_ and is even mentioned in the original -iterator PEP, PEP 234:: +iterator PEP, :pep:`234`:: (In retrospect, it might have been better to go for __next__() and have a new built-in, next(it), which calls it.__next__(). @@ -177,7 +175,7 @@ following conditions [5]_: Approval ======== -This PEP was accepted by Guido on March 6, 2007 [5]_. +This PEP was accepted by Guido on March 6, 2007 [6]_. Implementation @@ -197,7 +195,7 @@ References https://mail.python.org/pipermail/python-3000/2007-March/005965.html .. [3] 2to3 refactoring tool - http://svn.python.org/view/sandbox/trunk/2to3/ + https://github.com/python/cpython/tree/ef04c44e29a8276a484f58d03a75a2dec516302d/Lib/lib2to3 .. [4] PEP: rename it.next() to it.__next__()... (Collin Winter) https://mail.python.org/pipermail/python-3000/2007-March/006020.html @@ -213,14 +211,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3115.txt b/peps/pep-3115.rst similarity index 91% rename from pep-3115.txt rename to peps/pep-3115.rst index 1820ca41a..a8c2560dc 100644 --- a/pep-3115.txt +++ b/peps/pep-3115.rst @@ -1,14 +1,12 @@ PEP: 3115 Title: Metaclasses in Python 3000 -Version: $Revision$ -Last-Modified: $Date$ Author: Talin Status: Final Type: Standards Track Content-Type: text/x-rst Created: 07-Mar-2007 Python-Version: 3.0 -Post-History: 11-March-2007, 14-March-2007 +Post-History: 11-Mar-2007, 14-Mar-2007 Abstract @@ -139,8 +137,8 @@ an arbitrary length list of base classes. After the base classes, there may be one or more keyword arguments, one of which can be *metaclass*. Note that the *metaclass* argument is not included in *kwargs*, since it is filtered out by the normal parameter -assignment algorithm. (Note also that *metaclass* is a keyword- -only argument as per PEP 3102 [6]_.) +assignment algorithm. (Note also that *metaclass* is a +keyword-only argument as per :pep:`3102`.) Even though ``__prepare__`` is not required, the default metaclass ('type') implements it, for the convenience of subclasses calling @@ -297,35 +295,19 @@ the new syntax. References ========== -.. [1] [Python-3000] Metaclasses in Py3K (original proposal) - https://mail.python.org/pipermail/python-3000/2006-December/005030.html +[1] [Python-3000] Metaclasses in Py3K (original proposal) +\ https://mail.python.org/pipermail/python-3000/2006-December/005030.html -.. [2] [Python-3000] Metaclasses in Py3K (Guido's suggested syntax) - https://mail.python.org/pipermail/python-3000/2006-December/005033.html +[2] [Python-3000] Metaclasses in Py3K (Guido's suggested syntax) +\ https://mail.python.org/pipermail/python-3000/2006-December/005033.html -.. [3] [Python-3000] Metaclasses in Py3K (Objections to two-phase init) - https://mail.python.org/pipermail/python-3000/2006-December/005108.html +[3] [Python-3000] Metaclasses in Py3K (Objections to two-phase init) +\ https://mail.python.org/pipermail/python-3000/2006-December/005108.html -.. [4] [Python-3000] Metaclasses in Py3K (Always use an ordered dict) - https://mail.python.org/pipermail/python-3000/2006-December/005118.html - -.. [5] PEP 359: The 'make' statement - - http://www.python.org/dev/peps/pep-0359/ - -.. [6] PEP 3102: Keyword-only arguments - - http://www.python.org/dev/peps/pep-3102/ +[4] [Python-3000] Metaclasses in Py3K (Always use an ordered dict) +\ https://mail.python.org/pipermail/python-3000/2006-December/005118.html Copyright ========= This document has been placed in the public domain. - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3116.txt b/peps/pep-3116.rst similarity index 100% rename from pep-3116.txt rename to peps/pep-3116.rst diff --git a/pep-3117.txt b/peps/pep-3117.rst similarity index 98% rename from pep-3117.txt rename to peps/pep-3117.rst index 738effa60..1ce3241c8 100644 --- a/pep-3117.txt +++ b/peps/pep-3117.rst @@ -44,7 +44,7 @@ future-embracing: the introduction of Unicode characters as an integral constituent of source code. Unicode makes it possible to express much more with much less characters, which -is in accordance with the Zen ("Readability counts.") [ZEN]_. Additionally, it +is in accordance with the :pep:`Zen <20>` ("Readability counts."). Additionally, it eliminates the need for a separate type declaration statement, and last but not least, it makes Python measure up to Perl 6, which already uses Unicode for its operators. [#]_ @@ -205,8 +205,6 @@ References .. [#] Though, if you know the language in question, it may not be *that* unrelated. -.. [ZEN] http://www.python.org/dev/peps/pep-0020/ - .. [#] Well, it would, if there was a Perl 6. .. [#] Since the name ``TypeError`` is already in use, this name has been chosen diff --git a/pep-3118.txt b/peps/pep-3118.rst similarity index 100% rename from pep-3118.txt rename to peps/pep-3118.rst diff --git a/pep-3119.txt b/peps/pep-3119.rst similarity index 95% rename from pep-3119.txt rename to peps/pep-3119.rst index 8cabce613..fda56b404 100644 --- a/pep-3119.txt +++ b/peps/pep-3119.rst @@ -1,12 +1,11 @@ PEP: 3119 Title: Introducing Abstract Base Classes -Version: $Revision$ -Last-Modified: $Date$ Author: Guido van Rossum , Talin Status: Final Type: Standards Track Content-Type: text/x-rst Created: 18-Apr-2007 +Python-Version: 3.0 Post-History: 26-Apr-2007, 11-May-2007 @@ -30,7 +29,7 @@ specific mechanism of ABCs, as contrasted with Interfaces or Generic Functions (GFs), but about clarifying philosophical issues like "what makes a set", "what makes a mapping" and "what makes a sequence". -There's also a companion PEP 3141, which defines ABCs for numeric +There's also a companion :pep:`3141`, which defines ABCs for numeric types. @@ -115,7 +114,7 @@ documentation for the ABC. These standard semantic definitions are not enforced, but are strongly recommended. Like all other things in Python, these promises are in the nature of a -gentlemen's agreement, which in this case means that while the +friendly agreement, which in this case means that while the language does enforce some of the promises made in the ABC, it is up to the implementer of the concrete class to insure that the remaining ones are kept. @@ -139,7 +138,7 @@ The specification follows the categories listed in the abstract: Overloading ``isinstance()`` and ``issubclass()`` ------------------------------------------------- -During the development of this PEP and of its companion, PEP 3141, we +During the development of this PEP and of its companion, :pep:`3141`, we repeatedly faced the choice between standardizing more, fine-grained ABCs or fewer, coarse-grained ones. For example, at one stage, PEP 3141 introduced the following stack of base classes used for complex @@ -153,16 +152,16 @@ MutableComposableSet, HashableComposableSet. The dilemma here is that we'd rather have fewer ABCs, but then what should a user do who needs a less refined ABC? Consider e.g. the -plight of a mathematician who wants to define his own kind of +plight of a mathematician who wants to define their own kind of Transcendental numbers, but also wants float and int to be considered -Transcendental. PEP 3141 originally proposed to patch float.__bases__ +Transcendental. :pep:`3141` originally proposed to patch float.__bases__ for that purpose, but there are some good reasons to keep the built-in types immutable (for one, they are shared between all Python interpreters running in the same address space, as is used by mod_python [16]_). Another example would be someone who wants to define a generic -function (PEP 3124) for any sequence that has an ``append()`` method. +function (:pep:`3124`) for any sequence that has an ``append()`` method. The ``Sequence`` ABC (see below) doesn't promise the ``append()`` method, while ``MutableSequence`` requires not only ``append()`` but also various other mutating methods. @@ -431,7 +430,7 @@ partial ordering). But this cannot be the case: If both ``list`` and ``str`` derived from ``Ordering``, this would imply that ``[1, 2] < (1, 2)`` should be defined (and presumably return False), while in fact (in Python 3000!) such "mixed-mode comparisons" operations are -explicitly forbidden and raise ``TypeError``. See PEP 3100 and [14]_ +explicitly forbidden and raise ``TypeError``. See :pep:`3100` and [14]_ for more information. (This is a special case of a more general issue with operations that take another argument of the same type). @@ -749,7 +748,7 @@ character strings (``str``), deriving from ``Sequence`` and **Open issues:** define the base interfaces for these so alternative implementations and subclasses know what they are in for. This may be -the subject of a new PEP or PEPs (PEP 358 should be co-opted for the +the subject of a new PEP or PEPs (:pep:`358` should be co-opted for the ``bytes`` type). @@ -843,65 +842,54 @@ References .. [1] An Introduction to ABC's, by Talin (https://mail.python.org/pipermail/python-3000/2007-April/006614.html) -.. [2] Incomplete implementation prototype, by GvR - (http://svn.python.org/view/sandbox/trunk/abc/) +[2] Incomplete implementation prototype, by GvR +\ (https://web.archive.org/web/20170223133820/http://svn.python.org/view/sandbox/trunk/abc/) -.. [3] Possible Python 3K Class Tree?, wiki page created by Bill Janssen - (http://wiki.python.org/moin/AbstractBaseClasses) +[3] Possible Python 3K Class Tree?, wiki page created by Bill Janssen +\ (https://wiki.python.org/moin/AbstractBaseClasses) .. [4] Generic Functions implementation, by GvR - (http://svn.python.org/view/sandbox/trunk/overload/) + (https://web.archive.org/web/20170223135019/http://svn.python.org/view/sandbox/trunk/overload/) .. [5] Charming Python: Scaling a new PEAK, by David Mertz - (http://www-128.ibm.com/developerworks/library/l-cppeak2/) + (https://web.archive.org/web/20070515125102/http://www-128.ibm.com/developerworks/library/l-cppeak2/) .. [6] Implementation of @abstractmethod - (https://bugs.python.org/issue1706989) + (https://github.com/python/cpython/issues/44895) .. [7] Unifying types and classes in Python 2.2, by GvR - (http://www.python.org/download/releases/2.2.3/descrintro/) + (https://www.python.org/download/releases/2.2.3/descrintro/) .. [8] Putting Metaclasses to Work: A New Dimension in Object-Oriented Programming, by Ira R. Forman and Scott H. Danforth - (http://www.amazon.com/gp/product/0201433052) + (https://archive.org/details/PuttingMetaclassesToWork) -.. [9] Partial order, in Wikipedia - (http://en.wikipedia.org/wiki/Partial_order) +[9] Partial order, in Wikipedia +\ (https://en.wikipedia.org/wiki/Partial_order) -.. [10] Total order, in Wikipedia - (http://en.wikipedia.org/wiki/Total_order) +[10] Total order, in Wikipedia +\ (https://en.wikipedia.org/wiki/Total_order) .. [11] Finite set, in Wikipedia - (http://en.wikipedia.org/wiki/Finite_set) + (https://en.wikipedia.org/wiki/Finite_set) .. [12] Make isinstance/issubclass overloadable (https://bugs.python.org/issue1708353) .. [13] ABCMeta sample implementation - (http://svn.python.org/view/sandbox/trunk/abc/xyz.py) + (https://web.archive.org/web/20170224195724/http://svn.python.org/view/sandbox/trunk/abc/xyz.py) .. [14] python-dev email ("Comparing heterogeneous types") https://mail.python.org/pipermail/python-dev/2004-June/045111.html .. [15] Function ``frozenset_hash()`` in Object/setobject.c - (http://svn.python.org/view/python/trunk/Objects/setobject.c) + (https://web.archive.org/web/20170224204758/http://svn.python.org/view/python/trunk/Objects/setobject.c) .. [16] Multiple interpreters in mod_python - (http://www.modpython.org/live/current/doc-html/pyapi-interps.html) + (https://web.archive.org/web/20070515132123/http://www.modpython.org/live/current/doc-html/pyapi-interps.html) Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3120.txt b/peps/pep-3120.rst similarity index 90% rename from pep-3120.txt rename to peps/pep-3120.rst index b9cd19e88..1a9762b32 100644 --- a/pep-3120.txt +++ b/peps/pep-3120.rst @@ -15,7 +15,7 @@ Specification ============= This PEP proposes to change the default source encoding from ASCII to -UTF-8. Support for alternative source encodings [#pep263]_ continues to +UTF-8. Support for alternative source encodings (:pep:`263`) continues to exist; an explicit encoding declaration takes precedence over the default. @@ -38,7 +38,7 @@ character-by-character basis. As Unicode gives a fixed interpretation to code points, this algorithm effectively fixed a source encoding, at least for files containing non-ASCII characters in Unicode literals. -PEP 263 identified the problem that you can use only those Unicode +:pep:`263` identified the problem that you can use only those Unicode characters in a Unicode literal which are also in Latin-1, and introduced a syntax for declaring the source encoding. If no source encoding was given, the default should be ASCII. For compatibility @@ -50,10 +50,10 @@ encoding is declared. Rationale ========= -With PEP 263, using arbitrary non-ASCII characters in a Python file is +With :pep:`263`, using arbitrary non-ASCII characters in a Python file is possible, but tedious. One has to explicitly add an encoding declaration. Even though some editors (like IDLE and Emacs) support -the declarations of PEP 263, many editors still do not (and never +the declarations of :pep:`263`, many editors still do not (and never will); users have to explicitly adjust the encoding which the editor assumes on a file-by-file basis. @@ -82,14 +82,6 @@ as the parser converts all source code to UTF-8, anyway). IDLE needs to be changed to use UTF-8 as the default encoding. -References -========== - -.. [#pep263] - http://www.python.org/dev/peps/pep-0263/ - - - Copyright ========= diff --git a/pep-3121.txt b/peps/pep-3121.rst similarity index 100% rename from pep-3121.txt rename to peps/pep-3121.rst diff --git a/pep-3122.txt b/peps/pep-3122.rst similarity index 96% rename from pep-3122.txt rename to peps/pep-3122.rst index f799165fd..5591104bd 100644 --- a/pep-3122.txt +++ b/peps/pep-3122.rst @@ -17,7 +17,7 @@ Abstract ======== Because of how name resolution works for relative imports in a world -where PEP 328 is implemented, the ability to execute modules within a +where :pep:`328` is implemented, the ability to execute modules within a package ceases being possible. This failing stems from the fact that the module being executed as the "main" module replaces its ``__name__`` attribute with ``"__main__"`` instead of leaving it as @@ -31,17 +31,17 @@ module this will allow at least some instances of executing a module within a package that uses relative imports. This PEP does not address the idea of introducing a module-level -function that is automatically executed like PEP 299 proposes. +function that is automatically executed like :pep:`299` proposes. The Problem =========== -With the introduction of PEP 328, relative imports became dependent on +With the introduction of :pep:`328`, relative imports became dependent on the ``__name__`` attribute of the module performing the import. This is because the use of dots in a relative import are used to strip away parts of the calling module's name to calculate where in the package -hierarchy an import should fall (prior to PEP 328 relative +hierarchy an import should fall (prior to :pep:`328` relative imports could fail and would fall back on absolute imports which had a chance of succeeding). @@ -131,7 +131,7 @@ The stat calls alone can be expensive if the file system the executed script is on is something like NFS. Because of these issues, only when the ``-m`` command-line argument -(introduced by PEP 338) is used will ``__name__`` be set. Otherwise +(introduced by :pep:`338`) is used will ``__name__`` be set. Otherwise the fallback semantics of setting ``__name__`` to ``"__main__"`` will occur. ``sys.main`` will still be set to the proper value, regardless of what ``__name__`` is set to. diff --git a/pep-3123.txt b/peps/pep-3123.rst similarity index 100% rename from pep-3123.txt rename to peps/pep-3123.rst diff --git a/pep-3124.txt b/peps/pep-3124.rst similarity index 98% rename from pep-3124.txt rename to peps/pep-3124.rst index d8fad1ac0..e755609a2 100644 --- a/pep-3124.txt +++ b/peps/pep-3124.rst @@ -3,7 +3,7 @@ Title: Overloading, Generic Functions, Interfaces, and Adaptation Version: $Revision$ Last-Modified: $Date$ Author: Phillip J. Eby -Discussions-To: Python 3000 List +Discussions-To: python-3000@python.org Status: Deferred Type: Standards Track Content-Type: text/x-rst @@ -74,7 +74,7 @@ created by a third party. Therefore, this PEP proposes a standard library module to address these, and related issues, using decorators and argument annotations -(PEP 3107). The primary features to be provided are: +(:pep:`3107`). The primary features to be provided are: * a dynamic overloading facility, similar to the static overloading found in languages such as Java and C++, but including optional @@ -166,7 +166,7 @@ or this (to avoid copying the implementation):: from overloading import RuleSet RuleSet(flatten).copy_rules((basestring,), (MyString,)) -(Note also that, although PEP 3119 proposes that it should be possible +(Note also that, although :pep:`3119` proposes that it should be possible for abstract base classes like ``Iterable`` to allow classes like ``MyString`` to claim subclass-hood, such a claim is *global*, throughout the application. In contrast, adding a specific overload @@ -563,7 +563,7 @@ decorators could insert a custom metaclass to do processing of this sort. (This is how RuleDispatch, for example, implements the implicit class rule.) -PEP 3115, however, requires that a class' metaclass be determined +:pep:`3115`, however, requires that a class' metaclass be determined *before* the class body has executed, making it impossible to use this technique for class decoration any more. @@ -959,8 +959,8 @@ about. As a result, the vast majority of overloads can be found adjacent to either the function being overloaded, or to a newly-defined type for -which the overload is adding support. Thus, overloads are highly- -discoverable in the common case, as you are either looking at the +which the overload is adding support. Thus, overloads are +highly-discoverable in the common case, as you are either looking at the function or the type, or both. It is only in rather infrequent cases that one will have overloads in a @@ -1014,7 +1014,7 @@ included in PEAK-Rules at the present time. The "implicit class rule" has previously been implemented in the RuleDispatch library. However, it relies on the ``__metaclass__`` -hook that is currently eliminated in PEP 3115. +hook that is currently eliminated in :pep:`3115`. I don't currently know how to make ``@overload`` play nicely with ``classmethod`` and ``staticmethod`` in class bodies. It's not really diff --git a/pep-3125.txt b/peps/pep-3125.rst similarity index 98% rename from pep-3125.txt rename to peps/pep-3125.rst index 744de708c..9362bf81e 100644 --- a/pep-3125.txt +++ b/peps/pep-3125.rst @@ -219,14 +219,14 @@ References .. [#dedent] (email subject) PEP 30XZ: Simplified Parsing, van Rossum https://mail.python.org/pipermail/python-3000/2007-April/007063.html -.. [#lexical] (email subject) PEP-3125 -- remove backslash +.. [#lexical] (email subject) :pep:`3125` -- remove backslash continuation, Koenig https://mail.python.org/pipermail/python-3000/2007-May/007237.html .. [#snocone] The Snocone Programming Language, Koenig http://www.snobol4.com/report.htm -.. [#guidobughide] (email subject) PEP-3125 -- remove backslash +.. [#guidobughide] (email subject) :pep:`3125` -- remove backslash continuation, van Rossum https://mail.python.org/pipermail/python-3000/2007-May/007244.html diff --git a/pep-3126.txt b/peps/pep-3126.rst similarity index 98% rename from pep-3126.txt rename to peps/pep-3126.rst index 9149dc86d..bc993ce17 100644 --- a/pep-3126.txt +++ b/peps/pep-3126.rst @@ -171,8 +171,8 @@ Operator Precedence Guido indicated [#rcn-constantfold]_ that this change should be handled by PEP, because there were a few edge cases with other string operators, such as the %. (Assuming that str % stays -- it may be -eliminated in favor of PEP 3101 -- Advanced String Formatting. -[#PEP3101]_ [#elimpercent]_) +eliminated in favor of :pep:`3101` -- Advanced String Formatting. +[#elimpercent]_) The resolution is to use parentheses to enforce precedence -- the same solution that can be used today:: @@ -352,9 +352,6 @@ References van Rossum https://mail.python.org/pipermail/python-3000/2007-April/006563.html -.. [#PEP3101] PEP 3101, Advanced String Formatting, Talin - http://www.python.org/dev/peps/pep-3101/ - .. [#elimpercent] ps to question Re: Need help completing ABC pep, van Rossum https://mail.python.org/pipermail/python-3000/2007-April/006737.html diff --git a/pep-3127.txt b/peps/pep-3127.rst similarity index 95% rename from pep-3127.txt rename to peps/pep-3127.rst index 6c38e3ed8..ef840bea7 100644 --- a/pep-3127.txt +++ b/peps/pep-3127.rst @@ -3,7 +3,7 @@ Title: Integer Literal Support and Syntax Version: $Revision$ Last-Modified: $Date$ Author: Patrick Maupin -Discussions-To: Python-3000@python.org +Discussions-To: python-3000@python.org Status: Final Type: Standards Track Content-Type: text/x-rst @@ -80,7 +80,7 @@ as well as the grammar. The documentation will have to be changed as well: grammar.txt, as well as the integer literal section of the reference manual. -PEP 306 should be checked for other issues, and that PEP should +:pep:`306` should be checked for other issues, and that PEP should be updated if the procedure described therein is insufficient. int() specification @@ -133,7 +133,7 @@ option will need to be updated to add '0o' in front, instead of '0'. In 2.6, alternate octal formatting will continue to add only '0'. In neither 2.6 nor 3.0 will the % operator support binary output. This is because -binary output is already supported by PEP 3101 +binary output is already supported by :pep:`3101` (str.format), which is the preferred string formatting method. @@ -182,7 +182,7 @@ the string representation of integers relate to these features: * Under 2.6, long() is treated the same as int() - Formatting of integers into strings, either via the % string - operator or the new PEP 3101 advanced string formatting method. + operator or the new :pep:`3101` advanced string formatting method. It is presumed that: @@ -227,7 +227,7 @@ are confronted with non-decimal radices. However, in most situations, most people do not write gratuitous zeros in front of their decimal numbers. The primary exception is when an attempt is being made to line up columns of numbers. But -since PEP 8 specifically discourages the use of spaces to try to +since :pep:`8` specifically discourages the use of spaces to try to align Python code, one would suspect the same argument should apply to the use of leading zeros for the same purpose. @@ -239,13 +239,13 @@ Assume the rare complete newcomer to computing who *does*, either occasionally or as a matter of habit, use leading zeros for decimal numbers. Python could either: -a) silently do the wrong thing with his numbers, as it does now; +a) silently do the wrong thing with their numbers, as it does now; -b) immediately disabuse him of the notion that this is viable syntax +b) immediately disabuse them of the notion that this is viable syntax (and yes, the SyntaxWarning should be more gentle than it currently is, but that is a subject for a different PEP); or -c) let him continue to think that computers are happy with +c) let them continue to think that computers are happy with multi-digit decimal integers which start with "0". Some people passionately believe that (c) is the correct answer, @@ -253,12 +253,11 @@ and they would be absolutely right if we could be sure that new users will never blossom and grow and start writing AJAX applications. So while a new Python user may (currently) be mystified at the -delayed discovery that his numbers don't work properly, we can -fix it by explaining to him immediately that Python doesn't like +delayed discovery that their numbers don't work properly, we can +fix it by explaining to them immediately that Python doesn't like leading zeros (hopefully with a reasonable message!), or we can delegate this teaching experience to the JavaScript interpreter -in the Internet Explorer browser, and let him try to debug his -issue there. +in the browser, and let them try to debug their issue there. Supported radices ----------------- @@ -439,7 +438,7 @@ with the "x" for "heXadecimal". For the string % operator, "o" was already being used to denote octal. Binary formatting is not being added to the % operator -because PEP 3101 (Advanced String Formatting) already supports +because :pep:`3101` (Advanced String Formatting) already supports binary, % formatting will be deprecated in the future. At the end of the day, since uppercase "O" can look like a zero @@ -466,7 +465,7 @@ ample precedence for case sensitivity in the output format string, and there would need to be a consensus that there is a valid use-case for the "alternate form" of the string % operator to support uppercase 'B' or 'O' characters for binary or -octal output. Currently, PEP 3101 does not even support this +octal output. Currently, :pep:`3101` does not even support this alternate capability, and the hex() function does not allow the programmer to specify the case of the 'x' character. diff --git a/pep-3128.txt b/peps/pep-3128.rst similarity index 99% rename from pep-3128.txt rename to peps/pep-3128.rst index c4c3e8254..0721cb1a2 100644 --- a/pep-3128.txt +++ b/peps/pep-3128.rst @@ -3,7 +3,7 @@ Title: BList: A Faster List-like Type Version: $Revision$ Last-Modified: $Date$ Author: Daniel Stutzbach -Discussions-To: Python 3000 List +Discussions-To: python-3000@python.org Status: Rejected Type: Standards Track Content-Type: text/x-rst diff --git a/pep-3129.txt b/peps/pep-3129.rst similarity index 84% rename from pep-3129.txt rename to peps/pep-3129.rst index 6b9870998..410ec3b9a 100644 --- a/pep-3129.txt +++ b/peps/pep-3129.rst @@ -8,22 +8,23 @@ Type: Standards Track Content-Type: text/x-rst Created: 01-May-2007 Python-Version: 3.0 -Post-History: 7-May-2007 +Post-History: 07-May-2007 Abstract ======== This PEP proposes class decorators, an extension to the function -and method decorators introduced in PEP 318. +and method decorators introduced in :pep:`318`. Rationale ========= When function decorators were originally debated for inclusion in -Python 2.4, class decorators were seen as obscure and unnecessary -[#obscure]_ thanks to metaclasses. After several years' experience +Python 2.4, class decorators were seen as +:pep:`obscure and unnecessary <318#motivation>` +thanks to metaclasses. After several years' experience with the Python 2.4.x series of releases and an increasing familiarity with function decorators and their uses, the BDFL and the community re-evaluated class decorators and recommended their @@ -45,7 +46,8 @@ Semantics ========= The semantics and design goals of class decorators are the same as -for function decorators ([#semantics]_, [#goals]_); the only +for function decorators (:pep:`318#current-syntax`, :pep:`318#design-goals`); +the only difference is that you're decorating a class instead of a function. The following two snippets are semantically identical:: @@ -59,7 +61,7 @@ The following two snippets are semantically identical:: class A: pass -For a detailed examination of decorators, please refer to PEP 318. +For a detailed examination of decorators, please refer to :pep:`318`. Implementation @@ -103,21 +105,12 @@ The patch was committed to Subversion as revision 55430. References ========== -.. [#obscure] - http://www.python.org/dev/peps/pep-0318/#motivation - .. [#approval] https://mail.python.org/pipermail/python-dev/2006-March/062942.html .. [#motivation] https://mail.python.org/pipermail/python-dev/2006-March/062888.html -.. [#semantics] - http://www.python.org/dev/peps/pep-0318/#current-syntax - -.. [#goals] - http://www.python.org/dev/peps/pep-0318/#design-goals - .. [#implementation] https://bugs.python.org/issue1671208 diff --git a/pep-3130.txt b/peps/pep-3130.rst similarity index 100% rename from pep-3130.txt rename to peps/pep-3130.rst diff --git a/pep-3131.txt b/peps/pep-3131.rst similarity index 99% rename from pep-3131.txt rename to peps/pep-3131.rst index b402c5c12..0d8ce9a25 100644 --- a/pep-3131.txt +++ b/peps/pep-3131.rst @@ -193,7 +193,7 @@ A. Should identifiers be allowed to contain any Unicode letter? solved; tool support is weak. 5. Languages with non-ASCII identifiers use different character sets - and normalization schemes; PEP 3131's choices are non-obvious. + and normalization schemes; :pep:`3131`'s choices are non-obvious. 6. The Unicode bidi algorithm yields an extremely confusing display order for RTL text when digits or operators are nearby. diff --git a/pep-3132.txt b/peps/pep-3132.rst similarity index 100% rename from pep-3132.txt rename to peps/pep-3132.rst diff --git a/pep-3133.txt b/peps/pep-3133.rst similarity index 97% rename from pep-3133.txt rename to peps/pep-3133.rst index ae42c5cb1..45c415160 100644 --- a/pep-3133.txt +++ b/peps/pep-3133.rst @@ -15,8 +15,8 @@ Post-History: 13-May-2007 Rejection Notice ================ -This PEP has helped push PEP 3119 towards a saner, more minimalistic -approach. But given the latest version of PEP 3119 I much prefer +This PEP has helped push :pep:`3119` towards a saner, more minimalistic +approach. But given the latest version of :pep:`3119` I much prefer that. GvR. @@ -69,7 +69,7 @@ A Note on Syntax A syntax proposals in this PEP are tentative and should be considered to be strawmen. The necessary bits that this PEP depends -on -- namely PEP 3115's class definition syntax and PEP 3129's class +on -- namely :pep:`3115`'s class definition syntax and :pep:`3129`'s class decorators -- are still being formalized and may change. Function names will, of course, be subject to lengthy bikeshedding debates. @@ -112,7 +112,7 @@ Let's see if roles can help. :: class Rock(Mineral): ... -We use class decorators from PEP 3129 to associate a particular role +We use class decorators from :pep:`3129` to associate a particular role or roles with a class. Client code can now verify that an incoming object performs the ``Doglike`` role, allowing it to handle ``Wolf``, ``LaughingHyena`` and ``Aibo`` [#aibo]_ instances, too. @@ -302,7 +302,7 @@ Relationship to Abstract Base Classes ===================================== Early drafts of this PEP [#proposal]_ envisioned roles as competing -with the abstract base classes proposed in PEP 3119. After further +with the abstract base classes proposed in :pep:`3119`. After further discussion and deliberation, a compromise and a delegation of responsibilities and use-cases has been worked out as follows: @@ -507,7 +507,7 @@ into :: ... Assigning a role could take advantage of the class definition -arguments proposed in PEP 3115: :: +arguments proposed in :pep:`3115`: :: class MyClass(performs=MyRole): ... diff --git a/pep-3134.txt b/peps/pep-3134.rst similarity index 99% rename from pep-3134.txt rename to peps/pep-3134.rst index d9f8abfdc..97127dfdf 100644 --- a/pep-3134.txt +++ b/peps/pep-3134.rst @@ -14,7 +14,7 @@ Post-History: Numbering Note ============== -This PEP started its life as PEP 344. Since it is now targeted for Python +This PEP started its life as :pep:`344`. Since it is now targeted for Python 3000, it has been moved into the 3xxx space. @@ -70,7 +70,7 @@ original exception. Greg Ewing [4]_ and Guido van Rossum [5]_, and probably others, have previously mentioned adding a traceback attribute to Exception instances. -This is noted in PEP 3000. +This is noted in :pep:`3000`. This PEP was motivated by yet another recent Python-Dev reposting of the same ideas [6]_ [7]_. @@ -466,7 +466,7 @@ Possible Future Compatible Changes These changes are consistent with the appearance of exceptions as a single object rather than a triple at the interpreter level. -- If PEP 340 or PEP 343 is accepted, replace the three (``type``, ``value``, +- If :pep:`340` or :pep:`343` is accepted, replace the three (``type``, ``value``, ``traceback``) arguments to ``__exit__`` with a single exception argument. - Deprecate ``sys.exc_type``, ``sys.exc_value``, ``sys.exc_traceback``, and diff --git a/pep-3135.txt b/peps/pep-3135.rst similarity index 81% rename from pep-3135.txt rename to peps/pep-3135.rst index 6c93d49f9..cbd13aa8b 100644 --- a/pep-3135.txt +++ b/peps/pep-3135.rst @@ -1,7 +1,5 @@ PEP: 3135 Title: New Super -Version: $Revision$ -Last-Modified: $Date$ Author: Calvin Spealman , Tim Delaney , Lie Ryan @@ -10,14 +8,20 @@ Type: Standards Track Content-Type: text/x-rst Created: 28-Apr-2007 Python-Version: 3.0 -Post-History: 28-Apr-2007, 29-Apr-2007 (1), 29-Apr-2007 (2), 14-May-2007, 12-Mar-2009 +Post-History: `28-Apr-2007 `__, + `29-Apr-2007 `__, + `29-Apr-2007 `__, + `14-May-2007 `__, + `12-Mar-2009 `__ + Numbering Note ============== -This PEP started its life as PEP 367. Since it is now targeted +This PEP started its life as :pep:`367`. Since it is now targeted for Python 3000, it has been moved into the 3xxx space. + Abstract ======== @@ -34,6 +38,7 @@ to replace the old:: super(Foo, self).foo(1, 2) + Rationale ========= @@ -79,6 +84,7 @@ of ``super`` in a method definition and only passes in the ``__class__`` cell when this is found. Thus, calling a global alias of ``super`` without arguments will not necessarily work. + Closed Issues ============= @@ -103,7 +109,7 @@ However, this was found to be false, because calling an object only looks up the __call__ method directly on the object's type. The following example shows this in action. -:: +.. code:: python class A(object): def __call__(self): @@ -118,6 +124,7 @@ this in action. In any case, this issue goes away entirely because classic calls to ``super(, )`` are still supported with the same meaning. + Alternative Proposals ===================== @@ -158,10 +165,10 @@ super(self, \*args) or __super__(self, \*args) This solution only solves the problem of the type indication, does not handle differently named super methods, and is explicit about the name of the instance. It is less flexible without being able to enacted on other method -names, in cases where that is needed. One use case this fails is where a base- -class has a factory classmethod and a subclass has two factory classmethods, -both of which needing to properly make super calls to the one in the base- -class. +names, in cases where that is needed. One use case this fails is where a +base-class has a factory classmethod and a subclass has two factory +classmethods,both of which needing to properly make super calls to the one +in the base-class. super.foo(self, \*args) ----------------------- @@ -178,12 +185,12 @@ be equivalent to calling the method on the ``super`` object with the same name as the method currently being executed i.e. the following two methods would be equivalent: -:: +.. code:: python def f(self, *p, **kw): super.f(*p, **kw) -:: +.. code:: python def f(self, *p, **kw): super(*p, **kw) @@ -193,41 +200,34 @@ concerns are obvious. Guido has suggested that this should be excluded from this PEP on the principle of KISS (Keep It Simple Stupid). - History ======= -12-Mar-2009 - Updated to reflect the current state of implementation. -29-Apr-2007 - Changed title from "Super As A Keyword" to "New Super" - - Updated much of the language and added a terminology section - for clarification in confusing places. - - Added reference implementation and history sections. +29-Apr-2007 + - Changed title from "Super As A Keyword" to "New Super" + - Updated much of the language and added a terminology section + for clarification in confusing places. + - Added reference implementation and history sections. + +06-May-2007 + - Updated by Tim Delaney to reflect discussions on the python-3000 + and python-dev mailing lists. + +12-Mar-2009 + - Updated to reflect the current state of implementation. -06-May-2007 - Updated by Tim Delaney to reflect discussions on the python-3000 - and python-dev mailing lists. References ========== -.. [1] Fixing super anyone? - (https://mail.python.org/pipermail/python-3000/2007-April/006667.html) +[1] Fixing super anyone? +\ (https://mail.python.org/pipermail/python-3000/2007-April/006667.html) -.. [2] PEP 3130: Access to Module/Class/Function Currently Being Defined (this) - (https://mail.python.org/pipermail/python-ideas/2007-April/000542.html) +[2] PEP 3130: Access to Module/Class/Function Currently Being Defined (this) +\ (https://mail.python.org/pipermail/python-ideas/2007-April/000542.html) Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3136.txt b/peps/pep-3136.rst similarity index 100% rename from pep-3136.txt rename to peps/pep-3136.rst diff --git a/pep-3137.txt b/peps/pep-3137.rst similarity index 95% rename from pep-3137.txt rename to peps/pep-3137.rst index 7e94b5fa5..4db260b57 100644 --- a/pep-3137.txt +++ b/peps/pep-3137.rst @@ -17,7 +17,7 @@ After releasing Python 3.0a1 with a mutable bytes type, pressure mounted to add a way to represent immutable bytes. Gregory P. Smith proposed a patch that would allow making a bytes object temporarily immutable by requesting that the data be locked using the new buffer -API from PEP 3118. This did not seem the right approach to me. +API from :pep:`3118`. This did not seem the right approach to me. Jeffrey Yasskin, with the help of Adam Hupp, then prepared a patch to make the bytes type immutable (by crudely removing all mutating APIs) @@ -65,7 +65,7 @@ I propose the following type names at the Python level: - ``memoryview`` is a bytes view on another object (PyMemory) The old type named ``buffer`` is so similar to the new type -``memoryview``, introduce by PEP 3118, that it is redundant. The rest +``memoryview``, introduce by :pep:`3118`, that it is redundant. The rest of this PEP doesn't discuss the functionality of ``memoryview``; it is just mentioned here to justify getting rid of the old ``buffer`` type. (An earlier version of this PEP proposed ``buffer`` as the new name @@ -105,7 +105,7 @@ Functionality PEP 3118 Buffer API ------------------- -Both bytes and bytearray implement the PEP 3118 buffer API. The bytes +Both bytes and bytearray implement the :pep:`3118` buffer API. The bytes type only implements read-only requests; the bytearray type allows writable and data-locked requests as well. The element data type is always 'B' (i.e. unsigned byte). @@ -164,7 +164,7 @@ Slicing a bytes object returns a bytes object. Slicing a bytearray object returns a bytearray object. Slice assignment to a bytearray object accepts anything that -implements the PEP 3118 buffer API, or an iterable of integers in +implements the :pep:`3118` buffer API, or an iterable of integers in range(256). Indexing @@ -201,7 +201,7 @@ types, except where mentioned: - ``b *= n``: mutates b if it is a bytearray object. - ``b1 in b2``, ``b1 not in b2``: substring test; b1 can be any - object implementing the PEP 3118 buffer API. + object implementing the :pep:`3118` buffer API. - ``i in b``, ``i not in b``: single-byte membership test; i must be an integer (if it is a length-1 bytes array, it is considered @@ -218,7 +218,7 @@ Methods ------- The following methods are implemented by bytes as well as bytearray, with -similar semantics. They accept anything that implements the PEP 3118 +similar semantics. They accept anything that implements the :pep:`3118` buffer API for bytes arguments, and return the same type as the object whose method is called ("self"):: @@ -246,7 +246,7 @@ which constructs an object from a string containing hexadecimal values (with or without spaces between the bytes). The bytearray type implements these additional methods from the -MutableSequence ABC (see PEP 3119): +MutableSequence ABC (see :pep:`3119`): .extend(), .insert(), .append(), .reverse(), .pop(), .remove(). @@ -275,7 +275,7 @@ is just a special case of conversion to str. There is however no promise that printing a bytes object interprets the individual bytes as characters (unlike in Python 2.x). -The str type currently implements the PEP 3118 buffer API. While this +The str type currently implements the :pep:`3118` buffer API. While this is perhaps occasionally convenient, it is also potentially confusing, because the bytes accessed via the buffer API represent a platform-depending encoding: depending on the platform byte order and @@ -283,7 +283,7 @@ a compile-time configuration option, the encoding could be UTF-16-BE, UTF-16-LE, UTF-32-BE, or UTF-32-LE. Worse, a different implementation of the str type might completely change the bytes representation, e.g. to UTF-8, or even make it impossible to access the data as a -contiguous array of bytes at all. Therefore, the PEP 3118 buffer API +contiguous array of bytes at all. Therefore, the :pep:`3118` buffer API will be removed from the str type. The ``basestring`` Type diff --git a/pep-3138.txt b/peps/pep-3138.rst similarity index 99% rename from pep-3138.txt rename to peps/pep-3138.rst index b5018fc7a..d66498dde 100644 --- a/pep-3138.txt +++ b/peps/pep-3138.rst @@ -2,11 +2,12 @@ PEP: 3138 Title: String representation in Python 3000 Version: $Revision$ Last-Modified: $Date$ -Author: Atsuo Ishimoto +Author: Atsuo Ishimoto Status: Final Type: Standards Track Content-Type: text/x-rst Created: 05-May-2008 +Python-Version: 3.0 Post-History: 05-May-2008, 05-Jun-2008 @@ -103,7 +104,7 @@ Specification '\\uXXXX'. * Convert non-printable characters (Py_UNICODE_ISPRINTABLE() returns - 0) to 'xXX', '\\uXXXX' or '\\U00xxxxxx'. + 0) to '\\xXX', '\\uXXXX' or '\\U00xxxxxx'. * Backslash-escape quote characters (apostrophe, 0x27) and add a quote character at the beginning and the end. diff --git a/pep-3139.txt b/peps/pep-3139.rst similarity index 96% rename from pep-3139.txt rename to peps/pep-3139.rst index b2a19e006..d6b79ffff 100644 --- a/pep-3139.txt +++ b/peps/pep-3139.rst @@ -185,10 +185,11 @@ Copyright -Local Variables: -mode: indented-text -indent-tabs-mode: nil -sentence-end-double-space: t -fill-column: 70 -coding: utf-8 -End: +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff --git a/pep-3140.txt b/peps/pep-3140.rst similarity index 97% rename from pep-3140.txt rename to peps/pep-3140.rst index ffa947bfd..65060f490 100644 --- a/pep-3140.txt +++ b/peps/pep-3140.rst @@ -3,7 +3,7 @@ Title: str(container) should call str(item), not repr(item) Version: $Revision$ Last-Modified: $Date$ Author: Oleg Broytman , - Jim J. Jewett + Jim J. Jewett Discussions-To: python-3000@python.org Status: Rejected Type: Standards Track @@ -79,13 +79,13 @@ string if the input is non-ASCII string):: >>> print(['тДст']) ['\xd4\xc5\xd3\xd4'] -One of the motivations for PEP 3138 is that neither ``repr`` nor ``str`` +One of the motivations for :pep:`3138` is that neither ``repr`` nor ``str`` will allow the sensible printing of dicts whose keys are non-ASCII text strings. Now that Unicode identifiers are allowed, it includes Python's own attribute dicts. This also includes JSON serialization (and caused some hoops for the json lib). -PEP 3138 proposes to fix this by breaking the "repr is safe ASCII" +:pep:`3138` proposes to fix this by breaking the "repr is safe ASCII" invariant, and changing the way ``repr`` (which is used for persistence) outputs some objects, with system-dependent failures. diff --git a/pep-3141.txt b/peps/pep-3141.rst similarity index 98% rename from pep-3141.txt rename to peps/pep-3141.rst index 06ec45403..f68516ef4 100644 --- a/pep-3141.txt +++ b/peps/pep-3141.rst @@ -7,6 +7,7 @@ Status: Final Type: Standards Track Content-Type: text/x-rst Created: 23-Apr-2007 +Python-Version: 3.0 Post-History: 25-Apr-2007, 16-May-2007, 02-Aug-2007 @@ -34,7 +35,7 @@ Specification This PEP specifies a set of Abstract Base Classes, and suggests a general strategy for implementing some of the methods. It uses -terminology from PEP 3119, but the hierarchy is intended to be +terminology from :pep:`3119`, but the hierarchy is intended to be meaningful for any systematic method of defining sets of classes. The type checks in the standard library should use these classes @@ -84,7 +85,7 @@ numbers are supported by this hierarchy. :: @abstractproperty def imag(self): - """Retrieve the real component of this number. + """Retrieve the imaginary component of this number. This should subclass Real. """ @@ -518,9 +519,6 @@ tower. References ========== -.. [#pep3119] Introducing Abstract Base Classes - (http://www.python.org/dev/peps/pep-3119/) - .. [#classtree] Possible Python 3K Class Tree?, wiki page by Bill Janssen (http://wiki.python.org/moin/AbstractBaseClasses) diff --git a/pep-3142.txt b/peps/pep-3142.rst similarity index 88% rename from pep-3142.txt rename to peps/pep-3142.rst index ffa9eec51..15abbd841 100644 --- a/pep-3142.txt +++ b/peps/pep-3142.rst @@ -22,8 +22,8 @@ This PEP proposes an enhancement to generator expressions, adding a Rationale ========= -A generator expression (PEP 289 [1]_) is a concise method to serve -dynamically-generated objects to list comprehensions (PEP 202 [2]_). +A generator expression (:pep:`289`) is a concise method to serve +dynamically-generated objects to list comprehensions (:pep:`202`). Current generator expressions allow for an "if" clause to filter the objects that are returned to those meeting some set of criteria. However, since the "if" clause is evaluated for every @@ -33,7 +33,7 @@ objects would be rejected after a certain point. For example:: g = (n for n in range(100) if n*n < 50) which is equivalent to the using a generator function -(PEP 255 [3]_):: +(:pep:`255`):: def __gen(exp): for n in exp: @@ -105,19 +105,6 @@ Raymond Hettinger first proposed the concept of generator expressions in January 2002. -References -========== - -.. [1] PEP 289: Generator Expressions - http://www.python.org/dev/peps/pep-0289/ - -.. [2] PEP 202: List Comprehensions - http://www.python.org/dev/peps/pep-0202/ - -.. [3] PEP 255: Simple Generators - http://www.python.org/dev/peps/pep-0255/ - - Copyright ========= diff --git a/pep-3143.txt b/peps/pep-3143.rst similarity index 75% rename from pep-3143.txt rename to peps/pep-3143.rst index fa0858d25..a360aa374 100644 --- a/pep-3143.txt +++ b/peps/pep-3143.rst @@ -38,7 +38,7 @@ Specification Example usage ============= -Simple example of direct `DaemonContext` usage:: +Simple example of direct ``DaemonContext`` usage:: import daemon @@ -90,24 +90,24 @@ More complex example usage:: Interface ========= -A new package, `daemon`, is added to the standard library. +A new package, ``daemon``, is added to the standard library. -A class, `DaemonContext`, is defined to represent the settings and +A class, ``DaemonContext``, is defined to represent the settings and process context for the program running as a daemon process. ``DaemonContext`` objects ========================= -A `DaemonContext` instance represents the behaviour settings and +A ``DaemonContext`` instance represents the behaviour settings and process context for the program when it becomes a daemon. The behaviour and environment is customised by setting options on the -instance, before calling the `open` method. +instance, before calling the ``open`` method. -Each option can be passed as a keyword argument to the `DaemonContext` +Each option can be passed as a keyword argument to the ``DaemonContext`` constructor, or subsequently altered by assigning to an attribute on -the instance at any time prior to calling `open`. That is, for -options named `wibble` and `wubble`, the following invocation:: +the instance at any time prior to calling ``open``. That is, for +options named ``wibble`` and ``wubble``, the following invocation:: foo = daemon.DaemonContext(wibble=bar, wubble=baz) foo.open() @@ -121,24 +121,24 @@ is equivalent to:: The following options are defined. -`files_preserve` +``files_preserve`` :Default: ``None`` List of files that should *not* be closed when starting the daemon. If ``None``, all open file descriptors will be closed. Elements of the list are file descriptors (as returned by a file - object's `fileno()` method) or Python `file` objects. Each + object's ``fileno()`` method) or Python ``file`` objects. Each specifies a file that is not to be closed during daemon start. -`chroot_directory` +``chroot_directory`` :Default: ``None`` Full path to a directory to set as the effective root directory of the process. If ``None``, specifies that the root directory is not to be changed. -`working_directory` +``working_directory`` :Default: ``'/'`` Full path of the working directory to which the process should @@ -149,7 +149,7 @@ The following options are defined. be left at default or set to a directory that is a sensible “home directory” for the daemon while it is running. -`umask` +``umask`` :Default: ``0`` File access creation mask (“umask”) to set for the process on @@ -159,13 +159,13 @@ The following options are defined. starting the daemon will reset the umask to this value so that files are created by the daemon with access modes as it expects. -`pidfile` +``pidfile`` :Default: ``None`` Context manager for a PID lock file. When the daemon context opens - and closes, it enters and exits the `pidfile` context manager. + and closes, it enters and exits the ``pidfile`` context manager. -`detach_process` +``detach_process`` :Default: ``None`` If ``True``, detach the process context when opening the daemon @@ -174,10 +174,10 @@ The following options are defined. If unspecified (``None``) during initialisation of the instance, this will be set to ``True`` by default, and ``False`` only if detaching the process is determined to be redundant; for example, - in the case when the process was started by `init`, by `initd`, or - by `inetd`. + in the case when the process was started by ``init``, by ``initd``, or + by ``inetd``. -`signal_map` +``signal_map`` :Default: system-dependent Mapping from operating system signals to callback actions. @@ -215,10 +215,10 @@ The following options are defined. on how to determine what circumstances dictate the need for signal handlers. -`uid` +``uid`` :Default: ``os.getuid()`` -`gid` +``gid`` :Default: ``os.getgid()`` The user ID (“UID”) value and group ID (“GID”) value to switch @@ -228,122 +228,122 @@ The following options are defined. relinquish any effective privilege elevation inherited by the process. -`prevent_core` +``prevent_core`` :Default: ``True`` If true, prevents the generation of core files, in order to avoid - leaking sensitive information from daemons run as `root`. + leaking sensitive information from daemons run as ``root``. -`stdin` +``stdin`` :Default: ``None`` -`stdout` +``stdout`` :Default: ``None`` -`stderr` +``stderr`` :Default: ``None`` - Each of `stdin`, `stdout`, and `stderr` is a file-like object + Each of ``stdin``, ``stdout``, and ``stderr`` is a file-like object which will be used as the new file for the standard I/O stream - `sys.stdin`, `sys.stdout`, and `sys.stderr` respectively. The file + ``sys.stdin``, ``sys.stdout``, and ``sys.stderr`` respectively. The file should therefore be open, with a minimum of mode 'r' in the case - of `stdin`, and mode 'w+' in the case of `stdout` and `stderr`. + of ``stdin``, and mode 'w+' in the case of ``stdout`` and ``stderr``. - If the object has a `fileno()` method that returns a file + If the object has a ``fileno()`` method that returns a file descriptor, the corresponding file will be excluded from being closed during daemon start (that is, it will be treated as though - it were listed in `files_preserve`). + it were listed in ``files_preserve``). If ``None``, the corresponding system stream is re-bound to the - file named by `os.devnull`. + file named by ``os.devnull``. The following methods are defined. -`open()` +``open()`` :Return: ``None`` Open the daemon context, turning the current program into a daemon process. This performs the following steps: - * If this instance's `is_open` property is true, return - immediately. This makes it safe to call `open` multiple times on + * If this instance's ``is_open`` property is true, return + immediately. This makes it safe to call ``open`` multiple times on an instance. - * If the `prevent_core` attribute is true, set the resource limits + * If the ``prevent_core`` attribute is true, set the resource limits for the process to prevent any core dump from the process. - * If the `chroot_directory` attribute is not ``None``, set the + * If the ``chroot_directory`` attribute is not ``None``, set the effective root directory of the process to that directory (via - `os.chroot`). + ``os.chroot``). This allows running the daemon process inside a “chroot gaol” as a means of limiting the system's exposure to rogue behaviour by the process. Note that the specified directory needs to already be set up for this purpose. - * Set the process UID and GID to the `uid` and `gid` attribute + * Set the process UID and GID to the ``uid`` and ``gid`` attribute values. * Close all open file descriptors. This excludes those listed in - the `files_preserve` attribute, and those that correspond to the - `stdin`, `stdout`, or `stderr` attributes. + the ``files_preserve`` attribute, and those that correspond to the + ``stdin``, ``stdout``, or ``stderr`` attributes. * Change current working directory to the path specified by the - `working_directory` attribute. + ``working_directory`` attribute. * Reset the file access creation mask to the value specified by - the `umask` attribute. + the ``umask`` attribute. - * If the `detach_process` option is true, detach the current + * If the ``detach_process`` option is true, detach the current process into its own process group, and disassociate from any controlling terminal. - * Set signal handlers as specified by the `signal_map` attribute. + * Set signal handlers as specified by the ``signal_map`` attribute. - * If any of the attributes `stdin`, `stdout`, `stderr` are not - ``None``, bind the system streams `sys.stdin`, `sys.stdout`, - and/or `sys.stderr` to the files represented by the + * If any of the attributes ``stdin``, ``stdout``, ``stderr`` are not + ``None``, bind the system streams ``sys.stdin``, ``sys.stdout``, + and/or ``sys.stderr`` to the files represented by the corresponding attributes. Where the attribute has a file descriptor, the descriptor is duplicated (instead of re-binding the name). - * If the `pidfile` attribute is not ``None``, enter its context + * If the ``pidfile`` attribute is not ``None``, enter its context manager. - * Mark this instance as open (for the purpose of future `open` and - `close` calls). + * Mark this instance as open (for the purpose of future ``open`` and + ``close`` calls). - * Register the `close` method to be called during Python's exit + * Register the ``close`` method to be called during Python's exit processing. When the function returns, the running program is a daemon process. -`close()` +``close()`` :Return: ``None`` Close the daemon context. This performs the following steps: - * If this instance's `is_open` property is false, return - immediately. This makes it safe to call `close` multiple times + * If this instance's ``is_open`` property is false, return + immediately. This makes it safe to call ``close`` multiple times on an instance. - * If the `pidfile` attribute is not ``None``, exit its context + * If the ``pidfile`` attribute is not ``None``, exit its context manager. - * Mark this instance as closed (for the purpose of future `open` - and `close` calls). + * Mark this instance as closed (for the purpose of future ``open`` + and ``close`` calls). -`is_open` +``is_open`` :Return: ``True`` if the instance is open, ``False`` otherwise. This property exposes the state indicating whether the instance is - currently open. It is ``True`` if the instance's `open` method has - been called and the `close` method has not subsequently been + currently open. It is ``True`` if the instance's ``open`` method has + been called and the ``close`` method has not subsequently been called. -`terminate(signal_number, stack_frame)` +``terminate(signal_number, stack_frame)`` :Return: ``None`` Signal handler for the ``signal.SIGTERM`` signal. Performs the @@ -354,16 +354,16 @@ The following methods are defined. The class also implements the context manager protocol via ``__enter__`` and ``__exit__`` methods. -`__enter__()` +``__enter__()`` :Return: The ``DaemonContext`` instance - Call the instance's `open()` method, then return the instance. + Call the instance's ``open()`` method, then return the instance. -`__exit__(exc_type, exc_value, exc_traceback)` +``__exit__(exc_type, exc_value, exc_traceback)`` :Return: ``True`` or ``False`` as defined by the context manager protocol - Call the instance's `close()` method, then return ``True`` if the + Call the instance's ``close()`` method, then return ``True`` if the exception was handled or ``False`` if it was not. @@ -409,13 +409,13 @@ following steps to become a Unix daemon process. * Correctly handle the following circumstances: - * Started by System V `init` process. + * Started by System V ``init`` process. * Daemon termination by ``SIGTERM`` signal. * Children generate ``SIGCLD`` signal. -The `daemon` tool [slack-daemon]_ lists (in its summary of features) +The ``daemon`` tool [slack-daemon]_ lists (in its summary of features) behaviour that should be performed when turning a program into a well-behaved Unix daemon process. It differs from this PEP's intent in that it invokes a *separate* program as a daemon process. The @@ -424,7 +424,7 @@ once the program is already running: * Sets up the correct process context for a daemon. -* Behaves sensibly when started by `initd(8)` or `inetd(8)`. +* Behaves sensibly when started by ``initd(8)`` or ``inetd(8)``. * Revokes any suid or sgid privileges to reduce security risks in case daemon is incorrectly installed with special privileges. @@ -468,7 +468,7 @@ a service. Reference Implementation ======================== -The `python-daemon` package [python-daemon]_. +The ``python-daemon`` package [python-daemon]_. Other daemon implementations ============================ @@ -482,23 +482,23 @@ following implementations: * Many good ideas were contributed by the community to Python cookbook recipes #66012 [cookbook-66012]_ and #278731 [cookbook-278731]_. -* The `bda.daemon` library [bda.daemon]_ is an implementation of +* The ``bda.daemon`` library [bda.daemon]_ is an implementation of [cookbook-66012]_. It is the predecessor of [python-daemon]_. Other Python daemon implementations that differ from this PEP: -* The `zdaemon` tool [zdaemon]_ was written for the Zope project. Like +* The ``zdaemon`` tool [zdaemon]_ was written for the Zope project. Like [slack-daemon]_, it differs from this specification because it is used to run another program as a daemon process. -* The Python library `daemon` [clapper-daemon]_ is (according to its +* The Python library ``daemon`` [clapper-daemon]_ is (according to its homepage) no longer maintained. As of version 1.0.1, it implements the basic steps from [stevens]_. -* The `daemonize` library [seutter-daemonize]_ also implements the +* The ``daemonize`` library [seutter-daemonize]_ also implements the basic steps from [stevens]_. -* Ray Burr's `daemon.py` module [burr-daemon]_ provides the [stevens]_ +* Ray Burr's ``daemon.py`` module [burr-daemon]_ provides the [stevens]_ procedure as well as PID file handling and redirection of output to syslog. @@ -507,8 +507,8 @@ Other Python daemon implementations that differ from this PEP: with the rest of the Twisted framework; it differs significantly from the API in this PEP. -* The Python `initd` library [dagitses-initd]_, which uses - [clapper-daemon]_, implements an equivalent of Unix `initd(8)` for +* The Python ``initd`` library [dagitses-initd]_, which uses + [clapper-daemon]_, implements an equivalent of Unix ``initd(8)`` for controlling a daemon process. @@ -518,17 +518,17 @@ References .. [stevens] - `Unix Network Programming`, W. Richard Stevens, 1994 Prentice + ``Unix Network Programming``, W. Richard Stevens, 1994 Prentice Hall. .. [slack-daemon] - The (non-Python) “libslack” implementation of a `daemon` tool + The (non-Python) “libslack” implementation of a ``daemon`` tool ``_ by “raf” . .. [python-daemon] - The `python-daemon` library + The ``python-daemon`` library ``_ by Ben Finney et al. @@ -544,39 +544,39 @@ References .. [bda.daemon] - The `bda.daemon` library + The ``bda.daemon`` library ``_ by Robert Niederreiter et al. .. [zdaemon] - The `zdaemon` tool ``_ by + The ``zdaemon`` tool ``_ by Guido van Rossum et al. .. [clapper-daemon] - The `daemon` library ``_ by + The ``daemon`` library ``_ by Brian Clapper. .. [seutter-daemonize] - The `daemonize` library ``_ by + The ``daemonize`` library ``_ by Jerry Seutter. .. [burr-daemon] - The `daemon.py` module + The ``daemon.py`` module ``_ by Ray Burr. .. [twisted] - The `Twisted` application framework + The ``Twisted`` application framework ``_ by Glyph Lefkowitz et al. .. [dagitses-initd] - The Python `initd` library ``_ + The Python ``initd`` library ``_ by Michael Andreas Dagitses. diff --git a/pep-3144.txt b/peps/pep-3144.rst similarity index 98% rename from pep-3144.txt rename to peps/pep-3144.rst index 9168df17a..aff805637 100644 --- a/pep-3144.txt +++ b/peps/pep-3144.rst @@ -4,7 +4,7 @@ Version: $Revision$ Last-Modified: $Date$ Author: Peter Moody BDFL-Delegate: Nick Coghlan -Discussions-To: +Discussions-To: ipaddr-py-dev@googlegroups.com Status: Final Type: Standards Track Content-Type: text/x-rst @@ -47,7 +47,7 @@ seeks to provide. Background ========== -PEP 3144 and ``ipaddr`` have been up for inclusion before. The +:pep:`3144` and ``ipaddr`` have been up for inclusion before. The version of the library specified here is backwards incompatible with the version on PyPI and the one which was discussed before. In order to avoid confusing users of the current ``ipaddr``, I've diff --git a/pep-3145.txt b/peps/pep-3145.rst similarity index 83% rename from pep-3145.txt rename to peps/pep-3145.rst index 8a6a8865d..1053b72e3 100644 --- a/pep-3145.txt +++ b/peps/pep-3145.rst @@ -1,8 +1,6 @@ PEP: 3145 Title: Asynchronous I/O For subprocess.Popen -Version: $Revision$ -Last-Modified: $Date$ -Author: (James) Eric Pruitt, Charles R. McCreary, Josiah Carlson +Author: Eric Pruitt, Charles R. McCreary, Josiah Carlson Status: Withdrawn Type: Standards Track Content-Type: text/x-rst @@ -25,7 +23,7 @@ PEP Deferral ============ Further exploration of the concepts covered in this PEP has been deferred -at least until after PEP 3156 has been resolved. +at least until after :pep:`3156` has been resolved. PEP Withdrawal @@ -110,40 +108,38 @@ References ========== .. [1] [ python-Feature Requests-1191964 ] asynchronous Subprocess - https://mail.python.org/pipermail/python-bugs-list/2006-December/ - 036524.html + https://mail.python.org/pipermail/python-bugs-list/2006-December/036524.html .. [2] Daily Life in an Ivory Basement : /feb-07/problems-with-subprocess - http://ivory.idyll.org/blog/feb-07/problems-with-subprocess + http://ivory.idyll.org/blog/problems-with-subprocess.html .. [3] How can I run an external command asynchronously from Python? - Stack Overflow - http://stackoverflow.com/questions/636561/how-can-i-run-an-external- - command-asynchronously-from-python + https://stackoverflow.com/q/636561 .. [4] 18.1. subprocess - Subprocess management - Python v2.6.2 documentation - http://docs.python.org/library/subprocess.html#subprocess.Popen.wait + https://docs.python.org/2.6/library/subprocess.html#subprocess.Popen.wait .. [5] 18.1. subprocess - Subprocess management - Python v2.6.2 documentation - http://docs.python.org/library/subprocess.html#subprocess.Popen.kill + https://docs.python.org/2.6/library/subprocess.html#subprocess.Popen.kill .. [6] Issue 1191964: asynchronous Subprocess - Python tracker - http://bugs.python.org/issue1191964 + https://github.com/python/cpython/issues/41922 .. [7] Module to allow Asynchronous subprocess use on Windows and Posix platforms - ActiveState Code - http://code.activestate.com/recipes/440554/ + https://code.activestate.com/recipes/440554/ -.. [8] subprocess.rst - subprocdev - Project Hosting on Google Code - http://code.google.com/p/subprocdev/source/browse/doc/subprocess.rst?spec=svn2c925e935cad0166d5da85e37c742d8e7f609de5&r=2c925e935cad0166d5da85e37c742d8e7f609de5#437 +[8] subprocess.rst - subprocdev - Project Hosting on Google Code +\ https://web.archive.org/web/20130306074135/http://code.google.com/p/subprocdev/source/browse/doc/subprocess.rst?spec=svn2c925e935cad0166d5da85e37c742d8e7f609de5&r=2c925e935cad0166d5da85e37c742d8e7f609de5 .. [9] subprocdev - Project Hosting on Google Code - http://code.google.com/p/subprocdev + https://code.google.com/archive/p/subprocdev/ .. [10] Python Subprocess Dev - http://subdev.blogspot.com/ + https://subdev.blogspot.com/ -.. [11] https://bugs.python.org/issue18823 -- Idle: use pipes instead of +.. [11] https://github.com/python/cpython/issues/63023 -- Idle: use pipes instead of sockets to talk with user subprocess Copyright @@ -151,4 +147,3 @@ Copyright This P.E.P. is licensed under the Open Publication License; http://www.opencontent.org/openpub/. - diff --git a/pep-3146.txt b/peps/pep-3146.rst similarity index 99% rename from pep-3146.txt rename to peps/pep-3146.rst index 03f32b30d..dcbdbfe05 100644 --- a/pep-3146.txt +++ b/peps/pep-3146.rst @@ -132,10 +132,10 @@ they are used. However, if by chance the historically-untaken branch is now taken, or some integer-optimized ``a + b`` snippet receives two strings, we must support this. We cannot change Python semantics. Each of these sections of optimized machine -code is preceded by a `guard`, which checks whether the simplifying assumptions -we made when optimizing still hold. If the assumptions are still valid, we run -the optimized machine code; if they are not, we revert back to the interpreter -and pick up where we left off. +code is preceded by a ``guard``, which checks whether the simplifying +assumptions we made when optimizing still hold. If the assumptions are still +valid, we run the optimized machine code; if they are not, we revert back to +the interpreter and pick up where we left off. We have chosen to reuse a set of existing compiler libraries called LLVM [#llvm]_ for code generation and code optimization. This has saved our small @@ -848,8 +848,8 @@ Unladen Swallow [#us-oprofile-change]_, other profiling tools should be easy as well, provided they support a similar JIT interface [#oprofile-jit-interface]_. We have documented the process for using oProfile to profile Unladen Swallow -[#oprofile-workflow]_. This document will be merged into CPython's `Doc/` tree -in the merge. +[#oprofile-workflow]_. This document will be merged into CPython's ``Doc/`` +tree in the merge. Addition of C++ to CPython @@ -874,7 +874,7 @@ Lowlights: - Developers must know two related languages, C and C++ to work on the full range of CPython's internals. -- A C++ style guide will need to be developed and enforced. PEP 7 will be +- A C++ style guide will need to be developed and enforced. :pep:`7` will be extended [#pep7-cpp]_ to encompass C++ by taking the relevant parts of the C++ style guides from Unladen Swallow [#us-styleguide]_, LLVM [#llvm-styleguide]_ and Google [#google-styleguide]_. @@ -992,7 +992,7 @@ Proposed Merge Plan We propose focusing our efforts on eventual merger with CPython's 3.x line of development. The BDFL has indicated that 2.7 is to be the final release of CPython's 2.x line of development [#bdfl-27-final]_, and since 2.7 alpha 1 has -already been released [#cpy-27a1]_, we have missed the window. Python 3 is the +:pep:`already been released <373>`, we have missed the window. Python 3 is the future, and that is where we will target our performance efforts. We recommend the following plan for merger of Unladen Swallow into the CPython @@ -1203,12 +1203,6 @@ References .. [#bdfl-27-final] https://mail.python.org/pipermail/python-dev/2010-January/095682.html -.. [#cpy-27a1] - http://www.python.org/dev/peps/pep-0373/ - -.. [#cpy-32]_ - http://www.python.org/dev/peps/pep-0392/ - .. [#us-punchlist] http://code.google.com/p/unladen-swallow/issues/list?q=label:Merger diff --git a/pep-3147-1.dia b/peps/pep-3147-1.dia similarity index 100% rename from pep-3147-1.dia rename to peps/pep-3147-1.dia diff --git a/peps/pep-3147-1.png b/peps/pep-3147-1.png new file mode 100644 index 000000000..8c692667a Binary files /dev/null and b/peps/pep-3147-1.png differ diff --git a/pep-3147.txt b/peps/pep-3147.rst similarity index 67% rename from pep-3147.txt rename to peps/pep-3147.rst index 07a4e4b09..5f3e54fa6 100644 --- a/pep-3147.txt +++ b/peps/pep-3147.rst @@ -1,14 +1,12 @@ PEP: 3147 Title: PYC Repository Directories -Version: $Revision$ -Last-Modified: $Date$ Author: Barry Warsaw Status: Final Type: Standards Track Content-Type: text/x-rst Created: 16-Dec-2009 Python-Version: 3.2 -Post-History: 2010-01-30, 2010-02-25, 2010-03-03, 2010-04-12 +Post-History: 30-Jan-2010, 25-Feb-2010, 03-Mar-2010, 12-Apr-2010 Resolution: https://mail.python.org/pipermail/python-dev/2010-April/099414.html @@ -22,7 +20,7 @@ allowing more than one byte compilation file (.pyc files) to be co-located with the Python source file (.py file). The extension described here can also be used to support different Python compilation caches, such as JIT output that may be produced by an -Unladen Swallow [1]_ enabled C Python. +Unladen Swallow (:pep:`3146`) enabled C Python. Background @@ -32,7 +30,7 @@ CPython compiles its source code into "byte code", and for performance reasons, it caches this byte code on the file system whenever the source file has changes. This makes loading of Python modules much faster because the compilation phase can be bypassed. When your -source file is `foo.py`, CPython caches the byte code in a `foo.pyc` +source file is ``foo.py``, CPython caches the byte code in a ``foo.pyc`` file right next to the source. Byte code files contain two 32-bit big-endian numbers followed by the @@ -61,11 +59,11 @@ with Python 2.6 being the default. This causes a conflict for third party Python source files installed by the system, because you cannot compile a single Python source file -for more than one Python version at a time. When Python finds a `pyc` +for more than one Python version at a time. When Python finds a ``pyc`` file with a non-matching magic number, it falls back to the slower process of recompiling the source. Thus if your system installed a -`/usr/share/python/foo.py`, two different versions of Python would -fight over the `pyc` file and rewrite it each time the source is +``/usr/share/python/foo.py``, two different versions of Python would +fight over the ``pyc`` file and rewrite it each time the source is compiled. (The standard library is unaffected by this, since multiple versions of the stdlib *are* installed on such distributions..) @@ -79,7 +77,7 @@ Python release was added or removed from the distribution. Because of the sheer number of packages available, this amount of work is infeasible. -(PEP 384 [7]_ has been proposed to address binary compatibility issues +(:pep:`384` has been proposed to address binary compatibility issues of third party extension modules across different versions of Python.) Because these distributions cannot share pyc files, elaborate @@ -101,7 +99,7 @@ Proposal Python's import machinery is extended to write and search for byte code cache files in a single directory inside every Python package -directory. This directory will be called `__pycache__`. +directory. This directory will be called ``__pycache__``. Further, pyc file names will contain a magic string (called a "tag") that differentiates the Python version they were compiled for. This @@ -109,37 +107,37 @@ allows multiple byte compiled cache files to co-exist for a single Python source file. The magic tag is implementation defined, but should contain the -implementation name and a version number shorthand, e.g. `cpython-32`. +implementation name and a version number shorthand, e.g. ``cpython-32``. It must be unique among all versions of Python, and whenever the magic -number is bumped, a new magic tag must be defined. An example `pyc` -file for Python 3.2 is thus `foo.cpython-32.pyc`. +number is bumped, a new magic tag must be defined. An example ``pyc`` +file for Python 3.2 is thus ``foo.cpython-32.pyc``. -The magic tag is available in the `imp` module via the `get_tag()` -function. This is parallel to the `imp.get_magic()` function. +The magic tag is available in the ``imp`` module via the ``get_tag()`` +function. This is parallel to the ``imp.get_magic()`` function. This scheme has the added benefit of reducing the clutter in a Python package directory. When a Python source file is imported for the first time, a -`__pycache__` directory will be created in the package directory, if +``__pycache__`` directory will be created in the package directory, if one does not already exist. The pyc file for the imported source will -be written to the `__pycache__` directory, using the magic-tag -formatted name. If either the creation of the `__pycache__` directory +be written to the ``__pycache__`` directory, using the magic-tag +formatted name. If either the creation of the ``__pycache__`` directory or the pyc file inside that fails, the import will still succeed, just -as it does in a pre-PEP-3147 world. +as it does in a pre-:pep:`3147` world. -If the py source file is missing, the pyc file inside `__pycache__` +If the py source file is missing, the pyc file inside ``__pycache__`` will be ignored. This eliminates the problem of accidental stale pyc file imports. For backward compatibility, Python will still support pyc-only distributions, however it will only do so when the pyc file lives in the directory where the py file *would* have been, i.e. not in the -`__pycache__` directory. pyc file outside of `__pycache__` will only +``__pycache__`` directory. pyc file outside of ``__pycache__`` will only be imported if the py source file is missing. -Tools such as `py_compile` [15]_ and `compileall` [16]_ will be -extended to create PEP 3147 formatted layouts automatically, but will +Tools such as ``py_compile`` [15]_ and ``compileall`` [16]_ will be +extended to create :pep:`3147` formatted layouts automatically, but will have an option to create pyc-only distribution layouts. @@ -148,8 +146,8 @@ Examples What would this look like in practice? -Let's say we have a Python package named `alpha` which contains a -sub-package name `beta`. The source directory layout before byte +Let's say we have a Python package named ``alpha`` which contains a +sub-package name ``beta``. The source directory layout before byte compilation might look like this:: alpha/ @@ -220,7 +218,7 @@ As you can see, as long as the Python version identifier string is unique, any number of pyc files can co-exist. These identifier strings are described in more detail below. -A nice property of this layout is that the `__pycache__` directories +A nice property of this layout is that the ``__pycache__`` directories can generally be ignored, such that a normal directory listing would show something like this:: @@ -241,50 +239,50 @@ This is much less cluttered than even today's Python. Python behavior =============== -When Python searches for a module to import (say `foo`), it may find +When Python searches for a module to import (say ``foo``), it may find one of several situations. As per current Python rules, the term "matching pyc" means that the magic number matches the current interpreter's magic number, and the source file's timestamp matches -the timestamp in the `pyc` file exactly. +the timestamp in the ``pyc`` file exactly. Case 0: The steady state ------------------------ -When Python is asked to import module `foo`, it searches for a -`foo.py` file (or `foo` package, but that's not important for this -discussion) along its `sys.path`. If found, Python looks to see if -there is a matching `__pycache__/foo..pyc` file, and if so, -that `pyc` file is loaded. +When Python is asked to import module ``foo``, it searches for a +``foo.py`` file (or ``foo`` package, but that's not important for this +discussion) along its ``sys.path``. If found, Python looks to see if +there is a matching ``__pycache__/foo..pyc`` file, and if so, +that ``pyc`` file is loaded. Case 1: The first import ------------------------ -When Python locates the `foo.py`, if the `__pycache__/foo..pyc` +When Python locates the ``foo.py``, if the ``__pycache__/foo..pyc`` file is missing, Python will create it, also creating the -`__pycache__` directory if necessary. Python will parse and byte -compile the `foo.py` file and save the byte code in -`__pycache__/foo..pyc`. +``__pycache__`` directory if necessary. Python will parse and byte +compile the ``foo.py`` file and save the byte code in +``__pycache__/foo..pyc``. Case 2: The second import ------------------------- -When Python is asked to import module `foo` a second time (in a -different process of course), it will again search for the `foo.py` -file along its `sys.path`. When Python locates the `foo.py` file, it -looks for a matching `__pycache__/foo..pyc` and finding this, +When Python is asked to import module ``foo`` a second time (in a +different process of course), it will again search for the ``foo.py`` +file along its ``sys.path``. When Python locates the ``foo.py`` file, it +looks for a matching ``__pycache__/foo..pyc`` and finding this, it reads the byte code and continues as usual. Case 3: __pycache__/foo..pyc with no source --------------------------------------------------- -It's possible that the `foo.py` file somehow got removed, while +It's possible that the ``foo.py`` file somehow got removed, while leaving the cached pyc file still on the file system. If the -`__pycache__/foo..pyc` file exists, but the `foo.py` file used -to create it does not, Python will raise an `ImportError` when asked +``__pycache__/foo..pyc`` file exists, but the ``foo.py`` file used +to create it does not, Python will raise an ``ImportError`` when asked to import foo. In other words, Python will not import a pyc file from the cache directory unless the source file exists. @@ -293,8 +291,8 @@ Case 4: legacy pyc files and source-less imports ------------------------------------------------ Python will ignore all legacy pyc files when a source file exists next -to it. In other words, if a `foo.pyc` file exists next to the -`foo.py` file, the pyc file will be ignored in all cases +to it. In other words, if a ``foo.pyc`` file exists next to the +``foo.py`` file, the pyc file will be ignored in all cases In order to continue to support source-less distributions though, if the source file is missing, Python will import a lone pyc file if it @@ -304,9 +302,9 @@ lives where the source file would have been. Case 5: read-only file systems ------------------------------ -When the source lives on a read-only file system, or the `__pycache__` +When the source lives on a read-only file system, or the ``__pycache__`` directory or pyc file cannot otherwise be written, all the same rules -apply. This is also the case when `__pycache__` happens to be written +apply. This is also the case when ``__pycache__`` happens to be written with permissions which do not allow for writing containing pyc files. @@ -318,6 +316,7 @@ Here is a flow chart describing how modules are loaded: .. image:: pep-3147-1.png :scale: 75 + :class: invert-in-dark-mode Alternative Python implementations @@ -325,9 +324,9 @@ Alternative Python implementations Alternative Python implementations such as Jython [11]_, IronPython [12]_, PyPy [13]_, Pynie [14]_, and Unladen Swallow can also use the -`__pycache__` directory to store whatever compilation artifacts make +``__pycache__`` directory to store whatever compilation artifacts make sense for their platforms. For example, Jython could store the class -file for the module in `__pycache__/foo.jython-32.class`. +file for the module in ``__pycache__/foo.jython-32.class``. Implementation strategy @@ -337,7 +336,7 @@ This feature is targeted for Python 3.2, solving the problem for those and all future versions. It may be back-ported to Python 2.7. Vendors are free to backport the changes to earlier distributions as they see fit. For backports of this feature to Python 2, when the -`-U` flag is used, a file such as `foo.cpython-27u.pyc` can be +``-U`` flag is used, a file such as ``foo.cpython-27u.pyc`` can be written. @@ -361,9 +360,9 @@ The easiest way to detect whether your version of Python provides PEP __file__ --------- -In Python 3, when you import a module, its `__file__` attribute points -to its source `py` file (in Python 2, it points to the `pyc` file). A -package's `__file__` points to the `py` file for its `__init__.py`. +In Python 3, when you import a module, its ``__file__`` attribute points +to its source ``py`` file (in Python 2, it points to the ``pyc`` file). A +package's ``__file__`` points to the ``py`` file for its ``__init__.py``. E.g.:: >>> import foo @@ -374,65 +373,65 @@ E.g.:: >>> baz.__file__ 'baz/__init__.py' -Nothing in this PEP would change the semantics of `__file__`. +Nothing in this PEP would change the semantics of ``__file__``. -This PEP proposes the addition of an `__cached__` attribute to -modules, which will always point to the actual `pyc` file that was +This PEP proposes the addition of an ``__cached__`` attribute to +modules, which will always point to the actual ``pyc`` file that was read or written. When the environment variable -`$PYTHONDONTWRITEBYTECODE` is set, or the `-B` option is given, or if -the source lives on a read-only filesystem, then the `__cached__` -attribute will point to the location that the `pyc` file *would* have +``$PYTHONDONTWRITEBYTECODE`` is set, or the ``-B`` option is given, or if +the source lives on a read-only filesystem, then the ``__cached__`` +attribute will point to the location that the ``pyc`` file *would* have been written to if it didn't exist. This location of course includes -the `__pycache__` subdirectory in its path. +the ``__pycache__`` subdirectory in its path. -For alternative Python implementations which do not support `pyc` -files, the `__cached__` attribute may point to whatever information -makes sense. E.g. on Jython, this might be the `.class` file for the -module: `__pycache__/foo.jython-32.class`. Some implementations may +For alternative Python implementations which do not support ``pyc`` +files, the ``__cached__`` attribute may point to whatever information +makes sense. E.g. on Jython, this might be the ``.class`` file for the +module: ``__pycache__/foo.jython-32.class``. Some implementations may use multiple compiled files to create the module, in which case -`__cached__` may be a tuple. The exact contents of `__cached__` are +``__cached__`` may be a tuple. The exact contents of ``__cached__`` are Python implementation specific. It is recommended that when nothing sensible can be calculated, -implementations should set the `__cached__` attribute to `None`. +implementations should set the ``__cached__`` attribute to ``None``. py_compile and compileall ------------------------- -Python comes with two modules, `py_compile` [15]_ and `compileall` +Python comes with two modules, ``py_compile`` [15]_ and ``compileall`` [16]_ which support compiling Python modules external to the built-in -import machinery. `py_compile` in particular has intimate knowledge +import machinery. ``py_compile`` in particular has intimate knowledge of byte compilation, so these will be updated to understand the new -layout. The `-b` flag is added to `compileall` for writing legacy -`.pyc` byte-compiled file path names. +layout. The ``-b`` flag is added to ``compileall`` for writing legacy +``.pyc`` byte-compiled file path names. bdist_wininst and the Windows installer --------------------------------------- These tools also compile modules explicitly on installation. If they -do not use `py_compile` and `compileall`, then they would also have to +do not use ``py_compile`` and ``compileall``, then they would also have to be modified to understand the new layout. File extension checks --------------------- -There exists some code which checks for files ending in `.pyc` and -simply chops off the last character to find the matching `.py` file. +There exists some code which checks for files ending in ``.pyc`` and +simply chops off the last character to find the matching ``.py`` file. This code will obviously fail once this PEP is implemented. -To support this use case, we'll add two new methods to the `imp` +To support this use case, we'll add two new methods to the ``imp`` package [17]_: -* `imp.cache_from_source(py_path)` -> `pyc_path` -* `imp.source_from_cache(pyc_path)` -> `py_path` +* ``imp.cache_from_source(py_path)`` -> ``pyc_path`` +* ``imp.source_from_cache(pyc_path)`` -> ``py_path`` Alternative implementations are free to override these functions to return reasonable values based on their own support for this PEP. -These methods are allowed to return `None` when the implementation (or -PEP 302 loader [18]_ in effect) for whatever reason cannot calculate +These methods are allowed to return ``None`` when the implementation (or +:pep:`302` loader in effect) for whatever reason cannot calculate the appropriate file name. They should not raise exceptions. @@ -443,16 +442,16 @@ For versions of Python earlier than 3.2 (and possibly 2.7), it is possible to backport this PEP. However, in Python 3.2 (and possibly 2.7), this behavior will be turned on by default, and in fact, it will replace the old behavior. Backports will need to support the old -layout by default. We suggest supporting PEP 3147 through the use of -an environment variable called `$PYTHONENABLECACHEDIR` or the command -line switch `-Xenablecachedir` to enable the feature. +layout by default. We suggest supporting :pep:`3147` through the use of +an environment variable called ``$PYTHONENABLECACHEDIR`` or the command +line switch ``-Xenablecachedir`` to enable the feature. Makefiles and other dependency tools ------------------------------------ -Makefiles and other tools which calculate dependencies on `.pyc` files -(e.g. to byte-compile the source if the `.pyc` is missing) will have +Makefiles and other tools which calculate dependencies on ``.pyc`` files +(e.g. to byte-compile the source if the ``.pyc`` is missing) will have to be updated to check the new paths. @@ -466,7 +465,7 @@ were considered and rejected during the PEP's development. Hexadecimal magic tags ---------------------- -pyc files inside of the `__pycache__` directories contain a magic tag +pyc files inside of the ``__pycache__`` directories contain a magic tag in their file names. These are mnemonic tags for the actual magic numbers used by the importer. We could have used the hexadecimal representation [10]_ of the binary magic number as a unique @@ -484,17 +483,17 @@ proposed in this PEP. PEP 304 ------- -There is some overlap between the goals of this PEP and PEP 304 [19]_, -which has been withdrawn. However PEP 304 would allow a user to -create a shadow file system hierarchy in which to store `pyc` files. -This concept of a shadow hierarchy for `pyc` files could be used to -satisfy the aims of this PEP. Although the PEP 304 does not indicate +There is some overlap between the goals of this PEP and :pep:`304`, +which has been withdrawn. However :pep:`304` would allow a user to +create a shadow file system hierarchy in which to store ``pyc`` files. +This concept of a shadow hierarchy for ``pyc`` files could be used to +satisfy the aims of this PEP. Although the :pep:`304` does not indicate why it was withdrawn, shadow directories have a number of problems. -The location of the shadow `pyc` files would not be easily discovered +The location of the shadow ``pyc`` files would not be easily discovered and would depend on the proper and consistent use of the -`$PYTHONBYTECODE` environment variable both by the system and by end +``$PYTHONBYTECODE`` environment variable both by the system and by end users. There are also global implications, meaning that while the -system might want to shadow `pyc` files, users might not want to, but +system might want to shadow ``pyc`` files, users might not want to, but the PEP defines only an all-or-nothing approach. As an example of the problem, a common (though fragile) Python idiom @@ -504,9 +503,9 @@ for locating data files is to do something like this:: import foo.bar data_file = join(dirname(foo.bar.__file__), 'my.dat') -This would be problematic since `foo.bar.__file__` will give the -location of the `pyc` file in the shadow directory, and it may not be -possible to find the `my.dat` file relative to the source directory +This would be problematic since ``foo.bar.__file__`` will give the +location of the ``pyc`` file in the shadow directory, and it may not be +possible to find the ``my.dat`` file relative to the source directory from there. @@ -514,11 +513,11 @@ Fat byte compilation files -------------------------- An earlier version of this PEP described "fat" Python byte code files. -These files would contain the equivalent of multiple `pyc` files in a -single `pyf` file, with a lookup table keyed off the appropriate magic +These files would contain the equivalent of multiple ``pyc`` files in a +single ``pyf`` file, with a lookup table keyed off the appropriate magic number. This was an extensible file format so that the first 5 parallel Python implementations could be supported fairly efficiently, -but with extension lookup tables available to scale `pyf` byte code +but with extension lookup tables available to scale ``pyf`` byte code objects as large as necessary. The fat byte compilation files were fairly complex, and inherently @@ -542,7 +541,7 @@ tools that are dependent on the file extension. .pyc ---- -A proposal was floated to call the `__pycache__` directory `.pyc` or +A proposal was floated to call the ``__pycache__`` directory ``.pyc`` or some other dot-file name. This would have the effect on \*nix systems of hiding the directory. There are many reasons why this was rejected by the BDFL [20]_ including the fact that dot-files are only @@ -565,53 +564,45 @@ this is not an April Fools joke :). References ========== -.. [1] PEP 3146 - .. [2] The marshal module: - https://docs.python.org/dev/library/marshal.html + https://docs.python.org/3.1/library/marshal.html .. [3] import.c: - http://svn.python.org/view/python/branches/py3k/Python/import.c?view=markup + https://github.com/python/cpython/blob/v3.2a1/Python/import.c -.. [4] Ubuntu: +.. [4] Ubuntu: https://www.ubuntu.com -.. [5] Debian: +.. [5] Debian: https://www.debian.org .. [6] Debian Python Policy: - http://www.debian.org/doc/packaging-manuals/python-policy/ - -.. [7] PEP 384 + https://www.debian.org/doc/packaging-manuals/python-policy/ .. [8] python-support: - http://wiki.debian.org/DebianPythonFAQ#Whatispython-support.3F + https://web.archive.org/web/20100110123824/http://wiki.debian.org/DebianPythonFAQ#Whatispython-support.3F .. [9] python-central: - http://wiki.debian.org/DebianPythonFAQ#Whatispython-central.3F + https://web.archive.org/web/20100110123824/http://wiki.debian.org/DebianPythonFAQ#Whatispython-central.3F .. [10] binascii.hexlify(): - http://www.python.org/doc/current/library/binascii.html#binascii.hexlify + https://docs.python.org/3.1/library/binascii.html#binascii.hexlify .. [11] Jython: http://www.jython.org/ .. [12] IronPython: http://ironpython.net/ -.. [13] PyPy: http://codespeak.net/pypy/dist/pypy/doc/ +.. [13] PyPy: https://web.archive.org/web/20100310130136/http://codespeak.net/pypy/dist/pypy/doc/ -.. [14] Pynie: http://code.google.com/p/pynie/ +.. [14] Pynie: https://code.google.com/archive/p/pynie/ -.. [15] py_compile: http://docs.python.org/library/py_compile.html +.. [15] py_compile: https://docs.python.org/3.1/library/py_compile.html -.. [16] compileall: http://docs.python.org/library/compileall.html +.. [16] compileall: https://docs.python.org/3.1/library/compileall.html -.. [17] imp: http://www.python.org/doc/current/library/imp.html +.. [17] imp: https://docs.python.org/3.1/library/imp.html -.. [18] PEP 302 +.. [20] https://www.mail-archive.com/python-dev@python.org/msg45203.html -.. [19] PEP 304 - -.. [20] http://www.mail-archive.com/python-dev@python.org/msg45203.html - -.. [21] importlib: http://docs.python.org/3.1/library/importlib.html +[21] importlib: https://docs.python.org/3.1/library/importlib.html .. [22] https://code.launchpad.net/~barry/python/pep3147 @@ -625,7 +616,7 @@ ACKNOWLEDGMENTS Barry Warsaw's original idea was for fat Python byte code files. Martin von Loewis reviewed an early draft of the PEP and suggested the -simplification to store traditional `pyc` and `pyo` files in a +simplification to store traditional ``pyc`` and ``pyo`` files in a directory. Many other people reviewed early versions of this PEP and provided useful feedback including but not limited to: @@ -643,14 +634,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3148.txt b/peps/pep-3148.rst similarity index 72% rename from pep-3148.txt rename to peps/pep-3148.rst index 70c044f12..61c62616c 100644 --- a/pep-3148.txt +++ b/peps/pep-3148.rst @@ -57,21 +57,21 @@ in that module, which work across thread and process boundaries. Interface --------- -The proposed package provides two core classes: `Executor` and -`Future`. An `Executor` receives asynchronous work requests (in terms -of a callable and its arguments) and returns a `Future` to represent +The proposed package provides two core classes: ``Executor`` and +``Future``. An ``Executor`` receives asynchronous work requests (in terms +of a callable and its arguments) and returns a ``Future`` to represent the execution of that work request. Executor '''''''' -`Executor` is an abstract class that provides methods to execute calls +``Executor`` is an abstract class that provides methods to execute calls asynchronously. ``submit(fn, *args, **kwargs)`` Schedules the callable to be executed as ``fn(*args, **kwargs)`` - and returns a `Future` instance representing the execution of the + and returns a ``Future`` instance representing the execution of the callable. This is an abstract method and must be implemented by Executor @@ -81,10 +81,10 @@ asynchronously. Equivalent to ``map(func, *iterables)`` but func is executed asynchronously and several calls to func may be made concurrently. - The returned iterator raises a `TimeoutError` if `__next__()` is + The returned iterator raises a ``TimeoutError`` if ``__next__()`` is called and the result isn't available after *timeout* seconds from - the original call to `map()`. If *timeout* is not specified or - `None` then there is no limit to the wait time. If a call raises + the original call to ``map()``. If *timeout* is not specified or + ``None`` then there is no limit to the wait time. If a call raises an exception then that exception will be raised when its value is retrieved from the iterator. @@ -92,12 +92,12 @@ asynchronously. Signal the executor that it should free any resources that it is using when the currently pending futures are done executing. - Calls to `Executor.submit` and `Executor.map` and made after - shutdown will raise `RuntimeError`. + Calls to ``Executor.submit`` and ``Executor.map`` and made after + shutdown will raise ``RuntimeError``. - If wait is `True` then this method will not return until all the + If wait is ``True`` then this method will not return until all the pending futures are done executing and the resources associated - with the executor have been freed. If wait is `False` then this + with the executor have been freed. If wait is ``False`` then this method will return immediately and the resources associated with the executor will be freed when all pending futures are done executing. Regardless of the value of wait, the entire Python @@ -107,21 +107,21 @@ asynchronously. | ``__enter__()`` | ``__exit__(exc_type, exc_val, exc_tb)`` - When using an executor as a context manager, `__exit__` will call + When using an executor as a context manager, ``__exit__`` will call ``Executor.shutdown(wait=True)``. ProcessPoolExecutor ''''''''''''''''''' -The `ProcessPoolExecutor` class is an `Executor` subclass that uses a +The ``ProcessPoolExecutor`` class is an ``Executor`` subclass that uses a pool of processes to execute calls asynchronously. The callable -objects and arguments passed to `ProcessPoolExecutor.submit` must be +objects and arguments passed to ``ProcessPoolExecutor.submit`` must be pickleable according to the same limitations as the multiprocessing module. -Calling `Executor` or `Future` methods from within a callable -submitted to a `ProcessPoolExecutor` will result in deadlock. +Calling ``Executor`` or ``Future`` methods from within a callable +submitted to a ``ProcessPoolExecutor`` will result in deadlock. ``__init__(max_workers)`` @@ -132,11 +132,11 @@ submitted to a `ProcessPoolExecutor` will result in deadlock. ThreadPoolExecutor '''''''''''''''''' -The `ThreadPoolExecutor` class is an `Executor` subclass that uses a +The ``ThreadPoolExecutor`` class is an ``Executor`` subclass that uses a pool of threads to execute calls asynchronously. -Deadlock can occur when the callable associated with a `Future` waits -on the results of another `Future`. For example:: +Deadlock can occur when the callable associated with a ``Future`` waits +on the results of another ``Future``. For example:: import time def wait_on_b(): @@ -173,28 +173,28 @@ And:: Future Objects '''''''''''''' -The `Future` class encapsulates the asynchronous execution of a -callable. `Future` instances are returned by `Executor.submit`. +The ``Future`` class encapsulates the asynchronous execution of a +callable. ``Future`` instances are returned by ``Executor.submit``. ``cancel()`` Attempt to cancel the call. If the call is currently being executed then it cannot be cancelled and the method will return - `False`, otherwise the call will be cancelled and the method will - return `True`. + ``False``, otherwise the call will be cancelled and the method will + return ``True``. ``cancelled()`` - Return `True` if the call was successfully cancelled. + Return ``True`` if the call was successfully cancelled. ``running()`` - Return `True` if the call is currently being executed and cannot + Return ``True`` if the call is currently being executed and cannot be cancelled. ``done()`` - Return `True` if the call was successfully cancelled or finished + Return ``True`` if the call was successfully cancelled or finished running. ``result(timeout=None)`` @@ -202,10 +202,10 @@ callable. `Future` instances are returned by `Executor.submit`. Return the value returned by the call. If the call hasn't yet completed then this method will wait up to *timeout* seconds. If the call hasn't completed in *timeout* seconds then a - `TimeoutError` will be raised. If *timeout* is not specified or - `None` then there is no limit to the wait time. + ``TimeoutError`` will be raised. If *timeout* is not specified or + ``None`` then there is no limit to the wait time. - If the future is cancelled before completing then `CancelledError` + If the future is cancelled before completing then ``CancelledError`` will be raised. If the call raised then this method will raise the same exception. @@ -215,13 +215,13 @@ callable. `Future` instances are returned by `Executor.submit`. Return the exception raised by the call. If the call hasn't yet completed then this method will wait up to *timeout* seconds. If the call hasn't completed in *timeout* seconds then a - `TimeoutError` will be raised. If *timeout* is not specified or + ``TimeoutError`` will be raised. If *timeout* is not specified or ``None`` then there is no limit to the wait time. - If the future is cancelled before completing then `CancelledError` + If the future is cancelled before completing then ``CancelledError`` will be raised. - If the call completed without raising then `None` is returned. + If the call completed without raising then ``None`` is returned. ``add_done_callback(fn)`` @@ -231,9 +231,9 @@ callable. `Future` instances are returned by `Executor.submit`. Added callables are called in the order that they were added and are always called in a thread belonging to the process that added - them. If the callable raises an `Exception` then it will be + them. If the callable raises an ``Exception`` then it will be logged and ignored. If the callable raises another - `BaseException` then behavior is not defined. + ``BaseException`` then behavior is not defined. If the future has already completed or been cancelled then *fn* will be called immediately. @@ -241,43 +241,43 @@ callable. `Future` instances are returned by `Executor.submit`. Internal Future Methods ^^^^^^^^^^^^^^^^^^^^^^^ -The following `Future` methods are meant for use in unit tests and -`Executor` implementations. +The following ``Future`` methods are meant for use in unit tests and +``Executor`` implementations. ``set_running_or_notify_cancel()`` - Should be called by `Executor` implementations before executing - the work associated with the `Future`. + Should be called by ``Executor`` implementations before executing + the work associated with the ``Future``. - If the method returns `False` then the `Future` was cancelled, - i.e. `Future.cancel` was called and returned `True`. Any threads - waiting on the `Future` completing (i.e. through `as_completed()` - or `wait()`) will be woken up. + If the method returns ``False`` then the ``Future`` was cancelled, + i.e. ``Future.cancel`` was called and returned ``True``. Any threads + waiting on the ``Future`` completing (i.e. through ``as_completed()`` + or ``wait()``) will be woken up. - If the method returns `True` then the `Future` was not cancelled + If the method returns ``True`` then the ``Future`` was not cancelled and has been put in the running state, i.e. calls to - `Future.running()` will return `True`. + ``Future.running()`` will return ``True``. This method can only be called once and cannot be called after - `Future.set_result()` or `Future.set_exception()` have been + ``Future.set_result()`` or ``Future.set_exception()`` have been called. ``set_result(result)`` - Sets the result of the work associated with the `Future`. + Sets the result of the work associated with the ``Future``. ``set_exception(exception)`` - Sets the result of the work associated with the `Future` to the - given `Exception`. + Sets the result of the work associated with the ``Future`` to the + given ``Exception``. Module Functions '''''''''''''''' ``wait(fs, timeout=None, return_when=ALL_COMPLETED)`` - Wait for the `Future` instances (possibly created by different - `Executor` instances) given by *fs* to complete. Returns a named + Wait for the ``Future`` instances (possibly created by different + ``Executor`` instances) given by *fs* to complete. Returns a named 2-tuple of sets. The first set, named "done", contains the futures that completed (finished or were cancelled) before the wait completed. The second set, named "not_done", contains @@ -293,27 +293,27 @@ Module Functions ============================= ================================================== Constant Description ============================= ================================================== - `FIRST_COMPLETED` The method will return when any future finishes or + ``FIRST_COMPLETED`` The method will return when any future finishes or is cancelled. - `FIRST_EXCEPTION` The method will return when any future finishes by + ``FIRST_EXCEPTION`` The method will return when any future finishes by raising an exception. If not future raises an exception then it is equivalent to ALL_COMPLETED. - `ALL_COMPLETED` The method will return when all calls finish. + ``ALL_COMPLETED`` The method will return when all calls finish. ============================= ================================================== ``as_completed(fs, timeout=None)`` - Returns an iterator over the `Future` instances given by *fs* that + Returns an iterator over the ``Future`` instances given by *fs* that yields futures as they complete (finished or were cancelled). Any - futures that completed before `as_completed()` was called will be - yielded first. The returned iterator raises a `TimeoutError` if - `__next__()` is called and the result isn't available after - *timeout* seconds from the original call to `as_completed()`. If - *timeout* is not specified or `None` then there is no limit to the + futures that completed before ``as_completed()`` was called will be + yielded first. The returned iterator raises a ``TimeoutError`` if + ``__next__()`` is called and the result isn't available after + *timeout* seconds from the original call to ``as_completed()``. If + *timeout* is not specified or ``None`` then there is no limit to the wait time. - The `Future` instances can have been created by different - `Executor` instances. + The ``Future`` instances can have been created by different + ``Executor`` instances. Check Prime Example ------------------- @@ -408,7 +408,7 @@ list [3]_. The proposed design is explicit, i.e. it requires that clients be aware that they are consuming Futures. It would be possible to design -a module that would return proxy objects (in the style of `weakref`) +a module that would return proxy objects (in the style of ``weakref``) that could be used transparently. It is possible to build a proxy implementation on top of the proposed explicit mechanism. @@ -425,13 +425,13 @@ the API has been discussed in some detail on stdlib-sig [6]_. The proposed design was discussed on the Python-Dev mailing list [7]_. Following those discussions, the following changes were made: -* The `Executor` class was made into an abstract base class -* The `Future.remove_done_callback` method was removed due to a lack +* The ``Executor`` class was made into an abstract base class +* The ``Future.remove_done_callback`` method was removed due to a lack of convincing use cases -* The `Future.add_done_callback` method was modified to allow the +* The ``Future.add_done_callback`` method was modified to allow the same callable to be added many times -* The `Future` class's mutation methods were better documented to - indicate that they are private to the `Executor` that created them +* The ``Future`` class's mutation methods were better documented to + indicate that they are private to the ``Executor`` that created them ======================== Reference Implementation @@ -445,7 +445,7 @@ References ========== .. [1] - `java.util.concurrent` package documentation + ``java.util.concurrent`` package documentation http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/package-summary.html .. [2] @@ -453,15 +453,15 @@ References http://code.activestate.com/recipes/84317/ .. [3] - `Python-3000` thread, "mechanism for handling asynchronous concurrency" + ``Python-3000`` thread, "mechanism for handling asynchronous concurrency" https://mail.python.org/pipermail/python-3000/2006-April/000960.html .. [4] - `Python 3000` thread, "Futures in Python 3000 (was Re: mechanism for handling asynchronous concurrency)" + ``Python 3000`` thread, "Futures in Python 3000 (was Re: mechanism for handling asynchronous concurrency)" https://mail.python.org/pipermail/python-3000/2006-April/000970.html .. [5] - A discussion of `stream`, a similar concept proposed by Anh Hai Trinh + A discussion of ``stream``, a similar concept proposed by Anh Hai Trinh http://www.mail-archive.com/stdlib-sig@python.org/msg00480.html .. [6] @@ -473,7 +473,7 @@ References https://mail.python.org/pipermail/python-dev/2010-March/098169.html .. [8] - Reference `futures` implementation + Reference ``futures`` implementation http://code.google.com/p/pythonfutures/source/browse/#svn/branches/feedback ========= diff --git a/pep-3149.txt b/peps/pep-3149.rst similarity index 86% rename from pep-3149.txt rename to peps/pep-3149.rst index 537810053..9d0f3a2ad 100644 --- a/pep-3149.txt +++ b/peps/pep-3149.rst @@ -8,14 +8,14 @@ Type: Standards Track Content-Type: text/x-rst Created: 09-Jul-2010 Python-Version: 3.2 -Post-History: 2010-07-14, 2010-07-22 +Post-History: 14-Jul-2010, 22-Jul-2010 Resolution: https://mail.python.org/pipermail/python-dev/2010-September/103408.html Abstract ======== -PEP 3147 [1]_ described an extension to Python's import machinery that +:pep:`3147` described an extension to Python's import machinery that improved the sharing of Python source code, by allowing more than one byte compilation file (.pyc) to be co-located with each source file. @@ -28,10 +28,10 @@ more easily provide more than one Python major version at a time. Background ========== -PEP 3147 defined the file system layout for a pure-Python package, +:pep:`3147` defined the file system layout for a pure-Python package, where multiple versions of Python are available on the system. For -example, where the `alpha` package containing source modules `one.py` -and `two.py` exist on a system with Python 3.2 and 3.3, the post-byte +example, where the ``alpha`` package containing source modules ``one.py`` +and ``two.py`` exist on a system with Python 3.2 and 3.3, the post-byte compilation file system layout would be:: alpha/ @@ -53,7 +53,7 @@ to changes in the ABI. Different configuration/compilation options for the same Python version can result in different ABIs (e.g. --with-wide-unicode). -While PEP 384 [2]_ defines a stable ABI, it will minimize, but not +While :pep:`384` defines a stable ABI, it will minimize, but not eliminate extension module incompatibilities between Python builds or major versions. Thus a mechanism for discriminating extension module file names is proposed. @@ -69,18 +69,18 @@ with Python 2.6 being the default. In order to share as much as possible between the available Python versions, these distributions install third party package modules -(``.pyc`` and ``.so`` files) into `/usr/share/pyshared` and symlink to -them from `/usr/lib/pythonX.Y/dist-packages`. The symlinks exist -because in a pre-PEP 3147 world (i.e < Python 3.2), the `.pyc` files +(``.pyc`` and ``.so`` files) into ``/usr/share/pyshared`` and symlink to +them from ``/usr/lib/pythonX.Y/dist-packages``. The symlinks exist +because in a pre-:pep:`3147` world (i.e < Python 3.2), the ``.pyc`` files resulting from byte compilation by the various installed Pythons will name collide with each other. For Python versions >= 3.2, all -pure-Python packages can be shared, because the `.pyc` files will no +pure-Python packages can be shared, because the ``.pyc`` files will no longer cause file system naming conflicts. Eliminating these symlinks makes for a simpler, more robust Python distribution. A similar situation arises with shared library extensions. Because -extension modules are typically named `foo.so` for a `foo` extension -module, these would also name collide if `foo` was provided for more +extension modules are typically named ``foo.so`` for a ``foo`` extension +module, these would also name collide if ``foo`` was provided for more than one Python version. In addition, because different configuration/compilation options for @@ -93,7 +93,7 @@ module files. PyPy [5]_ can also benefit from this PEP, allowing it to avoid name collisions in extension modules built for its API, but with a -different `.so` tag. +different ``.so`` tag. Proposal @@ -139,7 +139,7 @@ Note that ``$SOABI`` contains just the tag, while ``$EXT_SUFFIX`` includes the platform extension for shared library files, and is the exact suffix added to the extension module name. -For an arbitrary package `foo`, you might see these files when the +For an arbitrary package ``foo``, you might see these files when the distribution package was installed:: /usr/lib/python/foo.cpython-32m.so @@ -159,7 +159,7 @@ This shared library tag would be used globally for all distutils-based extension modules, regardless of where on the file system they are built. Extension modules built by means other than distutils would either have to calculate the tag manually, or fallback to the -non-tagged `.so` file name. +non-tagged ``.so`` file name. Proven approach @@ -170,7 +170,7 @@ and Ubuntu system where different extensions are used for debug builds of Python and extension modules. Debug builds on Windows also already use a different file extension for dynamic libraries, and in fact encoded (in a different way than proposed in this PEP) the Python -major and minor version in the `.dll` file name. +major and minor version in the ``.dll`` file name. Windows @@ -187,16 +187,16 @@ useful for Windows. PEP 384 ======= -PEP 384 defines a stable ABI for extension modules. In theory, -universal adoption of PEP 384 would eliminate the need for this PEP +:pep:`384` defines a stable ABI for extension modules. In theory, +universal adoption of :pep:`384` would eliminate the need for this PEP because all extension modules could be compatible with any Python version. In practice of course, it will be impossible to achieve universal adoption, and as described above, different build-time flags still affect the ABI. Thus even with a stable ABI, this PEP may still -be necessary. While a complete specification is reserved for PEP 384, +be necessary. While a complete specification is reserved for :pep:`384`, here is a discussion of the relevant issues. -PEP 384 describes a change to ``PyModule_Create()`` where ``3`` is +:pep:`384` describes a change to ``PyModule_Create()`` where ``3`` is passed as the API version if the extension was compiled with ``Py_LIMITED_API``. This should be formalized into an official macro called ``PYTHON_ABI_VERSION`` to mirror ``PYTHON_API_VERSION``. If @@ -205,9 +205,9 @@ would be bumped. To facilitate sharing, Python would be extended to search for extension modules with the ``PYTHON_ABI_VERSION`` number in its name. The prefix ``abi`` is reserved for Python's use. -Thus, an initial implementation of PEP 384, when Python is configured +Thus, an initial implementation of :pep:`384`, when Python is configured with the default set of flags, would search for the following file -names when extension module `foo` is imported (in this order):: +names when extension module ``foo`` is imported (in this order):: foo.cpython-XYm.so foo.abi3.so @@ -222,7 +222,7 @@ by adding a keyword argument to the ``Extension`` class, such as:: Extension('foo', ['foo.c'], abi=3) Martin v. Löwis describes his thoughts [7]_ about the applicability of this -PEP to PEP 384. In summary: +PEP to :pep:`384`. In summary: * ``--with-pydebug`` would not be supported by the stable ABI because this changes the layout of ``PyObject``, which is an exposed @@ -246,9 +246,9 @@ Independent directories or symlinks Debian and Ubuntu could simply add a version-specific directory to ``sys.path`` that would contain just the extension modules for that -version of Python. Or the symlink trick eliminated in PEP 3147 could +version of Python. Or the symlink trick eliminated in :pep:`3147` could be retained for just shared libraries. This approach is rejected -because it propagates the essential complexity that PEP 3147 tries to +because it propagates the essential complexity that :pep:`3147` tries to avoid, and adds potentially several additional directories to search for all modules, even when the number of extension modules is much fewer than the total number of Python packages. For example, builds @@ -262,7 +262,7 @@ Don't share packages with extension modules It has been suggested that Python packages with extension modules not be shared among all supported Python versions on a distribution. Even -with adoption of PEP 3149, extension modules will have to be compiled +with adoption of :pep:`3149`, extension modules will have to be compiled for every supported Python version, so perhaps sharing of such packages isn't useful anyway. Not sharing packages with extensions though is infeasible for several reasons. @@ -271,7 +271,7 @@ If a pure-Python package is shared in one version, should it suddenly be not-shared if the next release adds an extension module for speed? Also, even though all extension shared libraries will be compiled and distributed once for every supported Python, there's a big difference -between duplicating the `.so` files and duplicating all `.py` files. +between duplicating the ``.so`` files and duplicating all ``.py`` files. The extra size increases the download time for such packages, and more immediately, increases the space pressures on already constrained distribution CD-ROMs. @@ -289,10 +289,6 @@ are uploaded. References ========== -.. [1] PEP 3147 - -.. [2] PEP 384 - .. [3] Ubuntu: .. [4] Debian: diff --git a/pep-3150.txt b/peps/pep-3150.rst similarity index 93% rename from pep-3150.txt rename to peps/pep-3150.rst index dfbf449fb..0406c92ed 100644 --- a/pep-3150.txt +++ b/peps/pep-3150.rst @@ -8,7 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 09-Jul-2010 Python-Version: 3.4 -Post-History: 2010-07-14, 2011-04-21, 2011-06-13 +Post-History: 14-Jul-2010, 21-Apr-2011, 13-Jun-2011 Abstract @@ -229,13 +229,13 @@ simplify the initial parsing step). New PEP 8 Guidelines -------------------- -As discussed on python-ideas ([7]_, [9]_) new PEP 8 guidelines would also +As discussed on python-ideas ([7]_, [9]_) new :pep:`8` guidelines would also need to be developed to provide appropriate direction on when to use the ``given`` clause over ordinary variable assignments. Based on the similar guidelines already present for ``try`` statements, this PEP proposes the following additions for ``given`` statements to the -"Programming Conventions" section of PEP 8: +"Programming Conventions" section of :pep:`8`: - for code that could reasonably be factored out into a separate function, but is not currently reused anywhere, consider using a ``given`` clause. @@ -336,17 +336,17 @@ That way lies C++ and Perl :) Relation to PEP 403 ------------------- -PEP 403 (General Purpose Decorator Clause) attempts to achieve the main +:pep:`403` (General Purpose Decorator Clause) attempts to achieve the main goals of this PEP using a less radical language change inspired by the existing decorator syntax. Despite having the same author, the two PEPs are in direct competition with -each other. PEP 403 represents a minimalist approach that attempts to achieve +each other. :pep:`403` represents a minimalist approach that attempts to achieve useful functionality with a minimum of change from the status quo. This PEP instead aims for a more flexible standalone statement design, which requires a larger degree of change to the language. -Note that where PEP 403 is better suited to explaining the behaviour of +Note that where :pep:`403` is better suited to explaining the behaviour of generator expressions correctly, this PEP is better able to explain the behaviour of decorator clauses in general. Both PEPs support adequate explanations for the semantics of container comprehensions. @@ -376,7 +376,7 @@ appear to misbehave at class scope: only the outermost iterator is evaluated at class scope, while all predicates, nested iterators and value expressions are evaluated inside a nested scope. -Not that, unlike PEP 403, the current version of this PEP *cannot* +Not that, unlike :pep:`403`, the current version of this PEP *cannot* provide a precisely equivalent expansion for a generator expression. The closest it can get is to define an additional level of scoping:: @@ -448,7 +448,7 @@ though those variables are only going to be used once? When should an inline while loop be replaced with a generator that implements the same logic? Opinions differ, and that's OK. -However, explicit PEP 8 guidance will be needed for CPython and the standard +However, explicit :pep:`8` guidance will be needed for CPython and the standard library, and that is discussed in the proposal above. @@ -718,7 +718,7 @@ Rejected Alternatives expressions to indicate a direct reference to the implied closure, thus preventing it from being called automatically to create the local namespace). All such attempts have appeared unattractive and confusing compared to - the simpler decorator-inspired proposal in PEP 403. + the simpler decorator-inspired proposal in :pep:`403`. Reference Implementation ======================== @@ -730,7 +730,7 @@ semantics and code compilation, feel free to try ;) TO-DO ===== -* Mention PEP 359 and possible uses for locals() in the ``given`` clause +* Mention :pep:`359` and possible uses for locals() in the ``given`` clause * Figure out if this can be used internally to make the implementation of zero-argument super() calls less awful @@ -738,48 +738,37 @@ TO-DO References ========== -.. [1] Explicitation lines in Python: - https://mail.python.org/pipermail/python-ideas/2010-June/007476.html +.. [1] `Explicitation lines in Python + `__ -.. [2] 'where' statement in Python: - https://mail.python.org/pipermail/python-ideas/2010-July/007584.html +.. [2] `'where' statement in Python + `__ -.. [3] Where-statement (Proposal for function expressions): - https://mail.python.org/pipermail/python-ideas/2009-July/005132.html +.. [3] `Where-statement (Proposal for function expressions) + `__ -.. [4] Name conflict with NumPy for 'where' keyword choice: - https://mail.python.org/pipermail/python-ideas/2010-July/007596.html +.. [4] `Name conflict with NumPy for 'where' keyword choice + `__ -.. [5] The "Status quo wins a stalemate" design principle: - http://www.boredomandlaziness.org/2011/02/status-quo-wins-stalemate.html +.. [6] `Assignments in list/generator expressions + `__ -.. [6] Assignments in list/generator expressions: - https://mail.python.org/pipermail/python-ideas/2011-April/009863.html +.. [7] `Possible PEP 3150 style guidelines (#1) + `__ -.. [7] Possible PEP 3150 style guidelines (#1): - https://mail.python.org/pipermail/python-ideas/2011-April/009869.html +.. [8] `Discussion of PEP 403 (statement local function definition) + `__ -.. [8] Discussion of PEP 403 (statement local function definition): - https://mail.python.org/pipermail/python-ideas/2011-October/012276.html +.. [9] `Possible PEP 3150 style guidelines (#2) + `__ -.. [9] Possible PEP 3150 style guidelines (#2): - https://mail.python.org/pipermail/python-ideas/2011-October/012341.html +* `The "Status quo wins a stalemate" design principle + `__ -.. [10] Multi-line lambdas (again!) - https://mail.python.org/pipermail/python-ideas/2013-August/022526.html +* `Multi-line lambdas (again!) + `__ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3151.txt b/peps/pep-3151.rst similarity index 96% rename from pep-3151.txt rename to peps/pep-3151.rst index 644309341..e1ba710e2 100644 --- a/pep-3151.txt +++ b/peps/pep-3151.rst @@ -99,8 +99,8 @@ exception specifics has to examine the ``errno`` or ``winerror`` attribute anyway. .. note:: - `Appendix B`_ surveys the use of the various exception types across - the interpreter and the standard library. + `Appendix B `_ surveys the use of the + various exception types across the interpreter and the standard library. Lack of fine-grained exceptions @@ -180,9 +180,9 @@ unconnected socket (ditto), a socket timeout, a file type mismatch, an invalid argument, a transmission failure, insufficient permissions, a non-existent directory, a full filesystem, etc. -(moreover, the use of certain of these exceptions is irregular; `Appendix B`_ -exposes the case of the `select`_ module, which raises different exceptions -depending on the implementation) +(moreover, the use of certain of these exceptions is irregular; `Appendix B +`_ exposes the case of the `select`_ module, +which raises different exceptions depending on the implementation) *Careful* code is defined as code which, when catching any of the above exceptions, examines the ``errno`` attribute to determine the actual error @@ -264,8 +264,8 @@ The rationale for keeping ``OSError`` as the official name for generic OS-related exceptions is that it, precisely, is more generic than ``IOError``. ``EnvironmentError`` is more tedious to type and also much lesser-known. -The survey in `Appendix B`_ shows that IOError is the dominant -error today in the standard library. As for third-party Python code, +The survey in `Appendix B `_ shows that IOError is the +dominant error today in the standard library. As for third-party Python code, Google Code Search shows IOError being ten times more popular than EnvironmentError in user code, and three times more popular than OSError [3]_. However, with no intention to deprecate IOError in the middle @@ -323,10 +323,10 @@ Step 2: define additional subclasses The second step of the resolution is to extend the hierarchy by defining subclasses which will be raised, rather than their parent, for specific errno values. Which errno values is subject to discussion, but a survey -of existing exception matching practices (see `Appendix A`_) helps us -propose a reasonable subset of all values. Trying to map all errno -mnemonics, indeed, seems foolish, pointless, and would pollute the root -namespace. +of existing exception matching practices (see `Appendix A +`_) helps us propose a reasonable subset of all values. +Trying to map all errno mnemonics, indeed, seems foolish, pointless, +and would pollute the root namespace. Furthermore, in a couple of cases, different errno values could raise the same exception subclass. For example, EAGAIN, EALREADY, EWOULDBLOCK @@ -499,7 +499,8 @@ of finer-grained exception classes and the coalescing of OSError and IOError. The removal of WindowsError alone has been discussed and rejected -as part of another PEP [2]_, but there seemed to be a consensus that the +as part of :pep:`another PEP <348#removing-windowserror>`, +but there seemed to be a consensus that the distinction with OSError wasn't meaningful. This supports at least its aliasing with OSError. @@ -585,7 +586,7 @@ While they would deserve less cryptic names, this can be handled separately from the exception hierarchy reorganization effort. -.. _Appendix A: +.. _PEP 3151 Appendix A: Appendix A: Survey of common errnos =================================== @@ -668,7 +669,7 @@ Common errnos with select.error * ``EINTR``: interrupted function call -.. _Appendix B: +.. _PEP 3151 Appendix B: Appendix B: Survey of raised OS and IO errors ============================================= @@ -694,7 +695,7 @@ Handling of PYTHONSTARTUP raises IOError (but the error gets discarded):: IOError: [Errno 2] No such file or directory: 'foox' ``PyObject_Print()`` raises IOError when ferror() signals an error on the -`FILE *` parameter (which, in the source tree, is always either stdout or +``FILE *`` parameter (which, in the source tree, is always either stdout or stderr). Unicode encoding and decoding using the ``mbcs`` encoding can raise @@ -944,9 +945,6 @@ References .. [1] "IO module precisions and exception hierarchy": https://mail.python.org/pipermail/python-dev/2009-September/092130.html -.. [2] Discussion of "Removing WindowsError" in PEP 348: - http://www.python.org/dev/peps/pep-0348/#removing-windowserror - .. [3] Google Code Search of ``IOError`` in Python code: `around 40000 results `_; ``OSError``: `around 15200 results @@ -960,14 +958,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-3152.txt b/peps/pep-3152.rst similarity index 99% rename from pep-3152.txt rename to peps/pep-3152.rst index 1fe27c316..46bcab5e3 100644 --- a/pep-3152.txt +++ b/peps/pep-3152.rst @@ -24,7 +24,7 @@ symptoms. This proposal builds on the 'yield from' mechanism described in PEP 380, and describes some of the semantics of cofunctions in terms of it. However, it would be possible to define and implement cofunctions -independently of PEP 380 if so desired. +independently of :pep:`380` if so desired. Rejection --------- diff --git a/pep-3153.txt b/peps/pep-3153.rst similarity index 99% rename from pep-3153.txt rename to peps/pep-3153.rst index bc7ba4b05..8256b0664 100644 --- a/pep-3153.txt +++ b/peps/pep-3153.rst @@ -7,7 +7,7 @@ Status: Superseded Type: Standards Track Content-Type: text/x-rst Created: 29-May-2011 -Post-History: TBD +Post-History: Superseded-By: 3156 Abstract diff --git a/pep-3154.txt b/peps/pep-3154.rst similarity index 97% rename from pep-3154.txt rename to peps/pep-3154.rst index 9e60e055d..6454c2432 100644 --- a/pep-3154.txt +++ b/peps/pep-3154.rst @@ -8,8 +8,7 @@ Type: Standards Track Content-Type: text/x-rst Created: 11-Aug-2011 Python-Version: 3.4 -Post-History: - https://mail.python.org/pipermail/python-dev/2011-August/112821.html +Post-History: `12-Aug-2011 `__ Resolution: https://mail.python.org/pipermail/python-dev/2013-November/130439.html @@ -219,11 +218,11 @@ Acknowledgments In alphabetic order: -* Alexandre Vassalotti, for starting the second PEP 3154 implementation [6]_ +* Alexandre Vassalotti, for starting the second :pep:`3154` implementation [6]_ * Serhiy Storchaka, for discussing the framing proposal [6]_ -* Stefan Mihaila, for starting the first PEP 3154 implementation as a +* Stefan Mihaila, for starting the first :pep:`3154` implementation as a Google Summer of Code project mentored by Alexandre Vassalotti [7]_. diff --git a/pep-3155.txt b/peps/pep-3155.rst similarity index 100% rename from pep-3155.txt rename to peps/pep-3155.rst diff --git a/pep-3156.txt b/peps/pep-3156.rst similarity index 98% rename from pep-3156.txt rename to peps/pep-3156.rst index 19b2b202e..a9343fa36 100644 --- a/pep-3156.txt +++ b/peps/pep-3156.rst @@ -4,11 +4,12 @@ Version: $Revision$ Last-Modified: $Date$ Author: Guido van Rossum BDFL-Delegate: Antoine Pitrou -Discussions-To: +Discussions-To: python-tulip@googlegroups.com Status: Final Type: Standards Track Content-Type: text/x-rst Created: 12-Dec-2012 +Python-Version: 3.3 Post-History: 21-Dec-2012 Replaces: 3153 Resolution: https://mail.python.org/pipermail/python-dev/2013-November/130419.html @@ -18,9 +19,9 @@ Abstract This is a proposal for asynchronous I/O in Python 3, starting at Python 3.3. Consider this the concrete proposal that is missing from -PEP 3153. The proposal includes a pluggable event loop, transport and +:pep:`3153`. The proposal includes a pluggable event loop, transport and protocol abstractions similar to those in Twisted, and a higher-level -scheduler based on ``yield from`` (PEP 380). The proposed package +scheduler based on ``yield from`` (:pep:`380`). The proposed package name is ``asyncio``. @@ -120,7 +121,7 @@ there is a system event loop that cannot be started or stopped; see The event loop API does not depend on ``await/yield from``. Rather, it uses a combination of callbacks, additional interfaces (transports and protocols), and Futures. The latter are similar to those defined in -PEP 3148, but have a different implementation and are not tied to +:pep:`3148`, but have a different implementation and are not tied to threads. In particular, the ``result()`` method raises an exception instead of blocking when a result is not yet ready; the user is expected to use callbacks (or ``await/yield from``) to wait for the result. @@ -135,7 +136,7 @@ coroutine or a Future into a Future.) For users (like myself) who don't like using callbacks, a scheduler is provided for writing asynchronous I/O code as coroutines using the PEP -380 ``yield from`` or PEP 492 ``await`` expressions. +380 ``yield from`` or :pep:`492` ``await`` expressions. The scheduler is not pluggable; pluggability occurs at the event loop level, and the standard scheduler implementation should work with any conforming event loop @@ -159,7 +160,7 @@ A less ambitious framework may just call the implementing its own event loop. The event loop API provides limited interoperability with threads: -there is an API to submit a function to an executor (see PEP 3148) +there is an API to submit a function to an executor (see :pep:`3148`) which returns a Future that is compatible with the event loop, and there is a method to schedule a callback with an event loop from another thread in a thread-safe manner. @@ -499,7 +500,7 @@ shared state isn't changed by another callback. clock. This may be ``time.time()`` or ``time.monotonic()`` or some other system-specific clock, but it must return a float expressing the time in units of approximately one second since some epoch. - (No clock is perfect -- see PEP 418.) + (No clock is perfect -- see :pep:`418`.) Note: A previous version of this PEP defined a method named ``call_repeatedly()``, which promised to call a callback at regular @@ -509,7 +510,7 @@ easily be emulated using a callback that reschedules itself using ``call_later()``; it is also easy to write coroutine containing a loop and a ``sleep()`` call (a toplevel function in the module, see below). On the other hand, due to the complexities of accurate timekeeping -there are many traps and pitfalls here for the unaware (see PEP 418), +there are many traps and pitfalls here for the unaware (see :pep:`418`), and different use cases require different behavior in edge cases. It is impossible to offer an API for this purpose that is bullet-proof in all cases, so it is deemed better to let application designers decide @@ -531,7 +532,7 @@ Thread interaction below. - ``run_in_executor(executor, callback, *args)``. Arrange to call - ``callback(*args)`` in an executor (see PEP 3148). Returns an + ``callback(*args)`` in an executor (see :pep:`3148`). Returns an ``asyncio.Future`` instance whose result on success is the return value of that call. This is equivalent to ``wrap_future(executor.submit(callback, *args))``. If ``executor`` @@ -542,7 +543,7 @@ Thread interaction 5 threads in this case.) - ``set_default_executor(executor)``. Set the default executor used - by ``run_in_executor()``. The argument must be a PEP 3148 + by ``run_in_executor()``. The argument must be a :pep:`3148` ``Executor`` instance or ``None``, in order to reset the default executor. @@ -1069,11 +1070,11 @@ Futures ------- The ``asyncio.Future`` class here is intentionally similar to the -``concurrent.futures.Future`` class specified by PEP 3148, but there +``concurrent.futures.Future`` class specified by :pep:`3148`, but there are slight differences. Whenever this PEP talks about Futures or futures this should be understood to refer to ``asyncio.Future`` unless ``concurrent.futures.Future`` is explicitly mentioned. The supported -public API is as follows, indicating the differences with PEP 3148: +public API is as follows, indicating the differences with :pep:`3148`: - ``cancel()``. If the Future is already done (or cancelled), do nothing and return ``False``. Otherwise, this attempts to cancel @@ -1092,23 +1093,23 @@ public API is as follows, indicating the differences with PEP 3148: - ``result()``. Returns the result set with ``set_result()``, or raises the exception set with ``set_exception()``. Raises - ``CancelledError`` if cancelled. Difference with PEP 3148: This has + ``CancelledError`` if cancelled. Difference with :pep:`3148`: This has no timeout argument and does *not* wait; if the future is not yet done, it raises an exception. - ``exception()``. Returns the exception if set with ``set_exception()``, or ``None`` if a result was set with ``set_result()``. Raises ``CancelledError`` if cancelled. - Difference with PEP 3148: This has no timeout argument and does + Difference with :pep:`3148`: This has no timeout argument and does *not* wait; if the future is not yet done, it raises an exception. - ``add_done_callback(fn)``. Add a callback to be run when the Future becomes done (or is cancelled). If the Future is already done (or cancelled), schedules the callback to using ``call_soon()``. - Difference with PEP 3148: The callback is never called immediately, + Difference with :pep:`3148`: The callback is never called immediately, and always in the context of the caller -- typically this is a thread. You can think of this as calling the callback through - ``call_soon()``. Note that in order to match PEP 3148, the callback + ``call_soon()``. Note that in order to match :pep:`3148`, the callback (unlike all other callbacks defined in this PEP, and ignoring the convention from the section "Callback Style" below) is always called with a single argument, the Future object. (The motivation for @@ -1116,18 +1117,18 @@ public API is as follows, indicating the differences with PEP 3148: applies here too.) - ``remove_done_callback(fn)``. Remove the argument from the list of - callbacks. This method is not defined by PEP 3148. The argument + callbacks. This method is not defined by :pep:`3148`. The argument must be equal (using ``==``) to the argument passed to ``add_done_callback()``. Returns the number of times the callback was removed. - ``set_result(result)``. The Future must not be done (nor cancelled) already. This makes the Future done and schedules the callbacks. - Difference with PEP 3148: This is a public API. + Difference with :pep:`3148`: This is a public API. - ``set_exception(exception)``. The Future must not be done (nor cancelled) already. This makes the Future done and schedules the - callbacks. Difference with PEP 3148: This is a public API. + callbacks. Difference with :pep:`3148`: This is a public API. The internal method ``set_running_or_notify_cancel()`` is not supported; there is no way to set the running state. Likewise, @@ -1186,7 +1187,7 @@ There are some public functions related to Futures: it is returned unchanged; if it is a coroutine object, it wraps it in a Task (remember that ``Task`` is a subclass of ``Future``). -- ``asyncio.wrap_future(future)``. This takes a PEP 3148 Future +- ``asyncio.wrap_future(future)``. This takes a :pep:`3148` Future (i.e., an instance of ``concurrent.futures.Future``) and returns a Future compatible with the event loop (i.e., a ``asyncio.Future`` instance). @@ -1583,7 +1584,7 @@ A coroutine is a generator that follows certain conventions. For documentation purposes, all coroutines should be decorated with ``@asyncio.coroutine``, but this cannot be strictly enforced. -Coroutines use the ``yield from`` syntax introduced in PEP 380, +Coroutines use the ``yield from`` syntax introduced in :pep:`380`, instead of the original ``yield`` syntax. The word "coroutine", like the word "generator", is used for two @@ -1647,7 +1648,7 @@ package are provided: ``timeout``, if not ``None``, specifies a timeout for the overall operation; ``return_when``, specifies when to stop. The constants ``FIRST_COMPLETED``, ``FIRST_EXCEPTION``, ``ALL_COMPLETED`` are - defined with the same values and the same meanings as in PEP 3148: + defined with the same values and the same meanings as in :pep:`3148`: - ``ALL_COMPLETED`` (default): Wait until all Futures are done (or until the timeout occurs). @@ -1657,7 +1658,7 @@ package are provided: - ``FIRST_EXCEPTION``: Wait until at least one Future is done but not cancelled with an exception set. (The exclusion of cancelled - Futures from the condition is surprising, but PEP 3148 does it + Futures from the condition is surprising, but :pep:`3148` does it this way.) - ``asyncio.as_completed(fs, timeout=None)``. Returns an iterator whose @@ -2038,19 +2039,19 @@ status allows revising these decisions for Python 3.5.) References ========== -- PEP 492 describes the semantics of ``async/await``. +- :pep:`492` describes the semantics of ``async/await``. -- PEP 380 describes the semantics of ``yield from``. +- :pep:`380` describes the semantics of ``yield from``. - Greg Ewing's ``yield from`` tutorials: http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/yield_from.html -- PEP 3148 describes ``concurrent.futures.Future``. +- :pep:`3148` describes ``concurrent.futures.Future``. -- PEP 3153, while rejected, has a good write-up explaining the need +- :pep:`3153`, while rejected, has a good write-up explaining the need to separate transports and protocols. -- PEP 418 discusses the issues of timekeeping. +- :pep:`418` discusses the issues of timekeeping. - Tulip repo: http://code.google.com/p/tulip/ @@ -2068,7 +2069,7 @@ References Acknowledgments =============== -Apart from PEP 3153, influences include PEP 380 and Greg Ewing's +Apart from :pep:`3153`, influences include :pep:`380` and Greg Ewing's tutorial for ``yield from``, Twisted, Tornado, ZeroMQ, pyftpdlib, and wattle (Steve Dower's counter-proposal). My previous work on asynchronous support in the NDB library for Google App Engine provided diff --git a/pep-3333.txt b/peps/pep-3333.rst similarity index 97% rename from pep-3333.txt rename to peps/pep-3333.rst index b7b96bbce..4ec70d84f 100644 --- a/pep-3333.txt +++ b/peps/pep-3333.rst @@ -1,9 +1,7 @@ PEP: 3333 Title: Python Web Server Gateway Interface v1.0.1 -Version: $Revision$ -Last-Modified: $Date$ -Author: P.J. Eby -Discussions-To: Python Web-SIG +Author: Phillip J. Eby +Discussions-To: web-sig@python.org Status: Final Type: Informational Content-Type: text/x-rst @@ -12,10 +10,10 @@ Post-History: 26-Sep-2010, 04-Oct-2010 Replaces: 333 -Preface for Readers of PEP \333 -=============================== +Preface for Readers of PEP 333 +============================== -This is an updated version of PEP 333, modified slightly to improve +This is an updated version of :pep:`333`, modified slightly to improve usability under Python 3, and to incorporate several long-standing de facto amendments to the WSGI protocol. (Its code samples have also been ported to Python 3.) @@ -41,8 +39,8 @@ servers and Python web applications or frameworks, to promote web application portability across a variety of web servers. -Original Rationale and Goals (from PEP \333) -============================================ +Original Rationale and Goals (from PEP 333) +=========================================== Python currently boasts a wide variety of web application frameworks, such as Zope, Quixote, Webware, SkunkWeb, PSO, and Twisted Web -- to @@ -555,7 +553,7 @@ current request, whether the request was completed normally, or terminated early due to an application error during iteration or an early disconnect of the browser. (The ``close()`` method requirement is to support resource release by the application. This protocol is intended -to complement PEP 342's generator support, and other common iterables +to complement :pep:`342`'s generator support, and other common iterables with ``close()`` methods.) Applications returning a generator or other custom iterator **should not** @@ -574,7 +572,7 @@ attributes of the iterable returned by the application, unless it is an instance of a type specific to that server or gateway, such as a "file wrapper" returned by ``wsgi.file_wrapper`` (see `Optional Platform-Specific File Handling`_). In the general case, only -attributes specified here, or accessed via e.g. the PEP 234 iteration +attributes specified here, or accessed via e.g. the :pep:`234` iteration APIs are acceptable. @@ -676,9 +674,9 @@ Variable Value ``wsgi.input`` An input stream (file-like object) from which the HTTP request body bytes can be read. (The server or gateway may perform reads on-demand as - requested by the application, or it may pre- - read the client's request body and buffer it - in-memory or on disk, or use any other + requested by the application, or it may + pre-read the client's request body and buffer + it in-memory or on disk, or use any other technique for providing such an input stream, according to its preference.) @@ -809,7 +807,7 @@ The ``status`` argument is an HTTP "status" string like ``"200 OK"`` or ``"404 Not Found"``. That is, it is a string consisting of a Status-Code and a Reason-Phrase, in that order and separated by a single space, with no surrounding whitespace or other characters. -(See RFC 2616, Section 6.1.1 for more information.) The string +(See :rfc:`2616`, Section 6.1.1 for more information.) The string **must not** contain control characters, and must not be terminated with a carriage return, linefeed, or combination thereof. @@ -817,7 +815,7 @@ The ``response_headers`` argument is a list of ``(header_name, header_value)`` tuples. It must be a Python list; i.e. ``type(response_headers) is ListType``, and the server **may** change its contents in any way it desires. Each ``header_name`` must be a -valid HTTP header field-name (as defined by RFC 2616, Section 4.2), +valid HTTP header field-name (as defined by :rfc:`2616`, Section 4.2), without a trailing colon or other punctuation. Each ``header_value`` **must not** include *any* control characters, @@ -942,12 +940,13 @@ whose ``len()`` is 1, then the server can automatically determine ``Content-Length`` by taking the length of the first bytestring yielded by the iterable. -And, if the server and client both support HTTP/1.1 "chunked -encoding" [3]_, then the server **may** use chunked encoding to send +And, if the server and client both support HTTP/1.1 +:rfc:`"chunked encoding"<2616#section-3.6.1>`, +then the server **may** use chunked encoding to send a chunk for each ``write()`` call or bytestring yielded by the iterable, thus generating a ``Content-Length`` header for each chunk. This allows the server to keep the client connection alive, if it wishes -to do so. Note that the server **must** comply fully with RFC 2616 +to do so. Note that the server **must** comply fully with :rfc:`2616` when doing this, or else fall back to one of the other strategies for dealing with the absence of ``Content-Length``. @@ -1094,8 +1093,8 @@ all strings passed to or from the server must be of type ``str`` or object where a string object is required, is undefined. Note also that strings passed to ``start_response()`` as a status or -as response headers **must** follow RFC 2616 with respect to encoding. -That is, they must either be ISO-8859-1 characters, or use RFC 2047 +as response headers **must** follow :rfc:`2616` with respect to encoding. +That is, they must either be ISO-8859-1 characters, or use :rfc:`2047` MIME encoding. On Python platforms where the ``str`` or ``StringType`` type is in @@ -1202,8 +1201,8 @@ may be done in any of several ways: Note that these behavior restrictions do not apply for HTTP 1.0 requests, or for requests that are not directed to an application -object. For more information on HTTP 1.1 Expect/Continue, see RFC -2616, sections 8.2.3 and 10.1.1. +object. For more information on HTTP 1.1 Expect/Continue, see +:rfc:`2616`, sections 8.2.3 and 10.1.1. Other HTTP Features @@ -1216,13 +1215,14 @@ response. It is always possible for the application developer to add middleware components to supply additional features, so server/gateway developers should be conservative in their implementation. In a sense, a server should consider itself to be like an HTTP "gateway server", -with the application being an HTTP "origin server". (See RFC 2616, +with the application being an HTTP "origin server". (See :rfc:`2616`, section 1.3, for the definition of these terms.) However, because WSGI servers and applications do not communicate via -HTTP, what RFC 2616 calls "hop-by-hop" headers do not apply to WSGI +HTTP, what :rfc:`2616` calls "hop-by-hop" headers do not apply to WSGI internal communications. WSGI applications **must not** generate any -"hop-by-hop" headers [4]_, attempt to use HTTP features that would +:rfc:`"hop-by-hop" headers <2616#section-13.5.1>`, +attempt to use HTTP features that would require them to generate such headers, or rely on the content of any incoming "hop-by-hop" headers in the ``environ`` dictionary. WSGI servers **must** handle any supported inbound "hop-by-hop" headers @@ -1418,7 +1418,7 @@ simply restrict themselves to using only a standard "for" loop to iterate over any iterable returned by an application. This is the only way to ensure source-level compatibility with both the pre-2.2 iterator protocol (discussed further below) and "today's" iterator -protocol (see PEP 234). +protocol (see :pep:`234`). (Note that this technique necessarily applies only to servers, gateways, or middleware that are written in Python. Discussion of @@ -1455,8 +1455,8 @@ use only language features available in the target version, use Optional Platform-Specific File Handling ---------------------------------------- -Some operating environments provide special high-performance file- -transmission facilities, such as the Unix ``sendfile()`` call. +Some operating environments provide special high-performance +file-transmission facilities, such as the Unix ``sendfile()`` call. Servers and gateways **may** expose this functionality via an optional ``wsgi.file_wrapper`` key in the ``environ``. An application **may** use this "file wrapper" to convert a file or file-like object @@ -1598,7 +1598,7 @@ Questions and Answers iterable is garbage collected. The ``close()`` idiom allows an application to release critical resources at the end of a request, and it's forward-compatible with the support for try/finally in - generators that's proposed by PEP 325. + generators that's proposed by :pep:`325`. 4. Why is this interface so low-level? I want feature X! (e.g. cookies, sessions, persistence, ...) @@ -1763,13 +1763,7 @@ References (https://wiki.python.org/moin/WebProgramming) .. [2] The Common Gateway Interface Specification, v 1.1, 3rd Draft - (https://tools.ietf.org/html/draft-coar-cgi-v11-03) - -.. [3] "Chunked Transfer Coding" -- HTTP/1.1, section 3.6.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1) - -.. [4] "End-to-end and Hop-by-hop Headers" -- HTTP/1.1, Section 13.5.1 - (http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1) + (https://datatracker.ietf.org/doc/html/draft-coar-cgi-v11-03) .. [5] mod_ssl Reference, "Environment Variables" (http://www.modssl.org/docs/2.8/ssl_reference.html#ToC25) @@ -1777,20 +1771,10 @@ References .. [6] Procedural issues regarding modifications to PEP \333 (https://mail.python.org/pipermail/python-dev/2010-September/104114.html) -.. [7] SVN revision history for PEP \3333, showing differences from PEP 333 +.. [7] SVN revision history for PEP 3333, showing differences from PEP 333 (http://svn.python.org/view/peps/trunk/pep-3333.txt?r1=84854&r2=HEAD) Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - End: diff --git a/pep-8000.rst b/peps/pep-8000.rst similarity index 84% rename from pep-8000.rst rename to peps/pep-8000.rst index b56c3cade..3361f2468 100644 --- a/pep-8000.rst +++ b/peps/pep-8000.rst @@ -1,8 +1,9 @@ PEP: 8000 Title: Python Language Governance Proposal Overview Author: Barry Warsaw -Status: Active +Status: Final Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 24-Aug-2018 @@ -13,19 +14,19 @@ Abstract This PEP provides an overview of the selection process for a new model of Python language governance in the wake of `Guido's retirement `_. -Once the governance model is selected, it will be codified in PEP 13. +Once the governance model is selected, it will be codified in :pep:`13`. Here is a list of PEPs related to the governance model selection process. PEPs in the lower 8000s describe the general process for selecting a governance model. -* PEP 8001 - Python Governance Voting Process +* :pep:`8001` - Python Governance Voting Process This PEP describes how the vote for the new governance model will be conducted. It outlines the voting method, timeline, criteria for participation, and explicit list of eligible voters. -* PEP 8002 - Open Source Governance Survey +* :pep:`8002` - Open Source Governance Survey Surveys will be conducted of governance models for similar open source and free software projects, and summaries of these models will be outlined in @@ -41,7 +42,7 @@ differences in details (such as the size of a governing council) will be covered in the same PEP, rather than in potentially vote-splitting individual PEPs. -* PEP 8010 - The Technical Leader Governance Model +* :pep:`8010` - The Technical Leader Governance Model This PEP proposes a continuation of the singular technical project leader model. Also within scope is whether an advisory council aids @@ -49,13 +50,13 @@ PEPs. BDFL, nor members of such an advisory council. For that, see PEP 13. -* PEP 8011 - Python Governance Model Lead by Trio of Pythonistas +* :pep:`8011` - Python Governance Model Lead by Trio of Pythonistas This PEP describes a new model of Python governance lead by a Trio of Pythonistas (TOP). It describes the role and responsibilities of the Trio. - This PEP does *not* name members of the Trio. For that, see PEP 13. + This PEP does *not* name members of the Trio. For that, see :pep:`13`. -* PEP 8012 - The Community Governance Model +* :pep:`8012` - The Community Governance Model This is a placeholder PEP for a new model of Python governance based on consensus and voting, without the role of a centralized singular leader or a @@ -63,16 +64,16 @@ PEPs. decisions affecting the Python language. It also describes the criteria for voting eligibility. -* PEP 8013 - The External Governance Model +* :pep:`8013` - The External Governance Model This PEP describes a new model of Python governance based on an external council who are responsible for ensuring good process. Elected by the core development team, this council may reject proposals that are not sufficiently detailed, do not consider all affected users, or are not appropriate for the upcoming release. This PEP does *not* name members of - such a council. For that, see PEP 13. + such a council. For that, see :pep:`13`. -* PEP 8014 - The Commons Governance Model +* :pep:`8014` - The Commons Governance Model This PEP describes a new model of Python governance based on a council of elders who are responsible for ensuring a PEP is supported by a sufficient @@ -81,7 +82,7 @@ PEPs. rights and what a majority vote consists of. In stead this is determined by the council of elders on a case-by-case basis. -* PEP 8015 - Organization of the Python community +* :pep:`8015` - Organization of the Python community This PEP formalizes the current organization of the Python community and proposes 3 main changes: formalize the existing concept of @@ -89,7 +90,7 @@ PEPs. (Guido van Rossum) with a new "Python board" of 3 members which has limited roles, mostly decide how a PEP is approved (or rejected). -* PEP 8016 - The Steering Council Model +* :pep:`8016` - The Steering Council Model This PEP proposes a model of Python governance based around a steering council. The council has broad authority, which they seek @@ -108,14 +109,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8001.rst b/peps/pep-8001.rst similarity index 87% rename from pep-8001.rst rename to peps/pep-8001.rst index d8ea4d0aa..d4020e2dc 100644 --- a/pep-8001.rst +++ b/peps/pep-8001.rst @@ -5,7 +5,7 @@ Author: Brett Cannon , Donald Stufft , Eric Snow , Gregory P. Smith , - Ɓukasz Langa + Ɓukasz Langa , Mariatta , Nathaniel J. Smith , Pablo Galindo Salgado , @@ -13,8 +13,9 @@ Author: Brett Cannon , Tal Einat , Tim Peters , Zachary Ware -Status: Accepted +Status: Final Type: Process +Topic: Governance Content-Type: text/x-rst Created: 24-Aug-2018 @@ -26,7 +27,7 @@ This PEP outlines the process for how the new model of Python governance is selected, in the wake of `Guido's retirement `_. Once the model is chosen by the procedures outlined here, it will be codified -in PEP 13. +in :pep:`13`. Motivation and Rationale @@ -57,7 +58,7 @@ What are we voting for? ----------------------- We are voting to choose which governance PEP should be implemented by -the Python project. The list of candidate PEPs is listed in PEP 8000 +the Python project. The list of candidate PEPs is listed in :pep:`8000` and consists of all PEPs numbered in the 801X range. To ensure the vote is legitimate, the aforementioned PEPs must not be @@ -92,7 +93,7 @@ The vote will happen using a "private" poll on the will receive an email with a link allowing them to rank the PEPs in their order of preference. -The election will be supervised by Ee W. Durbin III, The PSF Director of Infrastructure. +The election will be supervised by Ee Durbin, The PSF Director of Infrastructure. The results of the election, including anonymized ballots, will be made public on December 17th, after the election has closed. @@ -106,7 +107,7 @@ Description of the poll:: This is the vote to choose how the CPython project will govern itself, now that Guido has announced his retirement as BDFL. For full details, see PEP + href="https://peps.python.org/pep-8001/">PEP 8001. Many discussions have occurred under the "governance" tag on discuss.python.org. @@ -156,13 +157,13 @@ Description of the poll:: Candidates (note: linebreaks are significant here):: - PEP 8010: The Technical Leader Governance Model (Warsaw) (changelog) - PEP 8011: Python Governance Model Lead by Trio of Pythonistas (Mariatta, Warsaw) (changelog) - PEP 8012: The Community Governance Model (Langa) (changelog) - PEP 8013: The External Council Governance Model (Dower) (changelog) - PEP 8014: The Commons Governance Model (Jansen) (changelog) - PEP 8015: Organization of the Python community (Stinner) (changelog) - PEP 8016: The Steering Council Model (Smith, Stufft) (changelog) + PEP 8010: The Technical Leader Governance Model (Warsaw) (changelog) + PEP 8011: Python Governance Model Lead by Trio of Pythonistas (Mariatta, Warsaw) (changelog) + PEP 8012: The Community Governance Model (Langa) (changelog) + PEP 8013: The External Council Governance Model (Dower) (changelog) + PEP 8014: The Commons Governance Model (Jansen) (changelog) + PEP 8015: Organization of the Python community (Stinner) (changelog) + PEP 8016: The Steering Council Model (Smith, Stufft) (changelog) Further discussion Options:: @@ -329,14 +330,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8002.rst b/peps/pep-8002.rst similarity index 99% rename from pep-8002.rst rename to peps/pep-8002.rst index 90e4c7020..504ad7685 100644 --- a/pep-8002.rst +++ b/peps/pep-8002.rst @@ -1,10 +1,11 @@ PEP: 8002 Title: Open Source Governance Survey Author: Barry Warsaw , Ɓukasz Langa , - Antoine Pitrou , Doug Hellmann , - Carol Willing -Status: Active + Antoine Pitrou , Doug Hellmann , + Carol Willing +Status: Final Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 24-Aug-2018 @@ -532,7 +533,7 @@ Controversial decision process The Technical Board occasionally approves Django Enhancement Proposals (DEPs) but those are rare. The DEP process is roughly modeled after PEPs and documented in `DEP 1 -`_. +`_. DEPs are mostly used to design major new features, but also for information on general guidelines and process. @@ -594,7 +595,7 @@ TypeScript The governance structure is not externally documented besides the `CONTRIBUTING.md -`_ +`_ document in the main TypeScript repository. Key people and their functions @@ -1000,14 +1001,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8010.rst b/peps/pep-8010.rst similarity index 95% rename from pep-8010.rst rename to peps/pep-8010.rst index 92eae7b7e..c60ade457 100644 --- a/pep-8010.rst +++ b/peps/pep-8010.rst @@ -3,6 +3,7 @@ Title: The Technical Leader Governance Model Author: Barry Warsaw Status: Rejected Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 24-Aug-2018 @@ -35,18 +36,18 @@ This PEP describes: * Any changes to the PEP process to fit the new governance model; This PEP does *not* name a new BDFL. Should this model be adopted, it -will be codified in PEP 13 along with the names of all officeholders +will be codified in :pep:`13` along with the names of all officeholders described in this PEP. PEP Rejection ============= -PEP 8010 was rejected `by a core developer vote +:pep:`8010` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Open discussion points @@ -58,7 +59,7 @@ lengths of service, and voting procedures. These will be codified by the time the PEP is ready to be voted on. The voting procedures and events described in this PEP will default to -the voting method specified in PEP 8001, although as that PEP is still +the voting method specified in :pep:`8001`, although as that PEP is still in discussion at the time of this writing, this is subject to change. It is allowed, and perhaps even expected, that as experience is gained @@ -205,7 +206,7 @@ to the committers discussion forum. Maybe we'll even have debates! This phase of the election runs for two weeks. Core developers then have three weeks to vote, using the process -described in PEP 8001. +described in :pep:`8001`. The Council of Pythonistas (CoP) @@ -233,7 +234,7 @@ popular vote getter shall serve for 2 years, and CoP member with the least number of votes shall serve initially for 1 year. All ties in voting will be broken with a procedure to be determined in -PEP 8001. +:pep:`8001`. The nomination and voting process is similar as with the GUIDO. There is a three-week nomination period, where self-nominations are allowed @@ -250,7 +251,7 @@ No confidence votes As mentioned above, the CoP may, by unanimous decision, initiate a vote of no-confidence in the GUIDO. This process should not be undertaken lightly, but once begun, it triggers up to two votes. In -both cases, voting is done by the same procedure as in PEP 8001, and +both cases, voting is done by the same procedure as in :pep:`8001`, and all core developers may participate in no confidence votes. The first vote is whether to recall the current GUIDO or not. Should @@ -301,7 +302,7 @@ Version 2 - Renamed to "The Technical Leader Governance Model" - "singular leader" -> "singular technical leader" - - The adoption of PEP 8001 voting procedures is tentative until that + - The adoption of :pep:`8001` voting procedures is tentative until that PEP is approved - Describe what happens if the GUIDO steps down - Recall votes require a super majority of core devs to succeed @@ -314,14 +315,3 @@ This document has been placed in the public domain. .. _`Design by committee`: https://en.wikipedia.org/wiki/Design_by_committee - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8011.rst b/peps/pep-8011.rst similarity index 96% rename from pep-8011.rst rename to peps/pep-8011.rst index d6fca8bda..9ab91a4c5 100644 --- a/pep-8011.rst +++ b/peps/pep-8011.rst @@ -3,6 +3,7 @@ Title: Python Governance Model Lead by Trio of Pythonistas Author: Mariatta , Barry Warsaw Status: Rejected Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 24-Aug-2018 @@ -13,14 +14,14 @@ Abstract This PEP proposes a governance model for the Core Python development community, led by a trio of equally authoritative leaders. The Trio of Pythonistas (ToP, or simply Trio) is tasked with making final decisions for the language. -It differs from PEP 8010 by specifically not proposing a central singular leader, +It differs from :pep:`8010` by specifically not proposing a central singular leader, but instead a group of three people as the leaders. This PEP also proposes a formation of specialized workgroups to assist the leadership trio in making decisions. This PEP does *not* name the members of the Trio. Should this model be adopted, -it will be codified in PEP 13 along with the names of all officeholders +it will be codified in :pep:`13` along with the names of all officeholders described in this PEP. This PEP describes: @@ -35,11 +36,11 @@ This PEP describes: PEP Rejection ============= -PEP 8011 was rejected `by a core developer vote +:pep:`8011` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Open discussion points ====================== @@ -52,7 +53,7 @@ These will be codified by the time the PEP is ready to be voted on. It is allowed, and perhaps even expected, that as experience is gained with this model, these parameters may be tweaked in order to provide for a smoother governing process. The process for tweaking these parameters will generally -be the same voting process as described in PEP 8001. +be the same voting process as described in :pep:`8001`. Roles and responsibilities of the leadership trio @@ -84,7 +85,7 @@ decide whether a particular decision requires a PEP, and to resolve technical disputes in general. The trio's authority does not include changing the governance itself, or other non-technical disputes that may arise; these should be handled through the process described in -PEP 8001. +:pep:`8001`. What are NOT considered as the role responsibilities of the trio @@ -137,7 +138,7 @@ Only once this PEP is accepted, any active core developers (who are eligible to can submit nomination of groups of three. Once this PEP is accepted and core devs have submitted their nominations, voting -can begin, and the voting mechanism described in PEP 8001 will be used. +can begin, and the voting mechanism described in :pep:`8001` will be used. Qualities desired out of the trio: @@ -398,14 +399,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8012.rst b/peps/pep-8012.rst similarity index 96% rename from pep-8012.rst rename to peps/pep-8012.rst index 71b46954e..072f3851a 100644 --- a/pep-8012.rst +++ b/peps/pep-8012.rst @@ -3,6 +3,7 @@ Title: The Community Governance Model Author: Ɓukasz Langa Status: Rejected Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 03-Oct-2018 @@ -10,11 +11,11 @@ Created: 03-Oct-2018 PEP Rejection ============= -PEP 8012 was rejected `by a core developer vote +:pep:`8012` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Abstract ======== @@ -27,7 +28,7 @@ the role of a centralized singular leader or a governing council. It describes how, when, and why votes are conducted for decisions affecting the Python language. It also describes the criteria for voting eligibility. -Should this model be adopted, it will be codified in PEP 13. +Should this model be adopted, it will be codified in :pep:`13`. This model can be affectionately called "The Least Worst Governance Model" by its property that while far from ideal, it's still the most @@ -194,7 +195,7 @@ others. Contributors and any workgroups in this model are self-selecting. the interests of a single person or a small group of people. **Proven** This model works. There is a number of large open-source projects -run this way (two of which, Rust and Django, are described in PEP 8002). +run this way (two of which, Rust and Django, are described in :pep:`8002`). ECMAScript and C++ are similarly developed. @@ -263,7 +264,7 @@ Regular decision process ------------------------ Primary work happens through bug tracker issues and pull requests. -Core developers should avoind pushing their changes directly to the cpython +Core developers should avoid pushing their changes directly to the cpython repository, instead relying on pull requests. Approving a pull request by a core developer allows it to be merged without further process. @@ -302,7 +303,7 @@ PEP, Enhanced ~~~~~~~~~~~~~ The PEP process is augmented with the following changes and clarifications -over information already present in PEP 1: +over information already present in :pep:`1`: * PEPs are not merged until the final decision is made on them; they are open pull requests on GitHub until that moment; @@ -463,7 +464,7 @@ conduct-wg@python.org. Acknowledgements ================ -Thank you to the authors of PEP 8002 which was a helpful resource in +Thank you to the authors of :pep:`8002` which was a helpful resource in shaping this document. Thank you to Alex Crichton and the Rust team for a governance model @@ -474,14 +475,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8013.rst b/peps/pep-8013.rst similarity index 96% rename from pep-8013.rst rename to peps/pep-8013.rst index c55007703..a266d4b98 100644 --- a/pep-8013.rst +++ b/peps/pep-8013.rst @@ -3,6 +3,7 @@ Title: The External Council Governance Model Author: Steve Dower Status: Rejected Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 14-Sep-2018 @@ -11,8 +12,8 @@ Abstract This PEP proposes a new model of Python governance based on a Council of Auditors (CoA) tasked with making final decisions for the language. -It differs from PEP 8010 by specifically not proposing a central -singular leader, and from PEP 8011 by disallowing core committers from +It differs from :pep:`8010` by specifically not proposing a central +singular leader, and from :pep:`8011` by disallowing core committers from being council members. It describes the size and role of the council, how the initial group of council members will be chosen, any term limits of the council members, and how successors will be elected. @@ -27,17 +28,17 @@ circumstances. This only works when process is unspecified, but all participants have similar expectations. This PEP does *not* name the members of the CoA. Should this model be -adopted, it will be codified in PEP 13 along with the names of all +adopted, it will be codified in :pep:`13` along with the names of all officeholders described in this PEP. PEP Rejection ============= -PEP 8013 was rejected `by a core developer vote +:pep:`8013` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. The Importance of the Grey Area =============================== @@ -350,14 +351,3 @@ Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8014.rst b/peps/pep-8014.rst similarity index 97% rename from pep-8014.rst rename to peps/pep-8014.rst index 31ca81eb9..85a038b62 100644 --- a/pep-8014.rst +++ b/peps/pep-8014.rst @@ -3,6 +3,7 @@ Title: The Commons Governance Model Author: Jack Jansen Status: Rejected Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 16-Sep-2018 @@ -29,11 +30,11 @@ carried by a sufficient majority. PEP Rejection ============= -PEP 8014 was rejected `by a core developer vote +:pep:`8014` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Introduction ============ @@ -310,21 +311,10 @@ other person or body. Note that this proposal most likely favors conservatism over progression. Or, at least, the danger of it leading to stagnation is bigger than the danger of it leading to reckless blazing ahead into unknown territories. So: we should realise -that it is unlikely that a PEP like PEP 572 will pass if this model is in +that it is unlikely that a PEP like :pep:`572` will pass if this model is in place. Copyright ========= This document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8015.rst b/peps/pep-8015.rst similarity index 99% rename from pep-8015.rst rename to peps/pep-8015.rst index 036ddef75..64f93e53d 100644 --- a/pep-8015.rst +++ b/peps/pep-8015.rst @@ -3,6 +3,7 @@ Title: Organization of the Python community Author: Victor Stinner Status: Rejected Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 04-Oct-2018 @@ -25,11 +26,11 @@ developers, need ``>= 2/3`` majority). PEP Rejection ============= -PEP 8015 was rejected `by a core developer vote +:pep:`8015` was rejected `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. -PEP 8016 and the governance model it describes were chosen instead. +:pep:`8016` and the governance model it describes were chosen instead. Rationale diff --git a/pep-8016.rst b/peps/pep-8016.rst similarity index 96% rename from pep-8016.rst rename to peps/pep-8016.rst index a16de96e2..22810f02a 100644 --- a/pep-8016.rst +++ b/peps/pep-8016.rst @@ -3,6 +3,7 @@ Title: The Steering Council Model Author: Nathaniel J. Smith, Donald Stufft Status: Accepted Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 01-Nov-2018 @@ -10,7 +11,7 @@ Note ==== This PEP is retained for historical purposes, but the official -governance document is now PEP 13. +governance document is now :pep:`13`. Abstract @@ -30,9 +31,9 @@ decisions. PEP Acceptance ============== -PEP 8016 was accepted `by a core developer vote +:pep:`8016` was accepted `by a core developer vote `__ -described in PEP 8001 on Monday, December 17, 2018. +described in :pep:`8001` on Monday, December 17, 2018. Rationale @@ -322,7 +323,7 @@ active privileges like voting or nominating for the steering council, and commit access. The initial active core team members will consist of everyone -currently listed in the `"Python core" team on Github +currently listed in the `"Python core" team on GitHub `__, and the initial inactive members will consist of everyone else who has been a committer in the past. @@ -359,16 +360,5 @@ Copyright ========= Text copied from Django used under `their license -`__. The rest of +`__. The rest of this document has been placed in the public domain. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8100.rst b/peps/pep-8100.rst similarity index 75% rename from pep-8100.rst rename to peps/pep-8100.rst index 3a5090738..b42207cb7 100644 --- a/pep-8100.rst +++ b/peps/pep-8100.rst @@ -1,10 +1,9 @@ PEP: 8100 -Title: January 2019 steering council election -Version: $Revision$ -Last-Modified: $Date$ -Author: Nathaniel J. Smith , Ee W. Durbin III -Status: Active +Title: January 2019 Steering Council election +Author: Nathaniel J. Smith , Ee Durbin +Status: Final Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 03-Jan-2019 @@ -14,7 +13,7 @@ Abstract This document describes the schedule and other details of the January 2019 election for the Python steering council, as specified in -PEP 13. This is the first steering council election. +:pep:`13`. This is the first steering council election. Returns officer @@ -22,9 +21,9 @@ Returns officer In future elections, the returns officer will be appointed by the outgoing steering council. Since this is the first election, we have -no outgoing steering council, and PEP 13 says that the returns officer +no outgoing steering council, and :pep:`13` says that the returns officer is instead appointed by the PSF Executive Director, Ewa Jodlowska. -`She appointed Ee W. Durbin III +`She appointed Ee Durbin `__. @@ -144,7 +143,7 @@ The top five vote-getters are: * Guido van Rossum * Nick Coghlan -No conflict of interest as defined in PEP 13 were observed. +No conflict of interest as defined in :pep:`13` were observed. The full vote counts are as follows: @@ -200,115 +199,104 @@ Active Python core developers :: - Alex Gaynor - Alex Martelli - Alexander Belopolsky - Alexandre Vassalotti - Amaury Forgeot d'Arc - Andrew Kuchling - Andrew Svetlov - Antoine Pitrou - Armin Ronacher - Barry Warsaw - Benjamin Peterson - Berker Peksag - Brett Cannon - Brian Curtin - Carol Willing - Chris Jerdonek - Chris Withers - Christian Heimes - David Malcolm - David Wolever - Davin Potts - Dino Viehland - Donald Stufft - Doug Hellmann - Eli Bendersky - Emily Morehouse - Éric Araujo - Eric Snow - Eric V. Smith - Ethan Furman - Ezio Melotti - Facundo Batista - Fred Drake - Georg Brandl - Giampaolo Rodola' - Gregory P. Smith - Guido van Rossum - Hyeshik Chang - Hynek Schlawack - INADA Naoki - Ivan Levkivskyi - Jack Diederich - Jack Jansen - Jason R. Coombs - Jeff Hardy - Jeremy Hylton - JesĂșs Cea - Julien Palard - Kurt B. Kaiser - Kushal Das - Larry Hastings - Lars GustĂ€bel - Lisa Roach - Ɓukasz Langa - Marc-Andre Lemburg - Mariatta - Mark Dickinson - Mark Hammond - Mark Shannon - Martin Panter - Matthias Klose - Meador Inge - Michael Hudson-Doyle - Nathaniel J. Smith - Ned Deily - Neil Schemenauer - Nick Coghlan - Pablo Galindo - Paul Moore - Petr Viktorin - Petri Lehtinen - Philip Jenvey - R. David Murray - Raymond Hettinger - Robert Collins - Ronald Oussoren - Sandro Tosi - Senthil Kumaran - Serhiy Storchaka - Sjoerd Mullender - Stefan Krah - Steve Dower - Steven Daprano - T. Wouters - Tal Einat - Terry Jan Reedy - Thomas Heller - Tim Golden - Tim Peters - Trent Nelson - Victor Stinner - Vinay Sajip - Walter Dörwald - Xiang Zhang - Yury Selivanov - Zachary Ware + Alex Gaynor + Alex Martelli + Alexander Belopolsky + Alexandre Vassalotti + Amaury Forgeot d'Arc + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Armin Ronacher + Barry Warsaw + Benjamin Peterson + Berker Peksag + Brett Cannon + Brian Curtin + Carol Willing + Chris Jerdonek + Chris Withers + Christian Heimes + David Malcolm + David Wolever + Davin Potts + Dino Viehland + Donald Stufft + Doug Hellmann + Eli Bendersky + Emily Morehouse + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Ezio Melotti + Facundo Batista + Fred Drake + Georg Brandl + Giampaolo Rodola' + Gregory P. Smith + Guido van Rossum + Hyeshik Chang + Hynek Schlawack + INADA Naoki + Ivan Levkivskyi + Jack Diederich + Jack Jansen + Jason R. Coombs + Jeff Hardy + Jeremy Hylton + JesĂșs Cea + Julien Palard + Kurt B. Kaiser + Kushal Das + Larry Hastings + Lars GustĂ€bel + Lisa Roach + Ɓukasz Langa + Marc-Andre Lemburg + Mariatta + Mark Dickinson + Mark Hammond + Mark Shannon + Martin Panter + Matthias Klose + Meador Inge + Michael Hudson-Doyle + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Moore + Petr Viktorin + Petri Lehtinen + Philip Jenvey + R. David Murray + Raymond Hettinger + Robert Collins + Ronald Oussoren + Sandro Tosi + Senthil Kumaran + Serhiy Storchaka + Sjoerd Mullender + Stefan Krah + Steve Dower + Steven Daprano + T. Wouters + Tal Einat + Terry Jan Reedy + Thomas Heller + Tim Golden + Tim Peters + Trent Nelson + Victor Stinner + Vinay Sajip + Walter Dörwald + Xiang Zhang + Yury Selivanov + Zachary Ware .. [1] This repository is private and accessible only to Python Core Developers, administrators, and Python Software Foundation Staff as it contains personal email addresses. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8101.rst b/peps/pep-8101.rst similarity index 75% rename from pep-8101.rst rename to peps/pep-8101.rst index 8fd781b56..ea89abb76 100644 --- a/pep-8101.rst +++ b/peps/pep-8101.rst @@ -1,11 +1,10 @@ PEP: 8101 -Title: 2020 Term steering council election -Version: $Revision$ -Last-Modified: $Date$ -Author: Ewa Jodlowska , Ee W. Durbin III +Title: 2020 Term Steering Council election +Author: Ewa Jodlowska , Ee Durbin Sponsor: Brett Cannon -Status: Active +Status: Final Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 16-Nov-2019 @@ -15,7 +14,7 @@ Abstract This document describes the schedule and other details of the December 2019 election for the Python steering council, as specified in -PEP 13. This is steering council election for the 2020 term. +:pep:`13`. This is steering council election for the 2020 term. Election Administration @@ -23,7 +22,7 @@ Election Administration The steering council appointed the `Python Software Foundation `__ -Director of Infrastructure, Ee W. Durbin III, to implement the election, +Director of Infrastructure, Ee Durbin, to implement the election, and `Python Software Foundation `__ Executive Director, Ewa Jodlowska, to communicate announcements regarding the election. @@ -145,7 +144,7 @@ The top five vote-getters are: * Thomas Wouters * Victor Stinner -No conflict of interest as defined in PEP 13 were observed. +No conflict of interest as defined in :pep:`13` were observed. The full vote counts are as follows: @@ -186,101 +185,90 @@ Active Python core developers :: - Abhilash Raj - Alex Gaynor - Alex Martelli - Alexander Belopolsky - Andrew Kuchling - Andrew Svetlov - Antoine Pitrou - Barry Warsaw - Benjamin Peterson - Berker Peksağ - Brett Cannon - Brian Curtin - Brian Quinlan - Carol Willing - Cheryl Sabella - Chris Withers - Christian Heimes - Christian Tismer - Davin Potts - Dino Viehland - Donald Stufft - Emily Morehouse - Éric Araujo - Eric Snow - Eric V. Smith - Ethan Furman - Ezio Melotti - Facundo Batista - Fred Drake - Giampaolo RodolĂ  - Gregory P. Smith - Guido van Rossum - Inada Naoki - Ivan Levkivskyi - Jason R. Coombs - Jeremy Kloth - JesĂșs Cea - Joannah Nanjekye - Julien Palard - Kurt B. Kaiser - Kushal Das - Larry Hastings - Lisa Roach - Ɓukasz Langa - Marc-AndrĂ© Lemburg - Mariatta - Mark Dickinson - Mark Shannon - Matthias Klose - Michael Foord - Nathaniel J. Smith - Ned Deily - Neil Schemenauer - Nick Coghlan - Pablo Galindo - Paul Ganssle - Paul Moore - Petr Viktorin - R. David Murray - Raymond Hettinger - Robert Collins - Ronald Oussoren - Senthil Kumaran - Serhiy Storchaka - Skip Montanaro - Stefan Behnel - Stefan Krah - Steve Dower - Steven D'Aprano - StĂ©phane Wirtel - Tal Einat - Terry Jan Reedy - Thomas Wouters - Tim Golden - Tim Peters - Victor Stinner - Vinay Sajip - Walter Dörwald - Xavier de Gaye - Xiang Zhang - Yury Selivanov - Zachary Ware + Abhilash Raj + Alex Gaynor + Alex Martelli + Alexander Belopolsky + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Barry Warsaw + Benjamin Peterson + Berker Peksağ + Brett Cannon + Brian Curtin + Brian Quinlan + Carol Willing + Cheryl Sabella + Chris Withers + Christian Heimes + Christian Tismer + Davin Potts + Dino Viehland + Donald Stufft + Emily Morehouse + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Ezio Melotti + Facundo Batista + Fred Drake + Giampaolo RodolĂ  + Gregory P. Smith + Guido van Rossum + Inada Naoki + Ivan Levkivskyi + Jason R. Coombs + Jeremy Kloth + JesĂșs Cea + Joannah Nanjekye + Julien Palard + Kurt B. Kaiser + Kushal Das + Larry Hastings + Lisa Roach + Ɓukasz Langa + Marc-AndrĂ© Lemburg + Mariatta + Mark Dickinson + Mark Shannon + Matthias Klose + Michael Foord + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Ganssle + Paul Moore + Petr Viktorin + R. David Murray + Raymond Hettinger + Robert Collins + Ronald Oussoren + Senthil Kumaran + Serhiy Storchaka + Skip Montanaro + Stefan Behnel + Stefan Krah + Steve Dower + Steven D'Aprano + StĂ©phane Wirtel + Tal Einat + Terry Jan Reedy + Thomas Wouters + Tim Golden + Tim Peters + Victor Stinner + Vinay Sajip + Walter Dörwald + Xavier de Gaye + Xiang Zhang + Yury Selivanov + Zachary Ware .. [1] This repository is private and accessible only to Python Core Developers, administrators, and Python Software Foundation Staff as it contains personal email addresses. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/pep-8102.rst b/peps/pep-8102.rst similarity index 74% rename from pep-8102.rst rename to peps/pep-8102.rst index d8471203e..5cce7ccec 100644 --- a/pep-8102.rst +++ b/peps/pep-8102.rst @@ -1,11 +1,10 @@ PEP: 8102 -Title: 2021 Term steering council election -Version: $Revision$ -Last-Modified: $Date$ -Author: Ewa Jodlowska , Ee W. Durbin III , Joe Carey +Title: 2021 Term Steering Council election +Author: Ewa Jodlowska , Ee Durbin , Joe Carey Sponsor: Brett Cannon -Status: Active +Status: Final Type: Informational +Topic: Governance Content-Type: text/x-rst Created: 29-Oct-2020 @@ -15,7 +14,7 @@ Abstract This document describes the schedule and other details of the December 2020 election for the Python steering council, as specified in -PEP 13. This is the steering council election for the 2021 term. +:pep:`13`. This is the steering council election for the 2021 term. Election Administration @@ -23,7 +22,7 @@ Election Administration The steering council appointed the `Python Software Foundation `__ -Director of Infrastructure, Ee W. Durbin III, +Director of Infrastructure, Ee Durbin, and Accounting Manager, Joe Carey, to coadminister the election. `Python Software Foundation `__ @@ -75,7 +74,7 @@ Voter Roll ========== All active Python core team members are eligible to vote. Active status -is determined as described in `PEP 13 `_ +is determined as described in :pep:`PEP 13 <13#membership>` and implemented via the software at `python/voters `_ [1]_. Ballots will be distributed based on the `The Python Voter Roll for this @@ -151,7 +150,7 @@ The top five vote-getters are: * Pablo Galindo Salgado * Thomas Wouters -No conflict of interest as defined in PEP 13 were observed. +No conflict of interest as defined in :pep:`13` were observed. The full vote counts are as follows: @@ -194,110 +193,99 @@ Active Python core developers :: - Abhilash Raj - Alex Gaynor - Alex Martelli - Alexander Belopolsky - Andrew Kuchling - Andrew Svetlov - Antoine Pitrou - Barry Warsaw - Batuhan Taskaya - Benjamin Peterson - Berker Peksağ - Brandt Bucher - Brett Cannon - Brian Curtin - Brian Quinlan - Carol Willing - Cheryl Sabella - Chris Jerdonek - Chris Withers - Christian Heimes - Christian Tismer - Davin Potts - Dino Viehland - Donald Stufft - Dong-hee Na - Emily Morehouse - Éric Araujo - Eric Snow - Eric V. Smith - Ethan Furman - Ezio Melotti - Facundo Batista - Fred Drake - Georg Brandl - Giampaolo RodolĂ  - Gregory P. Smith - Guido van Rossum - Hynek Schlawack - Inada Naoki - Ivan Levkivskyi - Jack Jansen - Jason R. Coombs - Jeremy Kloth - JesĂșs Cea - Joannah Nanjekye - Julien Palard - Karthikeyan Singaravelan - Kurt B. Kaiser - Kushal Das - Kyle Stanley - Larry Hastings - Lisa Roach - Ɓukasz Langa - Lysandros Nikolaou - Marc-AndrĂ© Lemburg - Mariatta - Mark Dickinson - Mark Hammond - Mark Shannon - Matthias Klose - Michael Foord - Nathaniel J. Smith - Ned Deily - Neil Schemenauer - Nick Coghlan - Pablo Galindo - Paul Ganssle - Paul Moore - Petr Viktorin - R. David Murray - Raymond Hettinger - Robert Collins - Ronald Oussoren - Sandro Tosi - Senthil Kumaran - Serhiy Storchaka - Stefan Behnel - Steve Dower - Steven D'Aprano - StĂ©phane Wirtel - Tal Einat - Terry Jan Reedy - Thomas Wouters - Tim Golden - Tim Peters - Victor Stinner - Vinay Sajip - Walter Dörwald - Xiang Zhang - Yury Selivanov - Zachary Ware + Abhilash Raj + Alex Gaynor + Alex Martelli + Alexander Belopolsky + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Barry Warsaw + Batuhan Taskaya + Benjamin Peterson + Berker Peksağ + Brandt Bucher + Brett Cannon + Brian Curtin + Brian Quinlan + Carol Willing + Cheryl Sabella + Chris Jerdonek + Chris Withers + Christian Heimes + Christian Tismer + Davin Potts + Dino Viehland + Donald Stufft + Dong-hee Na + Emily Morehouse + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Ezio Melotti + Facundo Batista + Fred Drake + Georg Brandl + Giampaolo RodolĂ  + Gregory P. Smith + Guido van Rossum + Hynek Schlawack + Inada Naoki + Ivan Levkivskyi + Jack Jansen + Jason R. Coombs + Jeremy Kloth + JesĂșs Cea + Joannah Nanjekye + Julien Palard + Karthikeyan Singaravelan + Kurt B. Kaiser + Kushal Das + Kyle Stanley + Larry Hastings + Lisa Roach + Ɓukasz Langa + Lysandros Nikolaou + Marc-AndrĂ© Lemburg + Mariatta + Mark Dickinson + Mark Hammond + Mark Shannon + Matthias Klose + Michael Foord + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Ganssle + Paul Moore + Petr Viktorin + R. David Murray + Raymond Hettinger + Robert Collins + Ronald Oussoren + Sandro Tosi + Senthil Kumaran + Serhiy Storchaka + Stefan Behnel + Steve Dower + Steven D'Aprano + StĂ©phane Wirtel + Tal Einat + Terry Jan Reedy + Thomas Wouters + Tim Golden + Tim Peters + Victor Stinner + Vinay Sajip + Walter Dörwald + Xiang Zhang + Yury Selivanov + Zachary Ware .. [1] This repository is private and accessible only to Python Core Developers, administrators, and Python Software Foundation Staff as it contains personal email addresses. - - - -.. - Local Variables: - mode: indented-text - indent-tabs-mode: nil - sentence-end-double-space: t - fill-column: 70 - coding: utf-8 - End: diff --git a/peps/pep-8103.rst b/peps/pep-8103.rst new file mode 100644 index 000000000..ebc575854 --- /dev/null +++ b/peps/pep-8103.rst @@ -0,0 +1,273 @@ +PEP: 8103 +Title: 2022 Term Steering Council election +Author: Ewa Jodlowska , Ee Durbin , Joe Carey +Sponsor: Barry Warsaw +Status: Final +Type: Informational +Topic: Governance +Content-Type: text/x-rst +Created: 04-Oct-2021 + + +Abstract +======== + +This document describes the schedule and other details of the December +2021 election for the Python steering council, as specified in +:pep:`13`. This is the steering council election for the 2022 term +(i.e. Python 3.11). + + +Election Administration +======================= + +The steering council appointed the +`Python Software Foundation `__ +Director of Infrastructure, Ee Durbin +and Accounting Manager, Joe Carey, to coadminister the election. + + +Schedule +======== + +There will be a two-week nomination period, followed by a two-week +vote. + +The nomination period was: November 1, 2021 through November 16, 2021 12:00 UTC +(The end of November 15, 2021 `Anywhere on Earth +`_). As announced on `python-committers +`_, +and took place on `discuss.python.org `_. + +The voting period is: December 1, 2021 12:00 UTC through December 16, 2021 +12:00 UTC (The end of December 15, 2021 `Anywhere on Earth +`_). + + +Candidates +========== + +Candidates must be nominated by a core team member. If the candidate +is a core team member, they may nominate themselves. + +Nominees (in alphabetical order): + +- `Brett Cannon `_ +- `Mariatta `_ +- `David Mertz `_ +- `Dong-hee Na `_ +- `Pablo Galindo Salgado `_ +- `Gregory P. Smith `_ +- `Victor Stinner `_ +- `Petr Viktorin `_ +- `Barry Warsaw `_ +- `Thomas Wouters `_ + +Withdrawn nominations: + +- None + +Voter Roll +========== + +All active Python core team members are eligible to vote. Active status +is determined as described in :pep:`PEP 13 <13#membership>` +and implemented via the software at `python/voters `_ [1]_. + +Ballots will be distributed based on the `The Python Voter Roll for this +election +`_ +[1]_. + +While this file is not public as it contains private email addresses, the +`Complete Voter Roll`_ by name will be made available when the roll is +created. + +Election Implementation +======================= + +The election will be conducted using the `Helios Voting Service +`__. + + +Configuration +------------- + +Short name: ``2022-python-steering-council`` + +Name: ``2022 Python Steering Council Election`` + +Description: ``Election for the Python steering council, as specified in PEP 13. This is steering council election for the 2022 term.`` + +type: ``Election`` + +Use voter aliases: ``[X]`` + +Randomize answer order: ``[X]`` + +Private: ``[X]`` + +Help Email Address: ``psf-election@python.org`` + +Voting starts at: ``December 1, 2021 00:00 UTC`` + +Voting ends at: ``December 16, 2021 12:00 UTC`` + +This will create an election in which: + +* Voting is not open to the public, only those on the `Voter Roll`_ may + participate. Ballots will be emailed when voting starts. +* Candidates are presented in random order, to help avoid bias. +* Voter identities and ballots are protected against cryptographic advances. + +Questions +--------- + +Question 1 +~~~~~~~~~~ + +Select between ``0`` and ``- (approval)`` answers. Result Type: ``absolute`` + +Question: ``Select candidates for the Python Steering Council`` + +Answer #1 - #N: ``Candidates from Candidates_ Section`` + + + +Results +======= + +Of 85 eligible voters, 67 cast ballots. + +The top five vote-getters are: + +* Pablo Galindo Salgado +* Petr Viktorin +* Thomas Wouters +* Gregory P. Smith +* Brett Cannon + +No conflict of interest as defined in :pep:`13` were observed. + +The full vote counts are as follows: + ++-----------------------+----------------+ +| Candidate | Votes Received | ++=======================+================+ +| Pablo Galindo Salgado | 61 | ++-----------------------+----------------+ +| Petr Viktorin | 48 | ++-----------------------+----------------+ +| Thomas Wouters | 48 | ++-----------------------+----------------+ +| Gregory P. Smith | 44 | ++-----------------------+----------------+ +| Brett Cannon | 42 | ++-----------------------+----------------+ +| Barry Warsaw | 39 | ++-----------------------+----------------+ +| Victor Stinner | 35 | ++-----------------------+----------------+ +| Mariatta | 34 | ++-----------------------+----------------+ +| Dong-hee Na | 26 | ++-----------------------+----------------+ +| David Mertz | 24 | ++-----------------------+----------------+ + + +Copyright +========= + +This document has been placed in the public domain. + + +Complete Voter Roll +=================== + +Active Python core developers +----------------------------- + +:: + + Abhilash Raj + Alex Gaynor + Ammar Askar + Andrew Kuchling + Andrew Svetlov + Antoine Pitrou + Barry Warsaw + Batuhan Taskaya + Benjamin Peterson + Berker Peksağ + Brandt Bucher + Brett Cannon + Brian Curtin + Brian Quinlan + Carol Willing + Cheryl Sabella + Chris Jerdonek + Chris Withers + Christian Heimes + Dino Viehland + Dong-hee Na + Éric Araujo + Eric Snow + Eric V. Smith + Ethan Furman + Facundo Batista + Fred Drake + Giampaolo RodolĂ  + Gregory P. Smith + Guido van Rossum + Hynek Schlawack + Inada Naoki + Irit Katriel + Ivan Levkivskyi + Jason R. Coombs + Jeremy Kloth + JesĂșs Cea + Joannah Nanjekye + Julien Palard + Karthikeyan Singaravelan + Ken Jin + Kushal Das + Kyle Stanley + Larry Hastings + Lisa Roach + Ɓukasz Langa + Lysandros Nikolaou + Marc-AndrĂ© Lemburg + Mariatta + Mark Dickinson + Mark Shannon + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Ganssle + Paul Moore + Petr Viktorin + Raymond Hettinger + Ronald Oussoren + Senthil Kumaran + Serhiy Storchaka + Stefan Behnel + StĂ©phane Wirtel + Steve Dower + Tal Einat + Terry Jan Reedy + Thomas Wouters + Tim Golden + Tim Peters + Victor Stinner + Vinay Sajip + Xiang Zhang + Yury Selivanov + Zachary Ware + + +.. [1] This repository is private and accessible only to Python Core + Developers, administrators, and Python Software Foundation Staff as it + contains personal email addresses. diff --git a/peps/pep-8104.rst b/peps/pep-8104.rst new file mode 100644 index 000000000..438d8d0a2 --- /dev/null +++ b/peps/pep-8104.rst @@ -0,0 +1,271 @@ +PEP: 8104 +Title: 2023 Term Steering Council election +Author: Ee Durbin +Sponsor: Brett Cannon +Status: Active +Type: Informational +Topic: Governance +Content-Type: text/x-rst +Created: 08-Nov-2022 + + +Abstract +======== + +This document describes the schedule and other details of the December +2022 election for the Python steering council, as specified in +:pep:`13`. This is the steering council election for the 2023 term +(i.e. Python 3.12). + + +Election Administration +======================= + +The steering council appointed the +`Python Software Foundation `__ +Director of Infrastructure, Ee Durbin, to administer the election. + + +Schedule +======== + +There will be a two-week nomination period, followed by a two-week +vote. + +The nomination period was: November 14, 2022 through `November 28, 2022 AoE +`_ [#note-aoe]_. + +The voting period was: December 1, 2022 through `December 14, 2022 AoE +`_ [#note-aoe]_. + + +Candidates +========== + +Candidates must be nominated by a core team member. If the candidate +is a core team member, they may nominate themselves. + +Nominees (in alphabetical order): + +- `Brett Cannon `_ +- `Emily Morehouse `_ +- `Dong-hee Na `_ +- `Pablo Galindo Salgado `_ +- `Gregory P. Smith `_ +- `Victor Stinner `_ +- `Petr Viktorin `_ +- `Thomas Wouters `_ + +Withdrawn nominations: + +- None + +Voter Roll +========== + +All active Python core team members are eligible to vote. Active status +is determined as :pep:`described in PEP 13 <13#membership>` +and implemented via the software at `python/voters `_ +[#note-voters]_. + +Ballots will be distributed based on the the `Python Voter Roll +`_ [#note-voters]_ +for this election. + +While this file is not public as it contains private email addresses, the +`Complete Voter Roll`_ by name will be made available when the roll is +created. + +Election Implementation +======================= + +The election will be conducted using the `Helios Voting Service +`__. + + +Configuration +------------- + +Short name: ``2023-python-steering-council`` + +Name: ``2023 Python Steering Council Election`` + +Description: ``Election for the Python steering council, as specified in PEP 13. This is steering council election for the 2023 term.`` + +type: ``Election`` + +Use voter aliases: ``[X]`` + +Randomize answer order: ``[X]`` + +Private: ``[X]`` + +Help Email Address: ``psf-election@python.org`` + +Voting starts at: ``December 1, 2022 12:00 UTC`` + +Voting ends at: ``December 15, 2022 12:00 UTC`` + +This will create an election in which: + +* Voting is not open to the public, only those on the `Voter Roll`_ may + participate. Ballots will be emailed when voting starts. +* Candidates are presented in random order, to help avoid bias. +* Voter identities and ballots are protected against cryptographic advances. + +Questions +--------- + +Question 1 +~~~~~~~~~~ + +Select between ``0`` and ``- (approval)`` answers. Result Type: ``absolute`` + +Question: ``Select candidates for the Python Steering Council`` + +Answer #1 - #N: ``Candidates from Candidates_ Section`` + + + +Results +======= + +Of 85 eligible voters, 66 cast ballots. + +The top five vote-getters are: + +* Pablo Galindo Salgado +* Gregory P. Smith +* Emily Morehouse +* Brett Cannon +* Thomas Wouters + +No conflict of interest as defined in :pep:`13` were observed. + +The full vote counts are as follows: + ++-----------------------+----------------+ +| Candidate | Votes Received | ++=======================+================+ +| Pablo Galindo Salgado | 61 | ++-----------------------+----------------+ +| Gregory P. Smith | 48 | ++-----------------------+----------------+ +| Emily Morehouse | 47 | ++-----------------------+----------------+ +| Brett Cannon | 42 | ++-----------------------+----------------+ +| Thomas Wouters | 39 | ++-----------------------+----------------+ +| Petr Viktorin | 36 | ++-----------------------+----------------+ +| Victor Stinner | 34 | ++-----------------------+----------------+ +| Dong-hee Na | 29 | ++-----------------------+----------------+ + +Copyright +========= + +This document has been placed in the public domain. + + +Complete Voter Roll +=================== + +Active Python core developers +----------------------------- + +.. code-block:: text + + Alex Gaynor + Alex Waygood + Ammar Askar + Andrew Svetlov + Antoine Pitrou + Barry Warsaw + Batuhan Taskaya + Benjamin Peterson + Berker Peksağ + Brandt Bucher + Brett Cannon + Brian Curtin + Brian Quinlan + Carol Willing + Cheryl Sabella + Chris Jerdonek + Chris Withers + Christian Heimes + Dennis Sweeney + Dino Viehland + Dong-hee Na + Emily Morehouse + Éric Araujo + Eric Snow + Eric V. Smith + Erlend Egeberg Aasland + Ethan Furman + Ezio Melotti + Facundo Batista + Filipe LaĂ­ns + Fred Drake + Georg Brandl + Giampaolo RodolĂ  + Gregory P. Smith + Guido van Rossum + Hugo van Kemenade + Hynek Schlawack + Inada Naoki + Irit Katriel + Ivan Levkivskyi + Jason R. Coombs + Jelle Zijlstra + Jeremy Kloth + JesĂșs Cea + Joannah Nanjekye + Julien Palard + Karthikeyan Singaravelan + Ken Jin + Kumar Aditya + Kurt B. Kaiser + Kushal Das + Kyle Stanley + Larry Hastings + Ɓukasz Langa + Lysandros Nikolaou + Marc-AndrĂ© Lemburg + Mariatta + Mark Dickinson + Mark Shannon + Nathaniel J. Smith + Ned Deily + Neil Schemenauer + Nick Coghlan + Pablo Galindo + Paul Ganssle + Paul Moore + Petr Viktorin + R. David Murray + Raymond Hettinger + Ronald Oussoren + Senthil Kumaran + Serhiy Storchaka + Stefan Behnel + StĂ©phane Wirtel + Steve Dower + Steven D'Aprano + Tal Einat + Terry Jan Reedy + Thomas Wouters + Tim Golden + Tim Peters + Victor Stinner + Vinay Sajip + Yury Selivanov + Zachary Ware + + +.. [#note-voters] This repository is private and accessible only to Python Core + Developers, administrators, and Python Software Foundation Staff as it + contains personal email addresses. +.. [#note-aoe] AoE: `Anywhere on Earth `_. diff --git a/pyramid-pep-template b/pyramid-pep-template deleted file mode 100644 index f65a5ab5e..000000000 --- a/pyramid-pep-template +++ /dev/null @@ -1,6 +0,0 @@ - -%(body)s diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..10404cc0b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,16 @@ +[pytest] +# https://docs.pytest.org/en/7.3.x/reference/reference.html#command-line-flags +addopts = + -r a + --strict-config + --strict-markers + --import-mode=importlib + --cov check_peps --cov pep_sphinx_extensions + --cov-report html --cov-report xml +empty_parameter_set_mark = fail_at_collect +filterwarnings = + error +minversion = 6.0 +testpaths = pep_sphinx_extensions +xfail_strict = True +disable_test_id_escaping_and_forfeit_all_rights_to_community_support = True diff --git a/readthedocs.yaml b/readthedocs.yaml new file mode 100644 index 000000000..46d2f2139 --- /dev/null +++ b/readthedocs.yaml @@ -0,0 +1,15 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + + commands: + - make dirhtml JOBS=$(nproc) OUTPUT_DIR=_readthedocs/html + +sphinx: + builder: dirhtml + +search: + ignore: ['*'] diff --git a/requirements.txt b/requirements.txt index 837f41b3e..bf7a8470e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,10 @@ # Requirements for building PEPs with Sphinx -sphinx >= 3.5 -docutils >= 0.16 +Pygments >= 2.9.0 +# Sphinx 6.1.0 broke copying images in parallel builds; fixed in 6.1.2 +# See https://github.com/sphinx-doc/sphinx/pull/11100 +Sphinx >= 5.1.1, != 6.1.0, != 6.1.1 +docutils >= 0.19.0 -# For RSS -feedgen >= 0.9.0 # For RSS feed +# For tests +pytest +pytest-cov diff --git a/roman.py b/roman.py deleted file mode 100644 index 394a43ae9..000000000 --- a/roman.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Convert to and from Roman numerals""" - -__author__ = "Mark Pilgrim (f8dy@diveintopython.org)" -__version__ = "1.4" -__date__ = "8 August 2001" -__copyright__ = """Copyright (c) 2001 Mark Pilgrim - -This program is part of "Dive Into Python", a free Python tutorial for -experienced programmers. Visit http://diveintopython.org/ for the -latest version. - -This program is free software; you can redistribute it and/or modify -it under the terms of the Python 2.1.1 license, available at -http://www.python.org/2.1.1/license.html -""" - -import re - -# Define exceptions -class RomanError(Exception): pass -class OutOfRangeError(RomanError): pass -class NotIntegerError(RomanError): pass -class InvalidRomanNumeralError(RomanError): pass - -#Define digit mapping -romanNumeralMap = (('M', 1000), - ('CM', 900), - ('D', 500), - ('CD', 400), - ('C', 100), - ('XC', 90), - ('L', 50), - ('XL', 40), - ('X', 10), - ('IX', 9), - ('V', 5), - ('IV', 4), - ('I', 1)) - -def toRoman(n): - """convert integer to Roman numeral""" - if not (0 < n < 5000): - raise OutOfRangeError("number out of range (must be 1..4999)") - if int(n) != n: - raise NotIntegerError("decimals can not be converted") - - result = "" - for numeral, integer in romanNumeralMap: - while n >= integer: - result += numeral - n -= integer - return result - -#Define pattern to detect valid Roman numerals -romanNumeralPattern = re.compile(""" - ^ # beginning of string - M{0,4} # thousands - 0 to 4 M's - (CM|CD|D?C{0,3}) # hundreds - 900 (CM), 400 (CD), 0-300 (0 to 3 C's), - # or 500-800 (D, followed by 0 to 3 C's) - (XC|XL|L?X{0,3}) # tens - 90 (XC), 40 (XL), 0-30 (0 to 3 X's), - # or 50-80 (L, followed by 0 to 3 X's) - (IX|IV|V?I{0,3}) # ones - 9 (IX), 4 (IV), 0-3 (0 to 3 I's), - # or 5-8 (V, followed by 0 to 3 I's) - $ # end of string - """ ,re.VERBOSE) - -def fromRoman(s): - """convert Roman numeral to integer""" - if not s: - raise InvalidRomanNumeralError('Input can not be blank') - if not romanNumeralPattern.search(s): - raise InvalidRomanNumeralError('Invalid Roman numeral: %s' % s) - - result = 0 - index = 0 - for numeral, integer in romanNumeralMap: - while s[index:index+len(numeral)] == numeral: - result += integer - index += len(numeral) - return result - diff --git a/style.css b/style.css deleted file mode 100644 index 064fe688c..000000000 --- a/style.css +++ /dev/null @@ -1,19 +0,0 @@ -body { margin: 0px; - padding: 0px; } -.navigation { width: 100%; - background: #99ccff; } -.navigation .navicon { width: 150px; - height: 35; } -.navigation .textlinks { padding-left: 1em; - text-align: left; } - -.header { margin-top: 0.5em; } -.header, .content { margin-left: 1em; - margin-right: 1em; } - -.header table td { text-align: left; } -.header table th { text-align: right; - font-family: sans-serif; - padding-right: 0.5em; } - -h3 { font-family: sans-serif; } diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..ac54601b6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +[tox] +requires = + tox>=4.2 +env_list = + py{312, 311, 310, 39} +no_package = true + +[testenv] +deps = + -rrequirements.txt +pass_env = + FORCE_COLOR +commands = + python -bb -X dev -W error -m pytest {posargs}