From 49686bc4db0d9eab98843fbb33e2bae44a49c86a Mon Sep 17 00:00:00 2001 From: James Hilton-Balfe Date: Sun, 24 Jul 2022 17:07:39 +0100 Subject: [PATCH] PEP 696: Type defaults for TypeVars (#2717) Co-authored-by: Jelle Zijlstra Co-authored-by: Hugo van Kemenade --- .github/CODEOWNERS | 1 + pep-0696.rst | 442 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 443 insertions(+) create mode 100644 pep-0696.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index eb6b426fa..13b34db2b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -576,6 +576,7 @@ pep-0692.rst @jellezijlstra pep-0693.rst @Yhg1s pep-0694.rst @dstufft pep-0695.rst @gvanrossum +pep-0696.rst @jellezijlstra # ... # pep-0754.txt # ... diff --git a/pep-0696.rst b/pep-0696.rst new file mode 100644 index 000000000..06bb93172 --- /dev/null +++ b/pep-0696.rst @@ -0,0 +1,442 @@ +PEP: 696 +Title: Type defaults for TypeVarLikes +Author: James Hilton-Balfe +Sponsor: Jelle Zijlstra +Discussions-To: typing-sig@python.org +Status: Draft +Type: Standards Track +Topic: Typing +Content-Type: text/x-rst +Created: 14-Jul-2022 +Python-Version: 3.12 + +Abstract +-------- + +This PEP introduces the concept of type defaults for +``TypeVarLike``\ s (``TypeVar``, ``ParamSpec`` and ``TypeVarTuple``), +which act as defaults for a type parameter when none is specified. + +Default type argument support is available in some popular languages +such as C++, TypeScript, and Rust. A survey of type parameter syntax in +some common languages has been conducted by the author of :pep:`695` +and can be found in its +:pep:`Appendix A <695#appendix-a-survey-of-type-parameter-syntax>`. + + +Motivation +---------- + +.. code:: py + + T = TypeVar("T", default=int) # This means that if no type is specified T = int + + @dataclass + class Box(Generic[T]): + value: T | None = None + + reveal_type(Box()) # type is Box[int] + reveal_type(Box(value="Hello World!")) # type is Box[str] + +One place this `regularly comes +up `__ is ``Generator``. I +propose changing the *stub definition* to something like: + +.. code:: py + + YieldT = TypeVar("YieldT") + SendT = TypeVar("SendT", default=None) + ReturnT = TypeVar("ReturnT", default=None) + + class Generator(Generic[YieldT, SendT, ReturnT]): ... + + Generator[int] == Generator[int, None] == Generator[int, None, None] + +This is also useful for a ``Generic`` that is commonly over one type. + +.. code:: py + + class Bot: ... + + BotT = TypeVar("BotT", bound=Bot, default=Bot) + + class Context(Generic[BotT]): + bot: BotT + + class MyBot(Bot): ... + + reveal_type(Context().bot) # type is Bot # notice this is not Any which is what it would be currently + reveal_type(Context[MyBot]().bot) # type is MyBot + +Not only does this improve typing for those who explicitly use it, it +also helps non-typing users who rely on auto-complete to speed up their +development. + +This design pattern is common in projects like: + - `discord.py `__ - where the + example above was taken from. + - `NumPy `__ - the default for types + like ``ndarray``'s ``dtype`` would be ``float64``. Currently it's + ``Unknown`` or ``Any``. + - `TensorFlow `__ (this + could be used for Tensor similarly to ``numpy.ndarray`` and would be + useful to simplify the definition of ``Layer``). + + +Specification +------------- + +Default ordering and subscription rules +''''''''''''''''''''''''''''''''''''''' + +The order for defaults should follow the standard function parameter +rules, so a ``TypeVarLike`` with no ``default`` cannot follow one with +a ``default`` value. Doing so should ideally raise a ``TypeError`` in +``typing._GenericAlias``/``types.GenericAlias``, and a type checker +should flag this an error. + +.. code:: py + + DefaultStrT = TypeVar("DefaultStrT", default=str) + DefaultIntT = TypeVar("DefaultIntT", default=int) + DefaultBoolT = TypeVar("DefaultBoolT", default=bool) + T = TypeVar("T") + T2 = TypeVar("T2") + + class NonDefaultFollowsDefault(Generic[DefaultStrT, T]): ... # Invalid: non-default TypeVars cannot follow ones with defaults + + + class NoNonDefaults(Generic[DefaultStrT, DefaultIntT]): ... + + ( + NoNoneDefaults == + NoNoneDefaults[str] == + NoNoneDefaults[str, int] + ) # All valid + + + class OneDefault(Generic[T, DefaultBoolT]): ... + + OneDefault[float] == OneDefault[float, bool] # Valid + + + class AllTheDefaults(Generic[T1, T2, DefaultStrT, DefaultIntT, DefaultBoolT]): ... + + AllTheDefaults[int] # Invalid: expected 2 arguments to AllTheDefaults + ( + AllTheDefaults[int, complex] == + AllTheDefaults[int, complex, str] == + AllTheDefaults[int, complex, str, int] == + AllTheDefaults[int, complex, str, int, bool] + ) # All valid + +This cannot be enforced at runtime for functions, for now, but in the +future, this might be possible (see `Interaction with PEP +695 <#interaction-with-pep-695>`__). + +``ParamSpec`` Defaults +'''''''''''''''''''''' + +``ParamSpec`` defaults are defined using the same syntax as +``TypeVar`` \ s but use a ``list`` or ``tuple`` of types or an ellipsis +literal "``...``". + +.. code:: py + + DefaultP = ParamSpec("DefaultP", default=(str, int)) + + class Foo(Generic[DefaultP]): ... + + reveal_type(Foo()) # type is Foo[(str, int)] + reveal_type(Foo[(bool, bool)]()) # type is Foo[(bool, bool)] + +``TypeVarTuple`` Defaults +''''''''''''''''''''''''' + +``TypeVarTuple`` defaults are defined using the same syntax as +``TypeVar`` \ s but use an unpacked tuple of types instead of a single type. + +.. code:: py + + DefaultTs = TypeVarTuple("DefaultTs", default=Unpack[tuple[str, int]]) + + class Foo(Generic[DefaultTs]): ... + + reveal_type(Foo()) # type is Foo[str, int] + reveal_type(Foo[int, bool]()) # type is Foo[int, bool] + +Using another ``TypeVarLike`` as the default +'''''''''''''''''''''''''''''''''''''''''''' + +To use another ``TypeVarLike``\ s as the default they have to be of the +same type. When using another ``TypeVarLike`` (T1) as the default, the default +for the ``TypeVarLike`` (T2), T1 must be used before in the signature +of the class it appears in before T2. T2's bound must be a subtype of +T1's bound. + +`This could be used on builtins.slice `__ +where the ``start`` parameter should default to ``int``, ``stop`` +default to the type of ``start`` and step default to ``int | None``. + +.. code:: py + + StartT = TypeVar("StartT", default=int) + StopT = TypeVar("StopT", default=StartT) + StepT = TypeVar("StepT", default=int | None) + + class slice(Generic[StartT, StopT, StepT]): ... # Valid + + reveal_type(slice()) # type is slice[int, int, int | None] + reveal_type(slice[str]()) # type is slice[str, str, int | None] + reveal_type(slice[str, str, timedelta]()) # type is slice[str, str, timedelta] + + StartT = TypeVar("StartT", default="StopT") + StopT = TypeVar("StopT", default=int) + class slice(Generic[StartT, StopT, StepT]): ... + ^^^^^^ # Invalid: ordering does not allow StopT to bound yet + +``Generic`` ``TypeAlias``\ es +''''''''''''''''''''''''''''' + +``Generic`` ``TypeAlias``\ es should be able to be further subscripted +following normal subscription rules. If a ``TypeVarLike`` has a default +that hasn't been overridden it should be treated like it was +substituted into the ``TypeAlias``. However, it can be specialised +further down the line. + +.. code:: py + + class SomethingWithNoDefaults(Generic[T, T2]): ... + + MyAlias: TypeAlias = SomethingWithNoDefaults[int, DefaultStrT] # valid + reveal_type(MyAlias()) # type is SomethingWithNoDefaults[int, str] + reveal_type(MyAlias[bool]()) # type is SomethingWithNoDefaults[int, bool] + + MyAlias[bool, int] # Invalid: too many arguments passed to MyAlias + +Subclassing +''''''''''' + +Subclasses of ``Generic``\ s with ``TypeVarLike``\ s that have defaults +behave similarly to ``Generic`` ``TypeAlias``\ es. + +.. code:: py + + class SubclassMe(Generic[T, DefaultStrT]): ... + + class Bar(SubclassMe[int, DefaultStrT]): ... + reveal_type(Bar()) # type is Bar[str] + reveal_type(Bar[bool]()) # type is Bar[bool] + + class Foo(SubclassMe[int]): ... + + reveal_type(Foo()) # type is + + Foo[str] # Invalid: Foo cannot be further subscripted + + class Baz(Generic[DefaultIntT, DefaultStrT]): ... + + class Spam(Baz): ... + reveal_type(Spam()) # type is + +Using bound and default +''''''''''''''''''''''' + +If both ``bound`` and ``default`` are passed ``default`` must be a +subtype of ``bound``. Otherwise the type checker should generate an +error. + +.. code:: py + + TypeVar("Ok", bound=float, default=int) # Valid + TypeVar("Invalid", bound=str, default=int) # Invalid: the bound and default are incompatible + +Constraints +''''''''''' + +For constrained ``TypeVar``\ s, the default needs to be one of the +constraints. A type checker should generate an error even if it is a +subtype of one of the constraints. + +.. code:: py + + TypeVar("Ok", float, str, default=float) # Valid + TypeVar("Invalid", float, str, default=int) # Invalid: expected one of float or str got int + +Function Defaults +''''''''''''''''' + +``TypeVarLike``\ s currently can only be used where a parameter can go unsolved. + +.. code:: py + + def foo(a: DefaultStrT | None = None) -> DefaultStrT: ... + + reveal_type(foo(1)) # type is int + reveal_type(foo()) # type is str + +If they are used where the parameter type is known, the defaults +should just be ignored and a type checker can emit a warning. + + +Implementation +-------------- + +At runtime, this would involve the following changes to the ``typing`` +module. + +- a new class ``_DefaultMixin`` needs to be added which is a superclass + of ``TypeVar``, ``ParamSpec``, and ``TypeVarTuple``. + + - the type passed to default would be available as a ``__default__`` + attribute. + +The following changes would be required to both ``GenericAlias``\ es: + +- logic to determine the defaults required for a subscription. +- ideally, logic to determine if subscription (like + ``Generic[T, DefaultT]``) would be valid. + +A reference implementation of the type checker can be found at +https://github.com/Gobot1234/mypy/tree/TypeVar-defaults + + +Interaction with PEP 695 +------------------------ + +If this PEP is accepted, the syntax proposed in :pep:`695` will be +extended to introduce a way to specify defaults for type parameters +using the "=" operator inside of the square brackets like so: + +.. code:: py + + # TypeVars + class Foo[T = str]: ... + def bar[U = int](): ... + + # ParamSpecs + class Baz[**P = (int, str)]: ... + def spam[**Q = (bool,)](): ... + + # TypeVarTuples + class Qux[*Ts = *tuple[int, bool]]: ... + def ham[*Us = *tuple[str]](): ... + +This functionality was included in the initial draft of :pep:`695` but +was removed due to scope creep. + +Grammar Changes +''''''''''''''' + +:: + + type_param: + | a=NAME b=[type_param_bound] d=[type_param_default] + | a=NAME c=[type_param_constraint] d=[type_param_default] + | '*' a=NAME d=[type_param_default] + | '**' a=NAME d=[type_param_default] + + type_param_default: '=' e=expression + +This would mean that ``TypeVarLike``\ s with defaults proceeding those +with non-defaults can be checked at compile time. + + +Rejected Alternatives +--------------------- + +Allowing the ``TypeVarLike``\ s defaults to be passed to ``type.__new__``'s ``**kwargs`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +.. code:: py + + T = TypeVar("T") + + @dataclass + class Box(Generic[T], T=int): + value: T | None = None + +While this is much easier to read and follows a similar rationale to the +``TypeVar`` `unary +syntax `__, it would not be +backwards compatible as ``T`` might already be passed to a +metaclass/superclass or support classes that don't subclass ``Generic`` +at runtime. + +Ideally, if :pep:`637` wasn't rejected, the following would be acceptable: + +.. code:: py + + T = TypeVar("T") + + @dataclass + class Box(Generic[T = int]): + value: T | None = None + +Allowing non-defaults to follow defaults +'''''''''''''''''''''''''''''''''''''''' + +.. code:: py + + YieldT = TypeVar("YieldT", default=Any) + SendT = TypeVar("SendT", default=Any) + ReturnT = TypeVar("ReturnT") + + class Coroutine(Generic[YieldT, SendT, ReturnT]): ... + + Coroutine[int] == Coroutine[Any, Any, int] + +Allowing non-defaults to follow defaults would alleviate the issues with +returning types like ``Coroutine`` from functions where the most used +type argument is the last (the return). Allowing non-defaults to follow +defaults is too confusing and potentially ambiguous, even if only the +above two forms were valid. Changing the argument order now would also +break a lot of codebases. This is also solvable in most cases using a +``TypeAlias``. + +.. code:: py + + Coro: TypeAlias = Coroutine[Any, Any, T] + Coro[int] == Coroutine[Any, Any, int] + +Having ``default`` implicitly be ``bound`` +'''''''''''''''''''''''''''''''''''''''''' + +In an earlier version of this PEP, the ``default`` was implicitly set +to ``bound`` if no value was passed for ``default``. This while +convenient, could have a ``TypeVarLike`` with no default follow a +``TypeVarLike`` with a default. Consider: + +.. code:: py + + T = TypeVar("T", bound=int) # default is implicitly int + U = TypeVar("U") + + class Foo(Generic[T, U]): + ... + + # would expand to + + T = TypeVar("T", bound=int, default=int) + U = TypeVar("U") + + class Foo(Generic[T, U]): + ... + +This would have also been a breaking change for a small number of cases +where the code relied on ``Any`` being the implicit default. + + +Acknowledgements +---------------- + +Thanks to the following people for their feedback on the PEP: + +Eric Traut, Jelle Zijlstra, Joshua Butt, Danny Yamamoto, Kaylynn Morgan +and Jakub Kuczys + + +Copyright +--------- +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive.