Fix type variable defaults depending on previous variables (#21526)

Fixes #19192
Fixes #20027
Fixes few TODOs in tests (don't know if there are relevant issues)
Closes #19382

Core idea is simple: don't do bizarre things :-) More precisely, instead
of various in-place modifications (which is a big no-no for types),
simply expand defaults iteratively as we build the environment for the
type. Couple additional things:
* Update defaults in `freshen_function_type_vars()` this is required for
consistency when updating `TypeVarId`s.
* Use `fix_instance()` in `checkexpr.py` for type applications. This
will apply _all_ type arguments in cases like `x = Foo[int](...)`. The
PEP and the spec say this is how it should be.

Author: ilevkivskyi

mypy/applytype.py

@@ -37,10 +37,12 @@ def get_target_type(
     report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None],
     context: Context,
     skip_unsatisfied: bool,
+    id_to_type: dict[TypeVarId, Type],
 ) -> Type | None:
     p_type = get_proper_type(type)
     if isinstance(p_type, UninhabitedType) and p_type.ambiguous and tvar.has_default():
-        return tvar.default
+        # Gradually expand defaults, as they may depend on previous type variables.
+        return expand_type(tvar.default, id_to_type)
     if isinstance(tvar, ParamSpecType):
         return type
     if isinstance(tvar, TypeVarTupleType):
@@ -113,7 +115,13 @@ def apply_generic_arguments(
             continue
 
         target_type = get_target_type(
-            tvar, type, callable, report_incompatible_typevar_value, context, skip_unsatisfied
+            tvar,
+            type,
+            callable,
+            report_incompatible_typevar_value,
+            context,
+            skip_unsatisfied,
+            id_to_type,
         )
         if target_type is not None:
             id_to_type[tvar.id] = target_type

mypy/checkexpr.py

@@ -5079,11 +5079,7 @@ class C(Generic[T, Unpack[Ts]]): ...
                     return [AnyType(TypeOfAny.from_error)] * len(vars)
 
         # TODO: in future we may want to support type application to variadic functions.
-        if (
-            not vars
-            or not any(isinstance(v, TypeVarTupleType) for v in vars)
-            or not t.is_type_obj()
-        ):
+        if not vars or not t.is_type_obj() or t.type_object().fullname == "builtins.tuple":
             return list(args)
         info = t.type_object()
         # We reuse the logic from semanal phase to reduce code duplication.
@@ -5097,6 +5093,9 @@ class C(Generic[T, Unpack[Ts]]): ...
             )
             args = list(fake.args)
 
+        if not any(isinstance(v, TypeVarTupleType) for v in vars):
+            return args
+
         prefix = next(i for (i, v) in enumerate(vars) if isinstance(v, TypeVarTupleType))
         suffix = len(vars) - prefix - 1
         tvt = vars[prefix]

mypy/expandtype.py

@@ -129,6 +129,9 @@ def freshen_function_type_vars(callee: F) -> F:
             tv = v.new_unification_variable(v)
             tvs.append(tv)
             tvmap[v.id] = tv
+            if tv.has_default():
+                # Point to fresh ids in case defaults depend on previous variables.
+                tv.default = expand_type(tv.default, tvmap)
         fresh = expand_type(callee, tvmap).copy_modified(variables=tvs)
         return cast(F, fresh)
     else:
@@ -182,7 +185,6 @@ class ExpandTypeVisitor(TrivialSyntheticTypeTranslator):
     def __init__(self, variables: Mapping[TypeVarId, Type]) -> None:
         super().__init__()
         self.variables = variables
-        self.recursive_tvar_guard: dict[TypeVarId, Type | None] | None = None
 
     def visit_unbound_type(self, t: UnboundType) -> Type:
         return t
@@ -245,16 +247,6 @@ def visit_type_var(self, t: TypeVarType) -> Type:
             # TODO: do we really need to do this?
             # If I try to remove this special-casing ~40 tests fail on reveal_type().
             return repl.copy_modified(last_known_value=None)
-        if isinstance(repl, TypeVarType) and repl.has_default():
-            if self.recursive_tvar_guard is None:
-                self.recursive_tvar_guard = {}
-            if (tvar_id := repl.id) in self.recursive_tvar_guard:
-                return self.recursive_tvar_guard[tvar_id] or repl
-            self.recursive_tvar_guard[tvar_id] = None
-            repl.default = repl.default.accept(self)
-            expanded = repl.accept(self)  # Note: `expanded is repl` may be true.
-            repl = repl if isinstance(expanded, TypeVarType) else expanded
-            self.recursive_tvar_guard[tvar_id] = repl
         return repl
 
     def visit_param_spec(self, t: ParamSpecType) -> Type:

mypy/semanal.py

@@ -2466,9 +2466,6 @@ def tvar_defs_from_tvars(
         tvar_defs: list[TypeVarLikeType] = []
         last_tvar_name_with_default: str | None = None
         for name, tvar_expr in tvars:
-            tvar_expr.default = tvar_expr.default.accept(
-                TypeVarDefaultTranslator(self, tvar_expr.name, context)
-            )
             # PEP-695 type variables that are redeclared in an inner scope are warned
             # about elsewhere.
             if not tvar_expr.is_new_style and not self.tvar_scope.allow_binding(
@@ -2478,6 +2475,11 @@ def tvar_defs_from_tvars(
                     message_registry.TYPE_VAR_REDECLARED_IN_NESTED_CLASS.format(name), context
                 )
             tvar_def = self.tvar_scope.bind_new(name, tvar_expr, self.fail, context)
+            # Fix any residual UnboundTypes in the generated TypeVarLike, keep
+            # TypeVarLikeExpr untouched as it may be shared by multiple classes.
+            tvar_def.default = tvar_def.default.accept(
+                TypeVarDefaultTranslator(self, tvar_expr.name, context)
+            )
             if last_tvar_name_with_default is not None and not tvar_def.has_default():
                 self.msg.tvar_without_default_type(
                     tvar_def.name, last_tvar_name_with_default, context

mypy/tvar_scope.py

@@ -12,13 +12,8 @@
     TypeVarTupleExpr,
 )
 from mypy.types import (
-    AnyType,
     ParamSpecFlavor,
     ParamSpecType,
-    TrivialSyntheticTypeTranslator,
-    Type,
-    TypeAliasType,
-    TypeOfAny,
     TypeVarId,
     TypeVarLikeType,
     TypeVarTupleType,
@@ -28,54 +23,6 @@
 FailFunc: _TypeAlias = Callable[[str, Context], None]
 
 
-class TypeVarLikeDefaultFixer(TrivialSyntheticTypeTranslator):
-    """Set namespace for all TypeVarLikeTypes types."""
-
-    def __init__(
-        self,
-        scope: TypeVarLikeScope,
-        fail_func: FailFunc,
-        source_tv: TypeVarLikeExpr,
-        context: Context,
-    ) -> None:
-        self.scope = scope
-        self.fail_func = fail_func
-        self.source_tv = source_tv
-        self.context = context
-        super().__init__()
-
-    def visit_type_var(self, t: TypeVarType) -> Type:
-        existing = self.scope.get_binding(t.fullname)
-        if existing is None:
-            self._report_unbound_tvar(t)
-            return AnyType(TypeOfAny.from_error)
-        return existing
-
-    def visit_param_spec(self, t: ParamSpecType) -> Type:
-        existing = self.scope.get_binding(t.fullname)
-        if existing is None:
-            self._report_unbound_tvar(t)
-            return AnyType(TypeOfAny.from_error)
-        return existing
-
-    def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type:
-        existing = self.scope.get_binding(t.fullname)
-        if existing is None:
-            self._report_unbound_tvar(t)
-            return AnyType(TypeOfAny.from_error)
-        return existing
-
-    def visit_type_alias_type(self, t: TypeAliasType) -> Type:
-        return t
-
-    def _report_unbound_tvar(self, tvar: TypeVarLikeType) -> None:
-        self.fail_func(
-            f"Type variable {tvar.name} referenced in the default"
-            f" of {self.source_tv.name} is unbound",
-            self.context,
-        )
-
-
 class TypeVarLikeScope:
     """Scope that holds bindings for type variables and parameter specifications.
 
@@ -148,23 +95,14 @@ def bind_new(
             i = self.func_id
         namespace = self.namespace
 
-        # Defaults may reference other type variables. That is only valid when the
-        # referenced variable is already in scope (textually precedes the definition we're
-        # processing now).
-        default = tvar_expr.default.accept(
-            TypeVarLikeDefaultFixer(
-                self, fail_func=fail_func, source_tv=tvar_expr, context=context
-            )
-        )
-
         if isinstance(tvar_expr, TypeVarExpr):
             tvar_def: TypeVarLikeType = TypeVarType(
                 name=name,
                 fullname=tvar_expr.fullname,
                 id=TypeVarId(i, namespace=namespace),
                 values=tvar_expr.values,
                 upper_bound=tvar_expr.upper_bound,
-                default=default,
+                default=tvar_expr.default,
                 variance=tvar_expr.variance,
                 line=tvar_expr.line,
                 column=tvar_expr.column,
@@ -176,7 +114,7 @@ def bind_new(
                 id=TypeVarId(i, namespace=namespace),
                 flavor=ParamSpecFlavor.BARE,
                 upper_bound=tvar_expr.upper_bound,
-                default=default,
+                default=tvar_expr.default,
                 line=tvar_expr.line,
                 column=tvar_expr.column,
             )
@@ -187,7 +125,7 @@ def bind_new(
                 id=TypeVarId(i, namespace=namespace),
                 upper_bound=tvar_expr.upper_bound,
                 tuple_fallback=tvar_expr.tuple_fallback,
-                default=default,
+                default=tvar_expr.default,
                 line=tvar_expr.line,
                 column=tvar_expr.column,
             )

mypy/typeanal.py

@@ -2098,15 +2098,16 @@ def fix_instance(
                         disallow_any, fail, note, t, options, fullname, unexpanded_type
                     )
                 arg = any_type
+            with state.strict_optional_set(options.strict_optional):
+                # Gradually expand defaults, as they may depend on previous variables.
+                if tv.has_default():
+                    arg = expand_type(arg, env)
+                env[tv.id] = arg
             args.append(arg)
-        env[tv.id] = arg
+        else:
+            env[tv.id] = arg
     t.args = tuple(args)
     fix_type_var_tuple_argument(t)
-    if not t.type.has_type_var_tuple_type:
-        with state.strict_optional_set(True):
-            fixed = expand_type(t, env)
-        assert isinstance(fixed, Instance)
-        t.args = fixed.args
 
 
 def instantiate_type_alias(
@@ -2288,7 +2289,6 @@ def set_any_tvars(
 
     env: dict[TypeVarId, Type] = {}
     used_any_type = False
-    has_type_var_tuple_type = False
     for tv, arg in itertools.zip_longest(node.alias_tvars, args, fillvalue=None):
         if tv is None:
             continue
@@ -2300,16 +2300,16 @@ def set_any_tvars(
                 used_any_type = True
             if isinstance(tv, TypeVarTupleType):
                 # TODO Handle TypeVarTuple defaults
-                has_type_var_tuple_type = True
                 arg = UnpackType(Instance(tv.tuple_fallback.type, [any_type]))
+            with state.strict_optional_set(options.strict_optional):
+                # Gradually expand defaults, as they may depend on previous variables.
+                if tv.has_default():
+                    arg = expand_type(arg, env)
+                env[tv.id] = arg
             args.append(arg)
-        env[tv.id] = arg
+        else:
+            env[tv.id] = arg
     t = TypeAliasType(node, args, newline, newcolumn)
-    if not has_type_var_tuple_type:
-        with state.strict_optional_set(options.strict_optional):
-            fixed = expand_type(t, env)
-        assert isinstance(fixed, TypeAliasType)
-        t.args = fixed.args
 
     if used_any_type and disallow_any and node.alias_tvars:
         assert fail is not None

test-data/unit/check-python313.test

@@ -291,7 +291,7 @@ reveal_type(A2().x)  # N: Revealed type is "builtins.int"
 reveal_type(A3().x)  # N: Revealed type is "builtins.int"
 [builtins fixtures/tuple.pyi]
 
-[case testTypeVarDefaultToAnotherTypeVar]
+[case testTypeVarDefaultToAnotherTypeVarClass]
 class A[X, Y = X, Z = Y]:
     x: X
     y: Y
@@ -300,8 +300,7 @@ class A[X, Y = X, Z = Y]:
 a1: A[int]
 reveal_type(a1.x)  # N: Revealed type is "builtins.int"
 reveal_type(a1.y)  # N: Revealed type is "builtins.int"
-# TODO: this must reveal `int` as well:
-reveal_type(a1.z)  # N: Revealed type is "X`1"
+reveal_type(a1.z)  # N: Revealed type is "builtins.int"
 
 a2: A[int, str]
 reveal_type(a2.x)  # N: Revealed type is "builtins.int"
@@ -314,6 +313,35 @@ reveal_type(a3.y)  # N: Revealed type is "builtins.str"
 reveal_type(a3.z)  # N: Revealed type is "builtins.bool"
 [builtins fixtures/tuple.pyi]
 
+[case testTypeVarDefaultToAnotherTypeVarAlias]
+type A[X, Y = X, Z = Y] = tuple[X, Y, Z]
+
+a1: A[int]
+reveal_type(a1[0])  # N: Revealed type is "builtins.int"
+reveal_type(a1[1])  # N: Revealed type is "builtins.int"
+reveal_type(a1[2])  # N: Revealed type is "builtins.int"
+
+a2: A[int, str]
+reveal_type(a2[0])  # N: Revealed type is "builtins.int"
+reveal_type(a2[1])  # N: Revealed type is "builtins.str"
+reveal_type(a2[2])  # N: Revealed type is "builtins.str"
+
+a3: A[int, str, bool]
+reveal_type(a3[0])  # N: Revealed type is "builtins.int"
+reveal_type(a3[1])  # N: Revealed type is "builtins.str"
+reveal_type(a3[2])  # N: Revealed type is "builtins.bool"
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarDefaultAssertTypeType]
+from typing import assert_type
+
+class NoNonDefaults[T = str, S = int]: ...
+
+assert_type(NoNonDefaults[()], type[NoNonDefaults[str, int]])
+assert_type(NoNonDefaults[str], type[NoNonDefaults[str, int]])
+assert_type(NoNonDefaults[str, int], type[NoNonDefaults[str, int]])
+[builtins fixtures/tuple.pyi]
+
 [case testTypeVarDefaultToAnotherTypeVarWrong]
 class A[Y = X, X = int]: ...  # E: Name "X" is not defined
 

test-data/unit/check-typevar-defaults.test

@@ -137,8 +137,7 @@ def func_error_alias1(
     reveal_type(b)  # N: Revealed type is "builtins.dict[builtins.int, Any]"
     reveal_type(c)  # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"
 
-TERR2 = Dict[T4, T3]  # TODO should be an error  \
-                      # Type parameter "T4" has a default type that refers to one or more type variables that are out of scope
+TERR2 = Dict[T4, T3]  # E: Type parameter "T4" has a default type that refers to one or more type variables that are out of scope
 
 def func_error_alias2(
     a: TERR2,
@@ -518,8 +517,8 @@ def func_d3(
     reveal_type(b)  # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
     reveal_type(c)  # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]"
 
-    # k = ClassD3()
-    # reveal_type(k)  # Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]"  # TODO
+    k = ClassD3()
+    reveal_type(k)  # N: Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]"
     l = ClassD3[int]()
     reveal_type(l)  # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
     m = ClassD3[int, float]()
@@ -904,6 +903,19 @@ Alias: TypeAlias = "MyClass[T1, T2]"
 class MyClass(Generic["T1", "T2"]): ...
 [builtins fixtures/tuple.pyi]
 
+[case testTypeVariableDefaultNotSticky]
+from typing import Generic, TypeVar
+
+T1 = TypeVar("T1")
+T2 = TypeVar("T2", default=T1)
+
+class MyClass(Generic[T1, T2]): ...
+
+reveal_type(MyClass[int])  # N: Revealed type is "def () -> __main__.MyClass[builtins.int, builtins.int]"
+reveal_type(MyClass[str])  # N: Revealed type is "def () -> __main__.MyClass[builtins.str, builtins.str]"
+reveal_type(MyClass[bytes])  # N: Revealed type is "def () -> __main__.MyClass[builtins.bytes, builtins.bytes]"
+[builtins fixtures/tuple.pyi]
+
 [case testDefaultsApplicationInAliasNoCrashNested]
 from typing import Generic, TypeVar
 from typing_extensions import TypeAlias
@@ -928,19 +940,23 @@ T3 = TypeVar("T3", default=T2)
 class A(Generic[T1, T2, T3]): ...
 reveal_type(A)  # N: Revealed type is "def [T1, T2 = T1`1, T3 = T2`2 = T1`1] () -> __main__.A[T1`1, T2`2 = T1`1, T3`3 = T2`2 = T1`1]"
 a: A[int]
-reveal_type(a)  # N: Revealed type is "__main__.A[builtins.int, builtins.int, T1`1]"
+reveal_type(a)  # N: Revealed type is "__main__.A[builtins.int, builtins.int, builtins.int]"
+
+AA = tuple[T1, T2, T3]
+aa: AA[str]
+reveal_type(aa)  # N: Revealed type is "tuple[builtins.str, builtins.str, builtins.str]"
 
-class B(Generic[T1, T3]): ...  # E: Type variable T2 referenced in the default of T3 is unbound
+class B(Generic[T1, T3]): ...  # E: Type parameter "T3" has a default type that refers to one or more type variables that are out of scope
 reveal_type(B)  # N: Revealed type is "def [T1, T3 = Any] () -> __main__.B[T1`1, T3`2 = Any]"
 b: B[int]
 reveal_type(b)  # N: Revealed type is "__main__.B[builtins.int, Any]"
 
-class C(Generic[T2]): ...  # E: Type variable T1 referenced in the default of T2 is unbound
+class C(Generic[T2]): ...  # E: Type parameter "T2" has a default type that refers to one or more type variables that are out of scope
 reveal_type(C)  # N: Revealed type is "def [T2 = Any] () -> __main__.C[T2`1 = Any]"
 c: C
 reveal_type(c)  # N: Revealed type is "__main__.C[Any]"
 
-class D(Generic[T2, T1]): ...  # E: Type variable T1 referenced in the default of T2 is unbound \
+class D(Generic[T2, T1]): ...  # E: Type parameter "T2" has a default type that refers to one or more type variables that are out of scope \
                                # E: "T1" cannot appear after "T2" in type parameter list because it has no default type
 reveal_type(D)  # N: Revealed type is "def [T2 = Any, T1 = Any] () -> __main__.D[T2`1 = Any, T1`2 = Any]"
 d: D