From 503b4f3cb9c1ed6973ed3db22054ac45a3446749 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 24 Aug 2022 01:04:09 +0100 Subject: [PATCH 1/5] Use supertype context for variable inference --- mypy/checker.py | 10 ++++++- mypy/semanal.py | 45 ---------------------------- test-data/unit/check-classes.test | 6 ++-- test-data/unit/check-inference.test | 25 ++++++++++++++++ test-data/unit/check-literal.test | 41 +++++++++++++++++++++++++ test-data/unit/check-newsemanal.test | 2 +- 6 files changed, 79 insertions(+), 50 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 076f9e3763d99..1bed630054477 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2686,7 +2686,15 @@ def check_assignment( self.check_indexed_assignment(index_lvalue, rvalue, lvalue) if inferred: - rvalue_type = self.expr_checker.accept(rvalue) + type_context = None + if inferred.info: + for base in inferred.info.mro[1:]: + base_type, base_node = self.lvalue_type_from_base(inferred, base) + if base_type and isinstance(base_node, Var) and not base_node.is_inferred: + # Use most derived supertype as type context if available. + type_context = base_type + break + rvalue_type = self.expr_checker.accept(rvalue, type_context=type_context) if not ( inferred.is_final or (isinstance(lvalue, NameExpr) and lvalue.name == "__match_args__") diff --git a/mypy/semanal.py b/mypy/semanal.py index 4f62d3010a3b0..350547df2df5e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -88,7 +88,6 @@ AwaitExpr, Block, BreakStmt, - BytesExpr, CallExpr, CastExpr, ClassDef, @@ -244,8 +243,6 @@ CallableType, FunctionLike, Instance, - LiteralType, - LiteralValue, NoneType, Overloaded, Parameters, @@ -2776,7 +2773,6 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: return True def analyze_lvalues(self, s: AssignmentStmt) -> None: - # We cannot use s.type, because analyze_simple_literal_type() will set it. explicit = s.unanalyzed_type is not None if self.is_final_type(s.unanalyzed_type): # We need to exclude bare Final. @@ -3006,10 +3002,6 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: and not self.is_func_scope() ): self.fail("All protocol members must have explicitly declared types", s) - # Set the type if the rvalue is a simple literal (even if the above error occurred). - if len(s.lvalues) == 1 and isinstance(s.lvalues[0], RefExpr): - if s.lvalues[0].is_inferred_def: - s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) if s.type: # Store type into nodes. for lvalue in s.lvalues: @@ -3024,43 +3016,6 @@ def is_annotated_protocol_member(self, s: AssignmentStmt) -> bool: for lv in s.lvalues ) - def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Type | None: - """Return builtins.int if rvalue is an int literal, etc. - - If this is a 'Final' context, we return "Literal[...]" instead.""" - if self.options.semantic_analysis_only or self.function_stack: - # Skip this if we're only doing the semantic analysis pass. - # This is mostly to avoid breaking unit tests. - # Also skip inside a function; this is to avoid confusing - # the code that handles dead code due to isinstance() - # inside type variables with value restrictions (like - # AnyStr). - return None - if isinstance(rvalue, FloatExpr): - return self.named_type_or_none("builtins.float") - - value: LiteralValue | None = None - type_name: str | None = None - if isinstance(rvalue, IntExpr): - value, type_name = rvalue.value, "builtins.int" - if isinstance(rvalue, StrExpr): - value, type_name = rvalue.value, "builtins.str" - if isinstance(rvalue, BytesExpr): - value, type_name = rvalue.value, "builtins.bytes" - - if type_name is not None: - assert value is not None - typ = self.named_type_or_none(type_name) - if typ and is_final: - return typ.copy_modified( - last_known_value=LiteralType( - value=value, fallback=typ, line=typ.line, column=typ.column - ) - ) - return typ - - return None - def analyze_alias( self, rvalue: Expression, allow_placeholder: bool = False ) -> tuple[Type | None, list[str], set[str], list[str]]: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 20a0c4ae80ea4..53f4d62803119 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -4317,7 +4317,7 @@ class C(B): x = object() [out] main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") -main:6: error: Incompatible types in assignment (expression has type "object", base class "B" defined the type as "str") +main:6: error: Incompatible types in assignment (expression has type "object", base class "A" defined the type as "int") [case testClassOneErrorPerLine] class A: @@ -4327,7 +4327,7 @@ class B(A): x = 1.0 [out] main:4: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") -main:5: error: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") +main:5: error: Incompatible types in assignment (expression has type "float", base class "A" defined the type as "int") [case testClassIgnoreType_RedefinedAttributeAndGrandparentAttributeTypesNotIgnored] class A: @@ -4335,7 +4335,7 @@ class A: class B(A): x = '' # type: ignore class C(B): - x = '' + x = '' # E: Incompatible types in assignment (expression has type "str", base class "A" defined the type as "int") [out] [case testClassIgnoreType_RedefinedAttributeTypeIgnoredInChildren] diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index fc6cb6fc456af..723a82fc76e2a 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -3263,3 +3263,28 @@ from typing import Dict, Iterable, Tuple, Union def foo(x: Union[Tuple[str, Dict[str, int], str], Iterable[object]]) -> None: ... foo(("a", {"a": "b"}, "b")) [builtins fixtures/dict.pyi] + +[case testUseSupertypeAsInferenceContext] +# flags: --strict-optional +from typing import List, Optional + +class B: + x: List[Optional[int]] + +class C(B): + x = [1] + +reveal_type(C().x) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartial] +from typing import List + +class A: + x: List[str] + +class B(A): + x = [] + +reveal_type(B().x) # N: Revealed type is "builtins.list[builtins.str]" +[builtins fixtures/list.pyi] diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index b6eae1da7d848..da8f1570a4f42 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -2918,3 +2918,44 @@ def incorrect_return2() -> Union[Tuple[Literal[True], int], Tuple[Literal[False] else: return (bool(), 'oops') # E: Incompatible return value type (got "Tuple[bool, str]", expected "Union[Tuple[Literal[True], int], Tuple[Literal[False], str]]") [builtins fixtures/bool.pyi] + +[case testLiteralSubtypeContext] +from typing_extensions import Literal + +class A: + foo: Literal['bar', 'spam'] +class B(A): + foo = 'spam' + +reveal_type(B().foo) # N: Revealed type is "Literal['spam']" +[builtins fixtures/tuple.pyi] + +[case testLiteralSubtypeContextNested] +from typing import List +from typing_extensions import Literal + +class A: + foo: List[Literal['bar', 'spam']] +class B(A): + foo = ['spam'] + +reveal_type(B().foo) # N: Revealed type is "builtins.list[Union[Literal['bar'], Literal['spam']]]" +[builtins fixtures/tuple.pyi] + +[case testLiteralSubtypeContextGeneric] +from typing_extensions import Literal +from typing import Generic, List, TypeVar + +T = TypeVar("T", bound=str) + +class B(Generic[T]): + collection: List[T] + word: T + +class C(B[Literal["word"]]): + collection = ["word"] + word = "word" + +reveal_type(C().collection) # N: Revealed type is "builtins.list[Literal['word']]" +reveal_type(C().word) # N: Revealed type is "Literal['word']" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index bf612f95b3a2e..8e53bd0a58ab9 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -2474,7 +2474,7 @@ class DesiredTarget: attr: int [case testNewAnalyzerFirstVarDefinitionWins] -x = y +x = y # E: Cannot determine type of "y" x = 1 # We want to check that the first definition creates the variable. From 00667a72d9ac0af7c9790f883aa500af020eb1a5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 24 Aug 2022 01:52:53 +0100 Subject: [PATCH 2/5] Fix some existing tests --- mypy/checker.py | 8 ++++++-- mypy/nodes.py | 5 +++++ test-data/unit/check-inference.test | 9 +++++++++ test-data/unit/merge.test | 12 +++++------- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 1bed630054477..d79fa82da2ffa 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2690,7 +2690,9 @@ def check_assignment( if inferred.info: for base in inferred.info.mro[1:]: base_type, base_node = self.lvalue_type_from_base(inferred, base) - if base_type and isinstance(base_node, Var) and not base_node.is_inferred: + if base_type and not ( + isinstance(base_node, Var) and base_node.invalid_partial_type + ): # Use most derived supertype as type context if available. type_context = base_type break @@ -5878,7 +5880,9 @@ def enter_partial_types( self.msg.need_annotation_for_var(var, context, self.options.python_version) self.partial_reported.add(var) if var.type: - var.type = self.fixup_partial_type(var.type) + fixed = self.fixup_partial_type(var.type) + var.invalid_partial_type = fixed != var.type + var.type = fixed def handle_partial_var_type( self, typ: PartialType, is_lvalue: bool, node: Var, context: Context diff --git a/mypy/nodes.py b/mypy/nodes.py index 2b32d5f4f25ce..4856ce3035e8e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -939,6 +939,7 @@ def deserialize(cls, data: JsonDict) -> Decorator: "explicit_self_type", "is_ready", "is_inferred", + "invalid_partial_type", "from_module_getattr", "has_explicit_value", "allow_incompatible_override", @@ -975,6 +976,7 @@ class Var(SymbolNode): "from_module_getattr", "has_explicit_value", "allow_incompatible_override", + "invalid_partial_type", ) def __init__(self, name: str, type: mypy.types.Type | None = None) -> None: @@ -1024,6 +1026,9 @@ def __init__(self, name: str, type: mypy.types.Type | None = None) -> None: self.has_explicit_value = False # If True, subclasses can override this with an incompatible type. self.allow_incompatible_override = False + # If True, this means we didn't manage to infer full type and fall back to + # something like list[Any]. We may decide to not use such types as context. + self.invalid_partial_type = False @property def name(self) -> str: diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 723a82fc76e2a..ccd47e3c97fe0 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -3288,3 +3288,12 @@ class B(A): reveal_type(B().x) # N: Revealed type is "builtins.list[builtins.str]" [builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartialError] +class A: + x = ['a', 'b'] + +class B(A): + x = [] + x.append(2) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "str" +[builtins fixtures/list.pyi] diff --git a/test-data/unit/merge.test b/test-data/unit/merge.test index a593a064cbb25..9b1aba7208bc7 100644 --- a/test-data/unit/merge.test +++ b/test-data/unit/merge.test @@ -594,19 +594,17 @@ MypyFile:1<0>( MypyFile:1<1>( tmp/target.py AssignmentStmt:1<2>( - NameExpr(x [target.x<3>]) - IntExpr(1) - builtins.int<4>)) + NameExpr(x* [target.x<3>]) + IntExpr(1))) ==> MypyFile:1<0>( tmp/main Import:1(target)) MypyFile:1<1>( tmp/target.py - AssignmentStmt:1<5>( - NameExpr(x [target.x<3>]) - IntExpr(2) - builtins.int<4>)) + AssignmentStmt:1<4>( + NameExpr(x* [target.x<3>]) + IntExpr(2))) [case testNestedClassMethod_typeinfo] import target From 067318fcd41aaaaf4dc4f4865d5c175b6efb6a7a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 24 Aug 2022 02:19:27 +0100 Subject: [PATCH 3/5] Try fixing match test case --- mypy/checker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index d79fa82da2ffa..0ddb607b76a93 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1624,6 +1624,8 @@ def check_slots_definition(self, typ: Type, context: Context) -> None: def check_match_args(self, var: Var, typ: Type, context: Context) -> None: """Check that __match_args__ contains literal strings""" + if not self.scope.active_class(): + return typ = get_proper_type(typ) if not isinstance(typ, TupleType) or not all( [is_string_literal(item) for item in typ.items] From 3bcdcf2e6e1596a6c10a6a1c21c723ac4c8a0ffa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 24 Aug 2022 11:21:14 +0100 Subject: [PATCH 4/5] Put back special-casing in semanal.py --- mypy/semanal.py | 47 ++++++++++++++++++++++++++++ test-data/unit/check-newsemanal.test | 2 +- test-data/unit/merge.test | 12 ++++--- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 350547df2df5e..2946880b783e9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -88,6 +88,7 @@ AwaitExpr, Block, BreakStmt, + BytesExpr, CallExpr, CastExpr, ClassDef, @@ -243,6 +244,8 @@ CallableType, FunctionLike, Instance, + LiteralType, + LiteralValue, NoneType, Overloaded, Parameters, @@ -2773,6 +2776,7 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: return True def analyze_lvalues(self, s: AssignmentStmt) -> None: + # We cannot use s.type, because analyze_simple_literal_type() will set it. explicit = s.unanalyzed_type is not None if self.is_final_type(s.unanalyzed_type): # We need to exclude bare Final. @@ -3002,6 +3006,13 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: and not self.is_func_scope() ): self.fail("All protocol members must have explicitly declared types", s) + # Set the type if the rvalue is a simple literal (even if the above error occurred). + # We skip this step for type scope because it messes up with class attribute + # inference for literal types (also annotated and non-annotated variables at class + # scope are semantically different, so we should not souch statement type). + if len(s.lvalues) == 1 and isinstance(s.lvalues[0], RefExpr) and not self.type: + if s.lvalues[0].is_inferred_def: + s.type = self.analyze_simple_literal_type(s.rvalue, s.is_final_def) if s.type: # Store type into nodes. for lvalue in s.lvalues: @@ -3016,6 +3027,42 @@ def is_annotated_protocol_member(self, s: AssignmentStmt) -> bool: for lv in s.lvalues ) + def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Type | None: + """Return builtins.int if rvalue is an int literal, etc. + If this is a 'Final' context, we return "Literal[...]" instead.""" + if self.options.semantic_analysis_only or self.function_stack: + # Skip this if we're only doing the semantic analysis pass. + # This is mostly to avoid breaking unit tests. + # Also skip inside a function; this is to avoid confusing + # the code that handles dead code due to isinstance() + # inside type variables with value restrictions (like + # AnyStr). + return None + if isinstance(rvalue, FloatExpr): + return self.named_type_or_none("builtins.float") + + value: LiteralValue | None = None + type_name: str | None = None + if isinstance(rvalue, IntExpr): + value, type_name = rvalue.value, "builtins.int" + if isinstance(rvalue, StrExpr): + value, type_name = rvalue.value, "builtins.str" + if isinstance(rvalue, BytesExpr): + value, type_name = rvalue.value, "builtins.bytes" + + if type_name is not None: + assert value is not None + typ = self.named_type_or_none(type_name) + if typ and is_final: + return typ.copy_modified( + last_known_value=LiteralType( + value=value, fallback=typ, line=typ.line, column=typ.column + ) + ) + return typ + + return None + def analyze_alias( self, rvalue: Expression, allow_placeholder: bool = False ) -> tuple[Type | None, list[str], set[str], list[str]]: diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index 8e53bd0a58ab9..bf612f95b3a2e 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -2474,7 +2474,7 @@ class DesiredTarget: attr: int [case testNewAnalyzerFirstVarDefinitionWins] -x = y # E: Cannot determine type of "y" +x = y x = 1 # We want to check that the first definition creates the variable. diff --git a/test-data/unit/merge.test b/test-data/unit/merge.test index 9b1aba7208bc7..a593a064cbb25 100644 --- a/test-data/unit/merge.test +++ b/test-data/unit/merge.test @@ -594,17 +594,19 @@ MypyFile:1<0>( MypyFile:1<1>( tmp/target.py AssignmentStmt:1<2>( - NameExpr(x* [target.x<3>]) - IntExpr(1))) + NameExpr(x [target.x<3>]) + IntExpr(1) + builtins.int<4>)) ==> MypyFile:1<0>( tmp/main Import:1(target)) MypyFile:1<1>( tmp/target.py - AssignmentStmt:1<4>( - NameExpr(x* [target.x<3>]) - IntExpr(2))) + AssignmentStmt:1<5>( + NameExpr(x [target.x<3>]) + IntExpr(2) + builtins.int<4>)) [case testNestedClassMethod_typeinfo] import target From 01e6622995388250ed35e66179d90eaa29cad84f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 24 Aug 2022 11:59:36 +0100 Subject: [PATCH 5/5] More principled context selection; more tests --- mypy/checker.py | 32 ++++++++++++++++++++--------- test-data/unit/check-inference.test | 31 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 0ddb607b76a93..0498887acc877 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2688,16 +2688,7 @@ def check_assignment( self.check_indexed_assignment(index_lvalue, rvalue, lvalue) if inferred: - type_context = None - if inferred.info: - for base in inferred.info.mro[1:]: - base_type, base_node = self.lvalue_type_from_base(inferred, base) - if base_type and not ( - isinstance(base_node, Var) and base_node.invalid_partial_type - ): - # Use most derived supertype as type context if available. - type_context = base_type - break + type_context = self.get_variable_type_context(inferred) rvalue_type = self.expr_checker.accept(rvalue, type_context=type_context) if not ( inferred.is_final @@ -2710,6 +2701,27 @@ def check_assignment( # (type, operator) tuples for augmented assignments supported with partial types partial_type_augmented_ops: Final = {("builtins.list", "+"), ("builtins.set", "|")} + def get_variable_type_context(self, inferred: Var) -> Type | None: + type_contexts = [] + if inferred.info: + for base in inferred.info.mro[1:]: + base_type, base_node = self.lvalue_type_from_base(inferred, base) + if base_type and not ( + isinstance(base_node, Var) and base_node.invalid_partial_type + ): + type_contexts.append(base_type) + # Use most derived supertype as type context if available. + if not type_contexts: + return None + candidate = type_contexts[0] + for other in type_contexts: + if is_proper_subtype(other, candidate): + candidate = other + elif not is_subtype(candidate, other): + # Multiple incompatible candidates, cannot use any of them as context. + return None + return candidate + def try_infer_partial_generic_type_from_assignment( self, lvalue: Lvalue, rvalue: Expression, op: str ) -> None: diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index ccd47e3c97fe0..ffcd6d8d94ddb 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -3277,6 +3277,14 @@ class C(B): reveal_type(C().x) # N: Revealed type is "builtins.list[Union[builtins.int, None]]" [builtins fixtures/list.pyi] +[case testUseSupertypeAsInferenceContextInvalidType] +from typing import List +class P: + x: List[int] +class C(P): + x = ['a'] # E: List item 0 has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + [case testUseSupertypeAsInferenceContextPartial] from typing import List @@ -3297,3 +3305,26 @@ class B(A): x = [] x.append(2) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "str" [builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextPartialErrorProperty] +from typing import List + +class P: + @property + def x(self) -> List[int]: ... +class C(P): + x = [] + +C.x.append("no") # E: Argument 1 to "append" of "list" has incompatible type "str"; expected "int" +[builtins fixtures/list.pyi] + +[case testUseSupertypeAsInferenceContextConflict] +from typing import List +class P: + x: List[int] +class M: + x: List[str] +class C(P, M): + x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") +reveal_type(C.x) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/list.pyi]