Fix dmypy inspect for namespace packages (#16357)

Fixes #15781

The fix is to switch to already resolved paths instead of relying on
`crawl_up()`. This should be more robust w.r.t. various special cases. I
also tweak the tests slightly to show full file names, to have a more
consistent output.

Author: ilevkivskyi

mypy/dmypy_server.py

@@ -461,6 +461,7 @@ def initialize_fine_grained(
         messages = result.errors
         self.fine_grained_manager = FineGrainedBuildManager(result)
 
+        original_sources_len = len(sources)
         if self.following_imports():
             sources = find_all_sources_in_build(self.fine_grained_manager.graph, sources)
             self.update_sources(sources)
@@ -525,7 +526,8 @@ def initialize_fine_grained(
 
         __, n_notes, __ = count_stats(messages)
         status = 1 if messages and n_notes < len(messages) else 0
-        messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width)
+        # We use explicit sources length to match the logic in non-incremental mode.
+        messages = self.pretty_messages(messages, original_sources_len, is_tty, terminal_width)
         return {"out": "".join(s + "\n" for s in messages), "err": "", "status": status}
 
     def fine_grained_increment(

mypy/inspections.py

@@ -6,7 +6,6 @@
 from typing import Callable
 
 from mypy.build import State
-from mypy.find_sources import InvalidSourceList, SourceFinder
 from mypy.messages import format_type
 from mypy.modulefinder import PYTHON_EXTENSIONS
 from mypy.nodes import (
@@ -206,9 +205,6 @@ def __init__(
         force_reload: bool = False,
     ) -> None:
         self.fg_manager = fg_manager
-        self.finder = SourceFinder(
-            self.fg_manager.manager.fscache, self.fg_manager.manager.options
-        )
         self.verbosity = verbosity
         self.limit = limit
         self.include_span = include_span
@@ -561,16 +557,14 @@ def find_module(self, file: str) -> tuple[State | None, dict[str, object]]:
         if not any(file.endswith(ext) for ext in PYTHON_EXTENSIONS):
             return None, {"error": "Source file is not a Python file"}
 
-        try:
-            module, _ = self.finder.crawl_up(os.path.normpath(file))
-        except InvalidSourceList:
-            return None, {"error": "Invalid source file name: " + file}
-
-        state = self.fg_manager.graph.get(module)
+        # We are using a bit slower but robust way to find a module by path,
+        # to be sure that namespace packages are handled properly.
+        abs_path = os.path.abspath(file)
+        state = next((s for s in self.fg_manager.graph.values() if s.abspath == abs_path), None)
         self.module = state
         return (
             state,
-            {"out": f"Unknown module: {module}", "err": "", "status": 1} if state is None else {},
+            {"out": f"Unknown module: {file}", "err": "", "status": 1} if state is None else {},
         )
 
     def run_inspection(

mypy/test/testfinegrained.py

@@ -352,7 +352,7 @@ def maybe_inspect(self, step: int, server: Server, src: str) -> list[str]:
             )
             val = res["error"] if "error" in res else res["out"] + res["err"]
             output.extend(val.strip().split("\n"))
-        return normalize_messages(output)
+        return output
 
     def get_suggest(self, program_text: str, incremental_step: int) -> list[tuple[str, str]]:
         step_bit = "1?" if incremental_step == 1 else str(incremental_step)

test-data/unit/daemon.test

@@ -379,7 +379,7 @@ $ dmypy inspect foo.pyc:1:1:2:2
 Source file is not a Python file
 == Return code: 2
 $ dmypy inspect bar/baz.py:1:1:2:2
-Unknown module: baz
+Unknown module: bar/baz.py
 == Return code: 1
 $ dmypy inspect foo.py:3:1:1:1
 "end_line" must not be before "line"
@@ -434,7 +434,7 @@ $ dmypy inspect foo.pyc:1:2
 Source file is not a Python file
 == Return code: 2
 $ dmypy inspect bar/baz.py:1:2
-Unknown module: baz
+Unknown module: bar/baz.py
 == Return code: 1
 $ dmypy inspect foo.py:7:5 --include-span
 7:5:7:5 -> "int"
@@ -571,3 +571,17 @@ class A:
     x: int
 class B:
     x: int
+
+[case testDaemonInspectSelectCorrectFile]
+$ dmypy run test.py --export-types
+Daemon started
+Success: no issues found in 1 source file
+$ dmypy inspect demo/test.py:1:1
+"int"
+$ dmypy inspect test.py:1:1
+"str"
+[file test.py]
+b: str
+from demo.test import a
+[file demo/test.py]
+a: int

test-data/unit/fine-grained-inspect.test

@@ -1,8 +1,8 @@
 [case testInspectTypeBasic]
-# inspect2: --include-kind foo.py:10:13
-# inspect2: --show=type --include-kind foo.py:10:13
-# inspect2: --include-span -vv foo.py:12:5
-# inspect2: --include-span --include-kind foo.py:12:5:12:9
+# inspect2: --include-kind tmp/foo.py:10:13
+# inspect2: --show=type --include-kind tmp/foo.py:10:13
+# inspect2: --include-span -vv tmp/foo.py:12:5
+# inspect2: --include-span --include-kind tmp/foo.py:12:5:12:9
 import foo
 [file foo.py]
 from typing import TypeVar, Generic
@@ -29,10 +29,10 @@ MemberExpr -> "T"
 CallExpr:12:5:12:9 -> "C[int]"
 
 [case testInspectAttrsBasic]
-# inspect2: --show=attrs foo.py:6:1
-# inspect2: --show=attrs foo.py:7:1
-# inspect2: --show=attrs foo.py:10:1
-# inspect2: --show=attrs --include-object-attrs  foo.py:10:1
+# inspect2: --show=attrs tmp/foo.py:6:1
+# inspect2: --show=attrs tmp/foo.py:7:1
+# inspect2: --show=attrs tmp/foo.py:10:1
+# inspect2: --show=attrs --include-object-attrs  tmp/foo.py:10:1
 import foo
 [file foo.py]
 from bar import Meta
@@ -56,12 +56,12 @@ class Meta(type):
 {"function": ["__name__"], "object": ["__init__"]}
 
 [case testInspectDefBasic]
-# inspect2: --show=definition foo.py:5:5
-# inspect2: --show=definition --include-kind foo.py:6:3
-# inspect2: --show=definition --include-span foo.py:7:5
-# inspect2: --show=definition foo.py:8:1:8:4
-# inspect2: --show=definition foo.py:8:6:8:8
-# inspect2: --show=definition foo.py:9:3
+# inspect2: --show=definition tmp/foo.py:5:5
+# inspect2: --show=definition --include-kind tmp/foo.py:6:3
+# inspect2: --show=definition --include-span tmp/foo.py:7:5
+# inspect2: --show=definition tmp/foo.py:8:1:8:4
+# inspect2: --show=definition tmp/foo.py:8:6:8:8
+# inspect2: --show=definition tmp/foo.py:9:3
 import foo
 [file foo.py]
 from bar import var, test, A
@@ -95,18 +95,18 @@ def foo(x: Union[int, str]) -> None:
 [builtins fixtures/classmethod.pyi]
 [out]
 ==
-bar.py:4:0:meth
+tmp/bar.py:4:0:meth
 MemberExpr -> tmp/bar.py:2:5:x
 7:1:7:5 -> tmp/bar.py:6:9:y
-bar.py:9:1:test
-bar.py:8:1:var
-baz.py:3:2:foo
+tmp/bar.py:9:1:test
+tmp/bar.py:8:1:var
+tmp/baz.py:3:2:foo
 
 [case testInspectFallbackAttributes]
-# inspect2: --show=attrs --include-object-attrs foo.py:5:1
-# inspect2: --show=attrs foo.py:8:1
-# inspect2: --show=attrs --include-kind foo.py:10:1
-# inspect2: --show=attrs --include-kind --include-object-attrs foo.py:10:1
+# inspect2: --show=attrs --include-object-attrs tmp/foo.py:5:1
+# inspect2: --show=attrs tmp/foo.py:8:1
+# inspect2: --show=attrs --include-kind tmp/foo.py:10:1
+# inspect2: --show=attrs --include-kind --include-object-attrs tmp/foo.py:10:1
 import foo
 [file foo.py]
 class B: ...
@@ -128,7 +128,7 @@ NameExpr -> {}
 NameExpr -> {"object": ["__eq__", "__init__", "__ne__"]}
 
 [case testInspectTypeVarBoundAttrs]
-# inspect2: --show=attrs foo.py:8:13
+# inspect2: --show=attrs tmp/foo.py:8:13
 import foo
 [file foo.py]
 from typing import TypeVar
@@ -144,10 +144,10 @@ def foo(arg: T) -> T:
 {"C": ["x"]}
 
 [case testInspectTypeVarValuesAttrs]
-# inspect2: --show=attrs --force-reload foo.py:13:13
-# inspect2: --show=attrs --force-reload --union-attrs foo.py:13:13
-# inspect2: --show=attrs foo.py:16:5
-# inspect2: --show=attrs --union-attrs foo.py:16:5
+# inspect2: --show=attrs --force-reload tmp/foo.py:13:13
+# inspect2: --show=attrs --force-reload --union-attrs tmp/foo.py:13:13
+# inspect2: --show=attrs tmp/foo.py:16:5
+# inspect2: --show=attrs --union-attrs tmp/foo.py:16:5
 import foo
 [file foo.py]
 from typing import TypeVar, Generic
@@ -174,8 +174,8 @@ class C(Generic[T]):
 {"A": ["x", "z"], "B": ["y", "z"]}
 
 [case testInspectTypeVarBoundDef]
-# inspect2: --show=definition foo.py:9:13
-# inspect2: --show=definition foo.py:8:9
+# inspect2: --show=definition tmp/foo.py:9:13
+# inspect2: --show=definition tmp/foo.py:8:9
 import foo
 [file foo.py]
 from typing import TypeVar
@@ -189,13 +189,13 @@ def foo(arg: T) -> T:
     return arg
 [out]
 ==
-foo.py:7:9:arg
-foo.py:4:5:x
+tmp/foo.py:7:9:arg
+tmp/foo.py:4:5:x
 
 [case testInspectTypeVarValuesDef]
-# inspect2: --show=definition --force-reload foo.py:13:9
-# inspect2: --show=definition --force-reload foo.py:14:13
-# inspect2: --show=definition foo.py:18:7
+# inspect2: --show=definition --force-reload tmp/foo.py:13:9
+# inspect2: --show=definition --force-reload tmp/foo.py:14:13
+# inspect2: --show=definition tmp/foo.py:18:7
 import foo
 [file foo.py]
 from typing import TypeVar, Generic
@@ -218,12 +218,12 @@ class C(Generic[T]):
     x.z
 [out]
 ==
-foo.py:5:5:z, tmp/foo.py:9:5:z
-foo.py:12:9:arg
-foo.py:5:5:z, tmp/foo.py:9:5:z
+tmp/foo.py:5:5:z, tmp/foo.py:9:5:z
+tmp/foo.py:12:9:arg
+tmp/foo.py:5:5:z, tmp/foo.py:9:5:z
 
 [case testInspectModuleAttrs]
-# inspect2: --show=attrs foo.py:2:1
+# inspect2: --show=attrs tmp/foo.py:2:1
 import foo
 [file foo.py]
 from pack import bar
@@ -239,7 +239,7 @@ class C: ...
 {"<pack.bar>": ["C", "__annotations__", "__doc__", "__file__", "__name__", "__package__", "bar", "x"], "ModuleType": ["__file__", "__getattr__"]}
 
 [case testInspectModuleDef]
-# inspect2: --show=definition --include-kind foo.py:2:1
+# inspect2: --show=definition --include-kind tmp/foo.py:2:1
 import foo
 [file foo.py]
 from pack import bar
@@ -255,7 +255,7 @@ NameExpr -> tmp/pack/bar.py:1:1:bar
 MemberExpr -> tmp/pack/bar.py:3:5:x
 
 [case testInspectFunctionArgDef]
-# inspect2: --show=definition --include-span foo.py:4:13
+# inspect2: --show=definition --include-span tmp/foo.py:4:13
 # TODO: for now all arguments have line/column set to function definition.
 import foo
 [file foo.py]