From 281ee74c7eac0a53a23bb4aabfc049c1ab42ce8a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 13 Dec 2023 21:39:04 +0100 Subject: [PATCH] recursively evaluate TYPE_CHECKING blocks, fix #648 (#649) --- CHANGELOG.md | 3 + pdoc/doc_types.py | 33 ++++++- test/test_doc_types.py | 44 +++++++-- test/test_snapshot.py | 2 +- test/testdata/type_checking_imports.html | 96 +++++++++++++++---- test/testdata/type_checking_imports.py | 18 ---- test/testdata/type_checking_imports.txt | 4 +- .../type_checking_imports/__init__.py | 0 .../type_checking_imports/cached_submodule.py | 12 +++ .../cached_subsubmodule.py | 6 ++ test/testdata/type_checking_imports/main.py | 40 ++++++++ .../uncached_submodule.py | 6 ++ 12 files changed, 215 insertions(+), 49 deletions(-) mode change 100644 => 100755 test/testdata/type_checking_imports.html delete mode 100644 test/testdata/type_checking_imports.py mode change 100644 => 100755 test/testdata/type_checking_imports.txt create mode 100755 test/testdata/type_checking_imports/__init__.py create mode 100755 test/testdata/type_checking_imports/cached_submodule.py create mode 100755 test/testdata/type_checking_imports/cached_subsubmodule.py create mode 100755 test/testdata/type_checking_imports/main.py create mode 100755 test/testdata/type_checking_imports/uncached_submodule.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dcbff190..ef4f9d3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ - pdoc now documents PyO3 or pybind11 submodules that are not picked up by Python's builtin pkgutil module. ([#633](https://github.com/mitmproxy/pdoc/issues/633), @mhils) + - Imports in a TYPE_CHECKING section that reference members defined in another module's TYPE_CHECKING section now work + correctly. + ([#649](https://github.com/mitmproxy/pdoc/pull/649), @mhils) - pdoc now supports Python 3.12's `type` statements and has improved `TypeAlias` rendering. ([#651](https://github.com/mitmproxy/pdoc/pull/651), @mhils) - Add support for `code-block` ReST directives diff --git a/pdoc/doc_types.py b/pdoc/doc_types.py index 795db8a5..7b2520f6 100644 --- a/pdoc/doc_types.py +++ b/pdoc/doc_types.py @@ -124,9 +124,9 @@ def safe_eval_type( # Simple _eval_type has failed. We now execute all TYPE_CHECKING sections in the module and try again. if module: + assert module.__dict__ is globalns try: - code = compile(type_checking_sections(module), "", "exec") - eval(code, globalns, globalns) + _eval_type_checking_sections(module, set()) except Exception as e: warnings.warn( f"Failed to run TYPE_CHECKING code while parsing {t} type annotation for {fullname}: {e}" @@ -148,7 +148,34 @@ def safe_eval_type( f"Error parsing type annotation {t} for {fullname}. Import of {mod} failed: {err}" ) return t - return safe_eval_type(t, {mod: val, **globalns}, localns, module, fullname) + else: + globalns[mod] = val + return safe_eval_type(t, globalns, localns, module, fullname) + + +def _eval_type_checking_sections(module: types.ModuleType, seen: set) -> None: + """ + Evaluate all TYPE_CHECKING sections within a module. + + The added complication here is that TYPE_CHECKING sections may import members from other modules' TYPE_CHECKING + sections. So we try to recursively execute those other modules' TYPE_CHECKING sections as well. + See https://github.com/mitmproxy/pdoc/issues/648 for a real world example. + """ + if module.__name__ in seen: + raise RecursionError(f"Recursion error when importing {module.__name__}.") + seen.add(module.__name__) + + code = compile(type_checking_sections(module), "", "exec") + while True: + try: + eval(code, module.__dict__, module.__dict__) + except ImportError as e: + if e.name is not None and (mod := sys.modules.get(e.name, None)): + _eval_type_checking_sections(mod, seen) + else: + raise + else: + break def _eval_type(t, globalns, localns, recursive_guard=frozenset()): diff --git a/test/test_doc_types.py b/test/test_doc_types.py index 6e5cefef..63b47ad7 100644 --- a/test/test_doc_types.py +++ b/test/test_doc_types.py @@ -12,8 +12,9 @@ "typestr", ["totally_unknown_module", "!!!!", "html.unknown_attr"] ) def test_eval_fail(typestr): + a = types.ModuleType("a") with pytest.warns(UserWarning, match="Error parsing type annotation"): - assert safe_eval_type(typestr, {}, None, types.ModuleType("a"), "a") == typestr + assert safe_eval_type(typestr, a.__dict__, None, a, "a") == typestr def test_eval_fail2(monkeypatch): @@ -22,8 +23,9 @@ def test_eval_fail2(monkeypatch): "get_source", lambda _: "import typing\nif typing.TYPE_CHECKING:\n\traise RuntimeError()", ) + a = types.ModuleType("a") with pytest.warns(UserWarning, match="Failed to run TYPE_CHECKING code"): - assert safe_eval_type("xyz", {}, None, types.ModuleType("a"), "a") == "xyz" + assert safe_eval_type("xyz", a.__dict__, None, a, "a") == "xyz" def test_eval_fail3(monkeypatch): @@ -32,16 +34,24 @@ def test_eval_fail3(monkeypatch): "get_source", lambda _: "import typing\nif typing.TYPE_CHECKING:\n\tFooFn = typing.Callable[[],int]", ) + a = types.ModuleType("a") + a.__dict__["typing"] = typing with pytest.warns( UserWarning, match="Error parsing type annotation .+ after evaluating TYPE_CHECKING blocks", ): - assert ( - safe_eval_type( - "FooFn[int]", {"typing": typing}, None, types.ModuleType("a"), "a" - ) - == "FooFn[int]" - ) + assert safe_eval_type("FooFn[int]", a.__dict__, None, a, "a") == "FooFn[int]" + + +def test_eval_fail_import_nonexistent(monkeypatch): + monkeypatch.setattr( + doc_ast, + "get_source", + lambda _: "import typing\nif typing.TYPE_CHECKING:\n\timport nonexistent_module", + ) + a = types.ModuleType("a") + with pytest.warns(UserWarning, match="No module named 'nonexistent_module'"): + assert safe_eval_type("xyz", a.__dict__, None, a, "a") == "xyz" def test_eval_union_types_on_old_python(monkeypatch): @@ -53,3 +63,21 @@ def test_eval_union_types_on_old_python(monkeypatch): ): # str never implements `|`, so we can use that to trigger the error on newer versions. safe_eval_type('"foo" | "bar"', {}, None, None, "example") + + +def test_recurse(monkeypatch): + def get_source(mod): + if mod == a: + return "import typing\nif typing.TYPE_CHECKING:\n\tfrom b import Foo" + else: + return "import typing\nif typing.TYPE_CHECKING:\n\tfrom a import Foo" + + a = types.ModuleType("a") + b = types.ModuleType("b") + + monkeypatch.setattr(doc_ast, "get_source", get_source) + monkeypatch.setitem(sys.modules, "a", a) + monkeypatch.setitem(sys.modules, "b", b) + + with pytest.warns(UserWarning, match="Recursion error when importing a"): + assert safe_eval_type("xyz", a.__dict__, None, a, "a") == "xyz" diff --git a/test/test_snapshot.py b/test/test_snapshot.py index 23721d14..76eaf4d9 100755 --- a/test/test_snapshot.py +++ b/test/test_snapshot.py @@ -162,7 +162,7 @@ def outfile(self, format: str) -> Path: ), Snapshot("pyo3_sample_library", specs=["pdoc_pyo3_sample_library"]), Snapshot("top_level_reimports", ["top_level_reimports"]), - Snapshot("type_checking_imports"), + Snapshot("type_checking_imports", ["type_checking_imports.main"]), Snapshot("type_stub", min_version=(3, 10)), Snapshot( "visibility", diff --git a/test/testdata/type_checking_imports.html b/test/testdata/type_checking_imports.html old mode 100644 new mode 100755 index 31d6b547..fe8b97e8 --- a/test/testdata/type_checking_imports.html +++ b/test/testdata/type_checking_imports.html @@ -4,7 +4,7 @@ - type_checking_imports API documentation + type_checking_imports.main API documentation @@ -29,6 +29,12 @@

API Documentation

  • var
    • +
  • + imported_from_cached_module +
    • +
  • + imported_from_uncached_module +
    • @@ -43,31 +49,53 @@

    API Documentation

    -type_checking_imports

    +type_checking_imports.main - + - + 
     1from __future__ import annotations
  2
  3import typing
  4from typing import TYPE_CHECKING
  5
- 6if typing.TYPE_CHECKING:
- 7    from typing import Sequence
- 8
- 9if TYPE_CHECKING:
-10    from typing import Dict
-11
-12
-13def foo(a: Sequence[str], b: Dict[str, str]):
-14    """A method with TYPE_CHECKING type annotations."""
-15
+ 6from . import cached_submodule
+ 7
+ 8if typing.TYPE_CHECKING:
+ 9    from typing import Sequence
+10
+11if TYPE_CHECKING:
+12    from typing import Dict
+13
+14    from .cached_submodule import StrOrInt
+15    from .uncached_submodule import StrOrBool
 16
-17var: Sequence[int] = (1, 2, 3)
-18"""A variable with TYPE_CHECKING type annotations."""
+17assert cached_submodule
+18
+19
+20def foo(a: Sequence[str], b: Dict[str, str]):
+21    """A method with TYPE_CHECKING type annotations."""
+22
+23
+24var: Sequence[int] = (1, 2, 3)
+25"""A variable with TYPE_CHECKING type annotations."""
+26
+27
+28imported_from_cached_module: StrOrInt = 42
+29"""
+30A variable with a type annotation that's imported from another file's TYPE_CHECKING block.
+31
+32https://github.com/mitmproxy/pdoc/issues/648
+33"""
+34
+35imported_from_uncached_module: StrOrBool = True
+36"""
+37A variable with a type annotation that's imported from another file's TYPE_CHECKING block.
+38
+39In this case, the module is not in sys.modules outside of TYPE_CHECKING.
+40"""
    @@ -83,8 +111,8 @@

    - 
    14def foo(a: Sequence[str], b: Dict[str, str]):
-15    """A method with TYPE_CHECKING type annotations."""
+            
    21def foo(a: Sequence[str], b: Dict[str, str]):
+22    """A method with TYPE_CHECKING type annotations."""
 
    
 
 
@@ -106,6 +134,38 @@
    +

    +
    +
    + imported_from_cached_module: Union[str, int] = +42 + + +
    + + +

    A variable with a type annotation that's imported from another file's TYPE_CHECKING block.

    + +

    https://github.com/mitmproxy/pdoc/issues/648

    +
    + + +
    +
    +
    + imported_from_uncached_module: Union[str, bool] = +True + + +
    + + +

    A variable with a type annotation that's imported from another file's TYPE_CHECKING block.

    + +

    In this case, the module is not in sys.modules outside of TYPE_CHECKING.

    +
    + +
    diff --git a/test/testdata/type_checking_imports.py b/test/testdata/type_checking_imports.py deleted file mode 100644 index b54f8152..00000000 --- a/test/testdata/type_checking_imports.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -import typing -from typing import TYPE_CHECKING - -if typing.TYPE_CHECKING: - from typing import Sequence - -if TYPE_CHECKING: - from typing import Dict - - -def foo(a: Sequence[str], b: Dict[str, str]): - """A method with TYPE_CHECKING type annotations.""" - - -var: Sequence[int] = (1, 2, 3) -"""A variable with TYPE_CHECKING type annotations.""" diff --git a/test/testdata/type_checking_imports.txt b/test/testdata/type_checking_imports.txt old mode 100644 new mode 100755 index cc167678..c58b4a53 --- a/test/testdata/type_checking_imports.txt +++ b/test/testdata/type_checking_imports.txt @@ -1,4 +1,6 @@ - + + > \ No newline at end of file diff --git a/test/testdata/type_checking_imports/__init__.py b/test/testdata/type_checking_imports/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/test/testdata/type_checking_imports/cached_submodule.py b/test/testdata/type_checking_imports/cached_submodule.py new file mode 100755 index 00000000..f4eac877 --- /dev/null +++ b/test/testdata/type_checking_imports/cached_submodule.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import typing + +from . import cached_subsubmodule + +assert cached_subsubmodule + +if typing.TYPE_CHECKING: + from .cached_subsubmodule import StrOrInt + + assert StrOrInt diff --git a/test/testdata/type_checking_imports/cached_subsubmodule.py b/test/testdata/type_checking_imports/cached_subsubmodule.py new file mode 100755 index 00000000..2b6c909c --- /dev/null +++ b/test/testdata/type_checking_imports/cached_subsubmodule.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +import typing + +if typing.TYPE_CHECKING: + StrOrInt: typing.TypeAlias = "typing.Union[str, int]" diff --git a/test/testdata/type_checking_imports/main.py b/test/testdata/type_checking_imports/main.py new file mode 100755 index 00000000..974f474a --- /dev/null +++ b/test/testdata/type_checking_imports/main.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import typing +from typing import TYPE_CHECKING + +from . import cached_submodule + +if typing.TYPE_CHECKING: + from typing import Sequence + +if TYPE_CHECKING: + from typing import Dict + + from .cached_submodule import StrOrInt + from .uncached_submodule import StrOrBool + +assert cached_submodule + + +def foo(a: Sequence[str], b: Dict[str, str]): + """A method with TYPE_CHECKING type annotations.""" + + +var: Sequence[int] = (1, 2, 3) +"""A variable with TYPE_CHECKING type annotations.""" + + +imported_from_cached_module: StrOrInt = 42 +""" +A variable with a type annotation that's imported from another file's TYPE_CHECKING block. + +https://github.com/mitmproxy/pdoc/issues/648 +""" + +imported_from_uncached_module: StrOrBool = True +""" +A variable with a type annotation that's imported from another file's TYPE_CHECKING block. + +In this case, the module is not in sys.modules outside of TYPE_CHECKING. +""" diff --git a/test/testdata/type_checking_imports/uncached_submodule.py b/test/testdata/type_checking_imports/uncached_submodule.py new file mode 100755 index 00000000..bd4a6262 --- /dev/null +++ b/test/testdata/type_checking_imports/uncached_submodule.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +import typing + +if typing.TYPE_CHECKING: + StrOrBool: typing.TypeAlias = "typing.Union[str, bool]"