From 96b9e0525933f27f26e831e909666a45a9c9b54f Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 11 Jul 2023 00:42:35 -0400 Subject: [PATCH 01/12] Fix #295 --- pydoctor/astbuilder.py | 3 ++ pydoctor/epydoc2stan.py | 87 ++++++++++++++++++++++++++++++- pydoctor/linker.py | 22 ++++++-- pydoctor/model.py | 23 ++++++-- pydoctor/node2stan.py | 39 ++++++++------ pydoctor/test/test_epydoc2stan.py | 78 +++++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 25 deletions(-) diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index e1fc18004..a9199c922 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -176,6 +176,7 @@ def visit_Module(self, node: ast.Module) -> None: def depart_Module(self, node: ast.Module) -> None: self.builder.pop(self.module) + epydoc2stan.transform_parsed_names(self.module) def visit_ClassDef(self, node: ast.ClassDef) -> None: # Ignore classes within functions. @@ -1072,6 +1073,8 @@ def _annotation_for_elements(sequence: Iterable[object]) -> Optional[ast.expr]: class ASTBuilder: """ Keeps tracks of the state of the AST build, creates documentable and adds objects to the system. + + One ASTBuilder instance is only suitable to build one Module. """ ModuleVistor = ModuleVistor diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index fa34e94be..cfb4457fc 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -4,6 +4,8 @@ from collections import defaultdict import enum +import inspect +from itertools import chain from typing import ( TYPE_CHECKING, Any, Callable, ClassVar, DefaultDict, Dict, Generator, Iterator, List, Mapping, Optional, Sequence, Tuple, Union, @@ -12,8 +14,11 @@ import re import attr +from docutils.transforms import Transform +from docutils import nodes from pydoctor import model, linker, node2stan +from pydoctor.node2stan import parse_reference from pydoctor.astutils import is_none_literal from pydoctor.epydoc.markup import Field as EpydocField, ParseError, get_parser_by_name, processtypes from twisted.web.template import Tag, tags @@ -884,7 +889,10 @@ def get_parsed_type(obj: model.Documentable) -> Optional[ParsedDocstring]: # Only Attribute instances have the 'annotation' attribute. annotation: Optional[ast.expr] = getattr(obj, 'annotation', None) if annotation is not None: - return colorize_inline_pyval(annotation) + parsed_type = colorize_inline_pyval(annotation) + # cache parsed_type attribute + obj.parsed_type = parsed_type + return parsed_type return None @@ -1143,3 +1151,80 @@ def populate_constructors_extra_info(cls:model.Class) -> None: extra_epytext += '`%s <%s>`' % (short_text, c.fullName()) cls.extra_info.append(parse_docstring(cls, extra_epytext, cls, 'restructuredtext', section='constructor extra')) + +class _ReferenceTransform(Transform): + + def __init__(self, document:nodes.document, ctx:'model.Documentable'): + super().__init__(document) + self.ctx = ctx + + def apply(self): + ctx = self.ctx + module = self.ctx.module + for node in self.document.findall(nodes.title_reference): + _, target = parse_reference(node) + if target == node.attributes.get('refuri', target): + name, *rest = target.split('.') + # Only apply transformation to non-ambigous names, + # because we don't know if we're dealing with an annotation + # or an interpreted, so we must go with the conservative approach. + if ((module.isNameDefined(name) and + not ctx.isNameDefined(name, only_locals=True)) + or (ctx.isNameDefined(name, only_locals=True) and + not module.isNameDefined(name))): + + node.attributes['refuri'] = '.'.join(chain( + ctx._localNameToFullName(name).split('.'), rest)) + +def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable') -> None: + """ + Runs L{_ReferenceTransform} on the underlying docutils document. + No-op if L{to_node} raises L{NotImplementedError}. + """ + try: + document = doc.to_node() + except NotImplementedError: + return + else: + _ReferenceTransform(document, ctx).apply() + +def transform_parsed_names(node:'model.Module') -> None: + """ + Walk this module's content and apply in-place transformations to the + L{ParsedDocstring} instances that olds L{obj_reference} or L{title_reference} nodes. + + Fixing "Lookup of name in annotation fails on reparented object #295". + The fix is not 100% complete at the moment: attribute values and decorators + are not handled. + """ + from pydoctor import model, astbuilder + # resolve names early when possible + for ob in model.walk(node): + # resolve names in parsed_docstring, do not forget field bodies + if ob.parsed_docstring: + _apply_reference_transform(ob.parsed_docstring, ob) + for f in ob.parsed_docstring.fields: + _apply_reference_transform(f.body(), ob) + if isinstance(ob, model.Function): + if ob.signature: + for p in ob.signature.parameters.values(): + ann = p.annotation if p.annotation is not inspect.Parameter.empty else None + if isinstance(ann, astbuilder._ValueFormatter): + _apply_reference_transform(ann._colorized, ob) + default = p.default if p.default is not inspect.Parameter.empty else None + if isinstance(default, astbuilder._ValueFormatter): + _apply_reference_transform(default._colorized, ob) + # TODO: resolve function's annotations, they are currently presented twice + # we can only change signature, annotations in param table must be handled by + # introducing attribute parsed_annotations + elif isinstance(ob, model.Attribute): + # resolve attribute annotation with parsed_type attribute + parsed_type = get_parsed_type(ob) + if parsed_type: + _apply_reference_transform(parsed_type, ob) + # TODO: resolve parsed_value + # TODO: resolve parsed_decorators + elif isinstance(ob, model.Class): + # TODO: resolve parsed_class_signature + # TODO: resolve parsed_decorators + pass diff --git a/pydoctor/linker.py b/pydoctor/linker.py index c2430a4b2..8f0814591 100644 --- a/pydoctor/linker.py +++ b/pydoctor/linker.py @@ -133,9 +133,13 @@ def look_for_intersphinx(self, name: str) -> Optional[str]: def link_to(self, identifier: str, label: "Flattenable") -> Tag: fullID = self.obj.expandName(identifier) - target = self.obj.system.objForFullName(fullID) - if target is not None: - return taglink(target, self.page_url, label) + try: + target = self.obj.system.find_object(fullID) + except LookupError: + pass + else: + if target is not None: + return taglink(target, self.page_url, label) url = self.look_for_intersphinx(fullID) if url is not None: @@ -184,8 +188,18 @@ def _resolve_identifier_xref(self, if target is not None: return target - # Check if the fullID exists in an intersphinx inventory. fullID = self.obj.expandName(identifier) + + # Try fetching the name with it's outdated fullname + try: + target = self.obj.system.find_object(fullID) + except LookupError: + pass + else: + if target is not None: + return target + + # Check if the fullID exists in an intersphinx inventory. target_url = self.look_for_intersphinx(fullID) if not target_url: # FIXME: https://github.com/twisted/pydoctor/issues/125 diff --git a/pydoctor/model.py b/pydoctor/model.py index 8d0d6c2ba..be84dda66 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -111,6 +111,19 @@ class DocumentableKind(Enum): PROPERTY = 150 VARIABLE = 100 +def walk(node:'Documentable') -> Iterator['Documentable']: + """ + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you + only want to modify nodes in place and don't care about the context. + """ + from collections import deque + todo = deque([node]) + while todo: + node = todo.popleft() + todo.extend(node.contents.values()) + yield node + class Documentable: """An object that can be documented. @@ -271,7 +284,7 @@ def _handle_reparenting_post(self) -> None: def _localNameToFullName(self, name: str) -> str: raise NotImplementedError(self._localNameToFullName) - def isNameDefined(self, name:str) -> bool: + def isNameDefined(self, name:str, only_locals:bool=False) -> bool: """ Is the given name defined in the globals/locals of self-context? Only the first name of a dotted name is checked. @@ -411,13 +424,13 @@ def setup(self) -> None: super().setup() self._localNameToFullName_map: Dict[str, str] = {} - def isNameDefined(self, name: str) -> bool: + def isNameDefined(self, name: str, only_locals:bool=False) -> bool: name = name.split('.')[0] if name in self.contents: return True if name in self._localNameToFullName_map: return True - if not isinstance(self, Module): + if not isinstance(self, Module) and not only_locals: return self.module.isNameDefined(name) else: return False @@ -811,8 +824,8 @@ def docsources(self) -> Iterator[Documentable]: def _localNameToFullName(self, name: str) -> str: return self.parent._localNameToFullName(name) - def isNameDefined(self, name: str) -> bool: - return self.parent.isNameDefined(name) + def isNameDefined(self, name: str, only_locals:bool=False) -> bool: + return self.parent.isNameDefined(name, only_locals=only_locals) class Function(Inheritable): kind = DocumentableKind.FUNCTION diff --git a/pydoctor/node2stan.py b/pydoctor/node2stan.py index 6acab4e91..cb9fbee58 100644 --- a/pydoctor/node2stan.py +++ b/pydoctor/node2stan.py @@ -3,7 +3,7 @@ """ import re import optparse -from typing import Any, Callable, ClassVar, Iterable, List, Optional, Union, TYPE_CHECKING +from typing import Any, Callable, ClassVar, Iterable, List, Optional, Sequence, Tuple, Union, TYPE_CHECKING from docutils.writers import html4css1 from docutils import nodes, frontend, __version_info__ as docutils_version_info @@ -52,6 +52,25 @@ def gettext(node: Union[nodes.Node, List[nodes.Node]]) -> List[str]: filtered.extend(gettext(child)) return filtered +def parse_reference(node:nodes.title_reference) -> Tuple[Union[str, Sequence[nodes.Node]], str]: + """ + Split a reference into (label, target). + """ + label: "Flattenable" + if 'refuri' in node.attributes: + # Epytext parsed or manually constructed nodes. + label, target = node.children, node.attributes['refuri'] + else: + # RST parsed. + m = _TARGET_RE.match(node.astext()) + if m: + label, target = m.groups() + else: + label = target = node.astext() + # Support linking to functions and methods with () at the end + if target.endswith('()'): + target = target[:len(target)-2] + return label, target _TARGET_RE = re.compile(r'^(.*?)\s*<(?:URI:|URL:)?([^<>]+)>$') _VALID_IDENTIFIER_RE = re.compile('[^0-9a-zA-Z_]') @@ -105,22 +124,12 @@ def visit_obj_reference(self, node: nodes.Node) -> None: self._handle_reference(node, link_func=self._linker.link_to) def _handle_reference(self, node: nodes.Node, link_func: Callable[[str, "Flattenable"], "Flattenable"]) -> None: + node_label, target = parse_reference(node) label: "Flattenable" - if 'refuri' in node.attributes: - # Epytext parsed or manually constructed nodes. - label, target = node2stan(node.children, self._linker), node.attributes['refuri'] + if not isinstance(node_label, str): + label = node2stan(node_label, self._linker) else: - # RST parsed. - m = _TARGET_RE.match(node.astext()) - if m: - label, target = m.groups() - else: - label = target = node.astext() - - # Support linking to functions and methods with () at the end - if target.endswith('()'): - target = target[:len(target)-2] - + label = node_label self.body.append(flatten(link_func(target, label))) raise nodes.SkipNode() diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 4d4a8b521..80d0dd238 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -2106,3 +2106,81 @@ def func(): assert docstring2html(mod.contents['func'], docformat='plaintext') == expected captured = capsys.readouterr().out assert captured == '' + +def test_parsed_names_partially_resolved_early() -> None: + """ + Test for issue #295 + + Annotations are first locally resolved when we reach the end of the module, + then again when we actually resolve the name when generating the stan for the annotation. + """ + typing = '''\ + List = ClassVar = TypeVar = object() + ''' + + base = '''\ + import ast + class Vis(ast.NodeVisitor): + ... + ''' + src = '''\ + from typing import List + import typing as t + + from .base import Vis + + class Cls(Vis, t.Generic['_T']): + """ + L{Cls} + """ + clsvar:List[str] + clsvar2:t.ClassVar[List[str]] + + def __init__(self, a:'_T'): + self._a:'_T' = a + + C = Cls + _T = t.TypeVar('_T') + unknow: i|None|list + ann:Cls + ''' + + top = '''\ + # the order matters here + from .src import C, Cls, Vis + __all__ = ['Cls', 'C', 'Vis'] + ''' + + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(top, 'top', is_package=True) + builder.addModuleString(base, 'base', 'top') + builder.addModuleString(src, 'src', 'top') + builder.addModuleString(typing, 'typing') + builder.buildModules() + + Cls = system.allobjects['top.Cls'] + clsvar = Cls.contents['clsvar'] + clsvar2 = Cls.contents['clsvar2'] + a = Cls.contents['_a'] + assert clsvar.expandName('typing.List')=='typing.List' + assert '' in clsvar.parsed_type.to_node().pformat() + assert 'href="typing.html#List"' in flatten(clsvar.parsed_type.to_stan(clsvar.docstring_linker)) + assert 'href="typing.html#ClassVar"' in flatten(clsvar2.parsed_type.to_stan(clsvar2.docstring_linker)) + assert 'href="top.src.html#_T"' in flatten(a.parsed_type.to_stan(clsvar.docstring_linker)) + + # the reparenting/alias issue + ann = system.allobjects['top.src.ann'] + assert 'href="top.Cls.html"' in flatten(ann.parsed_type.to_stan(ann.docstring_linker)) + assert 'href="top.Cls.html"' in flatten(Cls.parsed_docstring.to_stan(Cls.docstring_linker)) + + unknow = system.allobjects['top.src.unknow'] + assert flatten_text(unknow.parsed_type.to_stan(unknow.docstring_linker)) == 'i|None|list' + + + + # TODO: test the __init__ signature and the class bases + + # TODO: Fix two new twisted warnings: + # twisted/internet/_sslverify.py:330: Cannot find link target for "twisted.internet.ssl.DN", resolved from "twisted.internet._sslverify.DistinguishedName" + # twisted/internet/_sslverify.py:347: Cannot find link target for "twisted.internet.ssl.DN", resolved from "twisted.internet._sslverify.DistinguishedName" \ No newline at end of file From 6550919743a2fb0ed08bd0ac99919da9cd379894 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 11 Jul 2023 17:40:37 -0400 Subject: [PATCH 02/12] Fix the _ReferenceTransform for annotations. Also enamble msg(once=True) when using report(). --- pydoctor/epydoc2stan.py | 37 ++++++++++++++++---------- pydoctor/linker.py | 26 ++++++++++--------- pydoctor/model.py | 6 +++-- pydoctor/test/test_epydoc2stan.py | 43 +++++++++++++++++++------------ 4 files changed, 67 insertions(+), 45 deletions(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index cfb4457fc..3f9141265 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -1154,9 +1154,11 @@ def populate_constructors_extra_info(cls:model.Class) -> None: class _ReferenceTransform(Transform): - def __init__(self, document:nodes.document, ctx:'model.Documentable'): + def __init__(self, document:nodes.document, + ctx:'model.Documentable', is_annotation:bool): super().__init__(document) self.ctx = ctx + self.is_annotation = is_annotation def apply(self): ctx = self.ctx @@ -1165,18 +1167,25 @@ def apply(self): _, target = parse_reference(node) if target == node.attributes.get('refuri', target): name, *rest = target.split('.') - # Only apply transformation to non-ambigous names, - # because we don't know if we're dealing with an annotation - # or an interpreted, so we must go with the conservative approach. - if ((module.isNameDefined(name) and - not ctx.isNameDefined(name, only_locals=True)) - or (ctx.isNameDefined(name, only_locals=True) and - not module.isNameDefined(name))): + + # kindda duplicate a little part of the annotation linker logic here, + # there are no simple way of doing it otherwise at the moment. + # Once all presented parsed elements are stored as Documentable attributes + # we might be able to simply use that and drop the use of the annotation linker, + # but for now this will to the trick: + lookup_context = ctx + if self.is_annotation and ctx is not module and module.isNameDefined(name, + only_locals=True) and ctx.isNameDefined(name, only_locals=True): + # If we're dealing with an annotation, give precedence to the module's + # lookup (wrt PEP 563) + lookup_context = module + linker.warn_ambiguous_annotation(module, ctx, target) - node.attributes['refuri'] = '.'.join(chain( - ctx._localNameToFullName(name).split('.'), rest)) + node.attributes['refuri'] = '.'.join(chain( + lookup_context._localNameToFullName(name).split('.'), rest)) -def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable') -> None: +def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable', + is_annotation:bool=False) -> None: """ Runs L{_ReferenceTransform} on the underlying docutils document. No-op if L{to_node} raises L{NotImplementedError}. @@ -1186,7 +1195,7 @@ def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable') -> except NotImplementedError: return else: - _ReferenceTransform(document, ctx).apply() + _ReferenceTransform(document, ctx, is_annotation).apply() def transform_parsed_names(node:'model.Module') -> None: """ @@ -1210,7 +1219,7 @@ def transform_parsed_names(node:'model.Module') -> None: for p in ob.signature.parameters.values(): ann = p.annotation if p.annotation is not inspect.Parameter.empty else None if isinstance(ann, astbuilder._ValueFormatter): - _apply_reference_transform(ann._colorized, ob) + _apply_reference_transform(ann._colorized, ob, is_annotation=True) default = p.default if p.default is not inspect.Parameter.empty else None if isinstance(default, astbuilder._ValueFormatter): _apply_reference_transform(default._colorized, ob) @@ -1221,7 +1230,7 @@ def transform_parsed_names(node:'model.Module') -> None: # resolve attribute annotation with parsed_type attribute parsed_type = get_parsed_type(ob) if parsed_type: - _apply_reference_transform(parsed_type, ob) + _apply_reference_transform(parsed_type, ob, is_annotation=True) # TODO: resolve parsed_value # TODO: resolve parsed_decorators elif isinstance(ob, model.Class): diff --git a/pydoctor/linker.py b/pydoctor/linker.py index 8f0814591..95162f005 100644 --- a/pydoctor/linker.py +++ b/pydoctor/linker.py @@ -252,6 +252,19 @@ def _resolve_identifier_xref(self, self.reporting_obj.report(message, 'resolve_identifier_xref', lineno) raise LookupError(identifier) +def warn_ambiguous_annotation(mod:'model.Documentable', + obj:'model.Documentable', + target:str) -> None: + # report a low-level message about ambiguous annotation + mod_ann = mod.expandName(target) + obj_ann = obj.expandName(target) + if mod_ann != obj_ann: + obj.report( + f'ambiguous annotation {target!r}, could be interpreted as ' + f'{obj_ann!r} instead of {mod_ann!r}', section='annotation', + thresh=1 + ) + class _AnnotationLinker(DocstringLinker): """ Specialized linker to resolve annotations attached to the given L{Documentable}. @@ -271,21 +284,10 @@ def __init__(self, obj:'model.Documentable') -> None: @property def obj(self) -> 'model.Documentable': return self._obj - - def warn_ambiguous_annotation(self, target:str) -> None: - # report a low-level message about ambiguous annotation - mod_ann = self._module.expandName(target) - obj_ann = self._scope.expandName(target) - if mod_ann != obj_ann: - self.obj.report( - f'ambiguous annotation {target!r}, could be interpreted as ' - f'{obj_ann!r} instead of {mod_ann!r}', section='annotation', - thresh=1 - ) def link_to(self, target: str, label: "Flattenable") -> Tag: if self._module.isNameDefined(target): - self.warn_ambiguous_annotation(target) + warn_ambiguous_annotation(self._module, self._obj, target) return self._module_linker.link_to(target, label) elif self._scope.isNameDefined(target): return self._scope_linker.link_to(target, label) diff --git a/pydoctor/model.py b/pydoctor/model.py index be84dda66..1e0bae1f9 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -380,7 +380,8 @@ def module(self) -> 'Module': def report(self, descr: str, section: str = 'parsing', lineno_offset: int = 0, thresh:int=-1) -> None: """ - Log an error or warning about this documentable object. + Log an error or warning about this documentable object. + A reported message will only be printed once. @param descr: The error/warning string @param section: What the warning is about. @@ -405,7 +406,8 @@ def report(self, descr: str, section: str = 'parsing', lineno_offset: int = 0, t self.system.msg( section, f'{self.description}:{linenumber}: {descr}', - thresh=thresh) + # some warnings can be reported more that once. + thresh=thresh, once=True) @property def docstring_linker(self) -> 'linker.DocstringLinker': diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 80d0dd238..fb80272ca 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -1029,21 +1029,22 @@ class Klass: mod.parsed_docstring.get_summary().to_stan(mod.docstring_linker) # type:ignore warnings = ['test:2: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] - if linkercls is linker._EpydocLinker: - warnings = warnings * 2 assert capsys.readouterr().out.strip().splitlines() == warnings - + + # reset warnings + mod.system.once_msgs = set() + # This is wrong: Klass.parsed_docstring.to_stan(mod.docstring_linker) # type:ignore Klass.parsed_docstring.get_summary().to_stan(mod.docstring_linker) # type:ignore # Because the warnings will be reported on line 2 warnings = ['test:2: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] - warnings = warnings * 2 assert capsys.readouterr().out.strip().splitlines() == warnings - # assert capsys.readouterr().out == '' + # reset warnings + mod.system.once_msgs = set() # Reset stan and summary, because they are supposed to be cached. Klass.parsed_docstring._stan = None # type:ignore @@ -1054,9 +1055,7 @@ class Klass: Klass.parsed_docstring.to_stan(mod.docstring_linker) # type:ignore Klass.parsed_docstring.get_summary().to_stan(mod.docstring_linker) # type:ignore - warnings = ['test:5: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] - warnings = warnings * 2 - + warnings = ['test:5: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] assert capsys.readouterr().out.strip().splitlines() == warnings def test_EpydocLinker_look_for_intersphinx_no_link() -> None: @@ -1291,8 +1290,6 @@ def test_EpydocLinker_warnings(capsys: CapSys) -> None: # The rationale about xref warnings is to warn when the target cannot be found. assert captured == ('module:3: Cannot find link target for "notfound"' - '\nmodule:3: Cannot find link target for "notfound"' - '\nmodule:5: Cannot find link target for "notfound"' '\nmodule:5: Cannot find link target for "notfound"\n') assert 'href="index.html#base"' in summary2html(mod) @@ -1999,7 +1996,6 @@ def f(self, x:typ) -> typ: assert capsys.readouterr().out == """\ m:5: ambiguous annotation 'typ', could be interpreted as 'm.C.typ' instead of 'm.typ' -m:5: ambiguous annotation 'typ', could be interpreted as 'm.C.typ' instead of 'm.typ' m:7: ambiguous annotation 'typ', could be interpreted as 'm.C.typ' instead of 'm.typ' """ @@ -2177,10 +2173,23 @@ def __init__(self, a:'_T'): unknow = system.allobjects['top.src.unknow'] assert flatten_text(unknow.parsed_type.to_stan(unknow.docstring_linker)) == 'i|None|list' - - - # TODO: test the __init__ signature and the class bases + # test the __init__ signature + assert 'href="top.src.html#_T"' in flatten(format_signature(Cls.contents['__init__'])) - # TODO: Fix two new twisted warnings: - # twisted/internet/_sslverify.py:330: Cannot find link target for "twisted.internet.ssl.DN", resolved from "twisted.internet._sslverify.DistinguishedName" - # twisted/internet/_sslverify.py:347: Cannot find link target for "twisted.internet.ssl.DN", resolved from "twisted.internet._sslverify.DistinguishedName" \ No newline at end of file +def test_reparented_ambiguous_annotation_confusion() -> None: + """ + Like L{test_top_level_type_alias_wins_over_class_level} but with reparented class. + """ + src = ''' + typ = object() + class C: + typ = int|str + var: typ + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='_m') + builder.addModuleString('from _m import C; __all__=["C"]', 'm') + builder.buildModules() + var = system.allobjects['m.C.var'] + assert 'href="_m.html#typ"' in flatten(var.parsed_type.to_stan(var.docstring_linker)) From 34e2c5433522757d0c2699c65d66d50d8c34d71b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 11 Jul 2023 17:51:56 -0400 Subject: [PATCH 03/12] Fix mypy --- pydoctor/epydoc2stan.py | 4 +++- pydoctor/node2stan.py | 2 +- pydoctor/templatewriter/__init__.py | 2 +- pydoctor/test/test_epydoc2stan.py | 18 +++++++++--------- pydoctor/test/test_sphinx.py | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index 3f9141265..e074866ea 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -1160,7 +1160,7 @@ def __init__(self, document:nodes.document, self.ctx = ctx self.is_annotation = is_annotation - def apply(self): + def apply(self) -> None: ctx = self.ctx module = self.ctx.module for node in self.document.findall(nodes.title_reference): @@ -1237,3 +1237,5 @@ def transform_parsed_names(node:'model.Module') -> None: # TODO: resolve parsed_class_signature # TODO: resolve parsed_decorators pass + +# do one test with parsed type docstrings \ No newline at end of file diff --git a/pydoctor/node2stan.py b/pydoctor/node2stan.py index cb9fbee58..b5ae44ec6 100644 --- a/pydoctor/node2stan.py +++ b/pydoctor/node2stan.py @@ -56,7 +56,7 @@ def parse_reference(node:nodes.title_reference) -> Tuple[Union[str, Sequence[nod """ Split a reference into (label, target). """ - label: "Flattenable" + label: Union[str, Sequence[nodes.Node]] if 'refuri' in node.attributes: # Epytext parsed or manually constructed nodes. label, target = node.children, node.attributes['refuri'] diff --git a/pydoctor/templatewriter/__init__.py b/pydoctor/templatewriter/__init__.py index ffaff07a1..3ae033261 100644 --- a/pydoctor/templatewriter/__init__.py +++ b/pydoctor/templatewriter/__init__.py @@ -39,7 +39,7 @@ def parse_xml(text: str) -> minidom.Document: """ try: # TODO: submit a PR to typeshed to add a return type for parseString() - return cast(minidom.Document, minidom.parseString(text)) + return minidom.parseString(text) except Exception as e: raise ValueError(f"Failed to parse template as XML: {e}") from e diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index fb80272ca..3a33dccc3 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -2160,21 +2160,21 @@ def __init__(self, a:'_T'): clsvar2 = Cls.contents['clsvar2'] a = Cls.contents['_a'] assert clsvar.expandName('typing.List')=='typing.List' - assert '' in clsvar.parsed_type.to_node().pformat() - assert 'href="typing.html#List"' in flatten(clsvar.parsed_type.to_stan(clsvar.docstring_linker)) - assert 'href="typing.html#ClassVar"' in flatten(clsvar2.parsed_type.to_stan(clsvar2.docstring_linker)) - assert 'href="top.src.html#_T"' in flatten(a.parsed_type.to_stan(clsvar.docstring_linker)) + assert '' in clsvar.parsed_type.to_node().pformat() #type: ignore + assert 'href="typing.html#List"' in flatten(clsvar.parsed_type.to_stan(clsvar.docstring_linker)) #type: ignore + assert 'href="typing.html#ClassVar"' in flatten(clsvar2.parsed_type.to_stan(clsvar2.docstring_linker)) #type: ignore + assert 'href="top.src.html#_T"' in flatten(a.parsed_type.to_stan(clsvar.docstring_linker)) #type: ignore # the reparenting/alias issue ann = system.allobjects['top.src.ann'] - assert 'href="top.Cls.html"' in flatten(ann.parsed_type.to_stan(ann.docstring_linker)) - assert 'href="top.Cls.html"' in flatten(Cls.parsed_docstring.to_stan(Cls.docstring_linker)) + assert 'href="top.Cls.html"' in flatten(ann.parsed_type.to_stan(ann.docstring_linker)) #type: ignore + assert 'href="top.Cls.html"' in flatten(Cls.parsed_docstring.to_stan(Cls.docstring_linker)) #type: ignore unknow = system.allobjects['top.src.unknow'] - assert flatten_text(unknow.parsed_type.to_stan(unknow.docstring_linker)) == 'i|None|list' + assert flatten_text(unknow.parsed_type.to_stan(unknow.docstring_linker)) == 'i|None|list' #type: ignore # test the __init__ signature - assert 'href="top.src.html#_T"' in flatten(format_signature(Cls.contents['__init__'])) + assert 'href="top.src.html#_T"' in flatten(format_signature(Cls.contents['__init__'])) #type: ignore def test_reparented_ambiguous_annotation_confusion() -> None: """ @@ -2192,4 +2192,4 @@ class C: builder.addModuleString('from _m import C; __all__=["C"]', 'm') builder.buildModules() var = system.allobjects['m.C.var'] - assert 'href="_m.html#typ"' in flatten(var.parsed_type.to_stan(var.docstring_linker)) + assert 'href="_m.html#typ"' in flatten(var.parsed_type.to_stan(var.docstring_linker)) #type: ignore diff --git a/pydoctor/test/test_sphinx.py b/pydoctor/test/test_sphinx.py index d05274c04..2f87e7850 100644 --- a/pydoctor/test/test_sphinx.py +++ b/pydoctor/test/test_sphinx.py @@ -110,7 +110,7 @@ def test_generate_empty_functional() -> None: @contextmanager def openFileForWriting(path: str) -> Iterator[io.BytesIO]: yield output - inv_writer._openFileForWriting = openFileForWriting # type: ignore[assignment] + inv_writer._openFileForWriting = openFileForWriting # type:ignore inv_writer.generate(subjects=[], basepath='base-path') From ef41a2fba5cd1685ea7e09bd9e93e4579e6bb8ab Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 11 Jul 2023 17:52:52 -0400 Subject: [PATCH 04/12] fix pyflakes --- pydoctor/templatewriter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoctor/templatewriter/__init__.py b/pydoctor/templatewriter/__init__.py index 3ae033261..e81da8934 100644 --- a/pydoctor/templatewriter/__init__.py +++ b/pydoctor/templatewriter/__init__.py @@ -1,5 +1,5 @@ """Render pydoctor data as HTML.""" -from typing import Any, Iterable, Iterator, Optional, Union, cast, TYPE_CHECKING +from typing import Any, Iterable, Iterator, Optional, Union, TYPE_CHECKING if TYPE_CHECKING: from typing_extensions import Protocol, runtime_checkable else: From bd8af401901abab41591176b9c7cc92e90f6ba79 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 11 Jul 2023 18:15:20 -0400 Subject: [PATCH 05/12] fix ref --- pydoctor/epydoc2stan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index e074866ea..0bd2ce13c 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -1200,7 +1200,7 @@ def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable', def transform_parsed_names(node:'model.Module') -> None: """ Walk this module's content and apply in-place transformations to the - L{ParsedDocstring} instances that olds L{obj_reference} or L{title_reference} nodes. + L{ParsedDocstring} instances that olds L{obj_reference} or L{nodes.title_reference} nodes. Fixing "Lookup of name in annotation fails on reparented object #295". The fix is not 100% complete at the moment: attribute values and decorators From bd69087992fbe15d711036de5734dda401fc04bf Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Wed, 12 Jul 2023 13:07:43 -0400 Subject: [PATCH 06/12] Fix issue regarding builtin names. Introduce the rawtarget attribute as well. --- pydoctor/epydoc2stan.py | 33 ++++++++++++--- pydoctor/sphinx.py | 6 +++ pydoctor/test/test_epydoc2stan.py | 68 ++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index 0bd2ce13c..ad812ed6c 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -5,6 +5,7 @@ from collections import defaultdict import enum import inspect +import builtins from itertools import chain from typing import ( TYPE_CHECKING, Any, Callable, ClassVar, DefaultDict, Dict, Generator, @@ -1152,6 +1153,8 @@ def populate_constructors_extra_info(cls:model.Class) -> None: cls.extra_info.append(parse_docstring(cls, extra_epytext, cls, 'restructuredtext', section='constructor extra')) +_builtin_names = set(dir(builtins)) + class _ReferenceTransform(Transform): def __init__(self, document:nodes.document, @@ -1165,14 +1168,34 @@ def apply(self) -> None: module = self.ctx.module for node in self.document.findall(nodes.title_reference): _, target = parse_reference(node) - if target == node.attributes.get('refuri', target): + + # we're setting two attributes here: 'refuri' and 'rawtarget'. + # 'refuri' might already be created by the colorizer or docstring parser, + # but 'rawtarget' is only created from within this transform, so we can + # use that information to ensure this process is only ever applied once + # per title_reference element. + attribs = node.attributes + if target == attribs.get('refuri', target) and 'rawtarget' not in attribs: + # save the raw target name + attribs['rawtarget'] = target + name, *rest = target.split('.') + is_name_defined = ctx.isNameDefined(name) + # check if it's a non-shadowed builtins + if not is_name_defined and name in _builtin_names: + # transform bare builtin name into builtins. + attribs['refuri'] = '.'.join(('builtins', name, *rest)) + return + # no-op for unbound name + if not is_name_defined: + attribs['refuri'] = target + return # kindda duplicate a little part of the annotation linker logic here, # there are no simple way of doing it otherwise at the moment. # Once all presented parsed elements are stored as Documentable attributes # we might be able to simply use that and drop the use of the annotation linker, - # but for now this will to the trick: + # but for now this will do the trick: lookup_context = ctx if self.is_annotation and ctx is not module and module.isNameDefined(name, only_locals=True) and ctx.isNameDefined(name, only_locals=True): @@ -1180,9 +1203,9 @@ def apply(self) -> None: # lookup (wrt PEP 563) lookup_context = module linker.warn_ambiguous_annotation(module, ctx, target) - - node.attributes['refuri'] = '.'.join(chain( - lookup_context._localNameToFullName(name).split('.'), rest)) + # save pre-resolved refuri + attribs['refuri'] = '.'.join(chain(lookup_context.expandName(name).split('.'), rest)) + def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable', is_annotation:bool=False) -> None: diff --git a/pydoctor/sphinx.py b/pydoctor/sphinx.py index 6e1a8ebad..2cc4862b9 100644 --- a/pydoctor/sphinx.py +++ b/pydoctor/sphinx.py @@ -135,6 +135,12 @@ def getLink(self, name: str) -> Optional[str]: """ Return link for `name` or None if no link is found. """ + # special casing the 'builtins' module because our name resolving + # replaces bare builtins names with builtins. in order not to confuse + # them with objects in the system when reparenting. + if name.startswith('builtins.'): + name = name[len('builtins.'):] + base_url, relative_link = self._links.get(name, (None, None)) if not relative_link: return None diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 3a33dccc3..2c8181422 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -1187,6 +1187,43 @@ def test_EpydocLinker_resolve_identifier_xref_intersphinx_link_not_found(capsys: assert expected == captured +def test_EpydocLinker_link_not_found_show_original(capsys: CapSys) -> None: + n = '' + m = '''\ + from n import Stuff + S = Stuff + ''' + src = '''\ + """ + L{S} + """ + class Cls: + """ + L{Stuff } + """ + from m import S + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(n, 'n') + builder.addModuleString(m, 'm') + builder.addModuleString(src, 'src') + builder.buildModules() + docstring2html(system.allobjects['src']) + captured = capsys.readouterr().out + # TODO: shoud say resolved from "S" + expected = ( + 'src:2: Cannot find link target for "n.Stuff", resolved from "m.S"\n' + ) + assert expected == captured + + docstring2html(system.allobjects['src.Cls']) + captured = capsys.readouterr().out + expected = ( + 'src:6: Cannot find link target for "n.Stuff", resolved from "m.S"\n' + ) + assert expected == captured + class InMemoryInventory: """ A simple inventory implementation which has an in-memory API link mapping. @@ -1412,6 +1449,8 @@ def __init__(self) -> None: self.requests: List[str] = [] def link_to(self, target: str, label: "Flattenable") -> Tag: + if target.startswith('builtins.'): + target = target[len('builtins.'):] self.requests.append(target) return tags.transparent(label) @@ -2160,7 +2199,7 @@ def __init__(self, a:'_T'): clsvar2 = Cls.contents['clsvar2'] a = Cls.contents['_a'] assert clsvar.expandName('typing.List')=='typing.List' - assert '' in clsvar.parsed_type.to_node().pformat() #type: ignore + assert 'refuri="typing.List"' in clsvar.parsed_type.to_node().pformat() #type: ignore assert 'href="typing.html#List"' in flatten(clsvar.parsed_type.to_stan(clsvar.docstring_linker)) #type: ignore assert 'href="typing.html#ClassVar"' in flatten(clsvar2.parsed_type.to_stan(clsvar2.docstring_linker)) #type: ignore assert 'href="top.src.html#_T"' in flatten(a.parsed_type.to_stan(clsvar.docstring_linker)) #type: ignore @@ -2193,3 +2232,30 @@ class C: builder.buildModules() var = system.allobjects['m.C.var'] assert 'href="_m.html#typ"' in flatten(var.parsed_type.to_stan(var.docstring_linker)) #type: ignore + +def test_reparented_builtins_confusion() -> None: + """ + - builtin links are resolved as such even when the new parent + declares a name shadowing a builtin. + """ + src = ''' + class C: + var: list + C = print('one') + ''' + top = ''' + list = object + print = partial(print, flush=True) + + from src import C + __all__=["C"] + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='src') + builder.addModuleString(top, modname='top') + builder.buildModules() + clsvar = system.allobjects['top.C.var'] + + assert 'refuri="builtins.list"' in clsvar.parsed_type.to_node().pformat() #type: ignore + # does not work for constant values at the moment From 0fd9982ba900b53c50a449a97d7cf08789432827 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Wed, 12 Jul 2023 13:27:34 -0400 Subject: [PATCH 07/12] Fix big loop issue --- pydoctor/epydoc2stan.py | 81 ++++++++++++++++--------------- pydoctor/test/test_epydoc2stan.py | 18 +++++++ 2 files changed, 59 insertions(+), 40 deletions(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index ad812ed6c..d0407f3b2 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -1161,50 +1161,51 @@ def __init__(self, document:nodes.document, ctx:'model.Documentable', is_annotation:bool): super().__init__(document) self.ctx = ctx + self.module = ctx.module self.is_annotation = is_annotation - def apply(self) -> None: + def _transform(self, node:nodes.title_reference) -> None: ctx = self.ctx - module = self.ctx.module + module = self.module + _, target = parse_reference(node) + # we're setting two attributes here: 'refuri' and 'rawtarget'. + # 'refuri' might already be created by the colorizer or docstring parser, + # but 'rawtarget' is only created from within this transform, so we can + # use that information to ensure this process is only ever applied once + # per title_reference element. + attribs = node.attributes + if target == attribs.get('refuri', target) and 'rawtarget' not in attribs: + # save the raw target name + attribs['rawtarget'] = target + name, *rest = target.split('.') + is_name_defined = ctx.isNameDefined(name) + # check if it's a non-shadowed builtins + if not is_name_defined and name in _builtin_names: + # transform bare builtin name into builtins. + attribs['refuri'] = '.'.join(('builtins', name, *rest)) + return + # no-op for unbound name + if not is_name_defined: + attribs['refuri'] = target + return + # kindda duplicate a little part of the annotation linker logic here, + # there are no simple way of doing it otherwise at the moment. + # Once all presented parsed elements are stored as Documentable attributes + # we might be able to simply use that and drop the use of the annotation linker, + # but for now this will do the trick: + lookup_context = ctx + if self.is_annotation and ctx is not module and module.isNameDefined(name, + only_locals=True) and ctx.isNameDefined(name, only_locals=True): + # If we're dealing with an annotation, give precedence to the module's + # lookup (wrt PEP 563) + lookup_context = module + linker.warn_ambiguous_annotation(module, ctx, target) + # save pre-resolved refuri + attribs['refuri'] = '.'.join(chain(lookup_context.expandName(name).split('.'), rest)) + + def apply(self) -> None: for node in self.document.findall(nodes.title_reference): - _, target = parse_reference(node) - - # we're setting two attributes here: 'refuri' and 'rawtarget'. - # 'refuri' might already be created by the colorizer or docstring parser, - # but 'rawtarget' is only created from within this transform, so we can - # use that information to ensure this process is only ever applied once - # per title_reference element. - attribs = node.attributes - if target == attribs.get('refuri', target) and 'rawtarget' not in attribs: - # save the raw target name - attribs['rawtarget'] = target - - name, *rest = target.split('.') - is_name_defined = ctx.isNameDefined(name) - # check if it's a non-shadowed builtins - if not is_name_defined and name in _builtin_names: - # transform bare builtin name into builtins. - attribs['refuri'] = '.'.join(('builtins', name, *rest)) - return - # no-op for unbound name - if not is_name_defined: - attribs['refuri'] = target - return - - # kindda duplicate a little part of the annotation linker logic here, - # there are no simple way of doing it otherwise at the moment. - # Once all presented parsed elements are stored as Documentable attributes - # we might be able to simply use that and drop the use of the annotation linker, - # but for now this will do the trick: - lookup_context = ctx - if self.is_annotation and ctx is not module and module.isNameDefined(name, - only_locals=True) and ctx.isNameDefined(name, only_locals=True): - # If we're dealing with an annotation, give precedence to the module's - # lookup (wrt PEP 563) - lookup_context = module - linker.warn_ambiguous_annotation(module, ctx, target) - # save pre-resolved refuri - attribs['refuri'] = '.'.join(chain(lookup_context.expandName(name).split('.'), rest)) + self._transform(node) def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable', diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 2c8181422..eebef6aad 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -2259,3 +2259,21 @@ class C: assert 'refuri="builtins.list"' in clsvar.parsed_type.to_node().pformat() #type: ignore # does not work for constant values at the moment + +def test_link_resolving_unbound_names() -> None: + """ + - unbdound names are not touched, and does not stop the process. + """ + src = ''' + class C: + var: unknown|list + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='src') + builder.buildModules() + clsvar = system.allobjects['src.C.var'] + + assert 'refuri="builtins.list"' in clsvar.parsed_type.to_node().pformat() #type: ignore + assert 'refuri="unknown"' in clsvar.parsed_type.to_node().pformat() #type: ignore + # does not work for constant values at the moment \ No newline at end of file From b84f24c4b98111ee7d0a24616f97c8383a164ae0 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Wed, 12 Jul 2023 15:14:58 -0400 Subject: [PATCH 08/12] Better test coverage for the linker --- pydoctor/test/test_epydoc2stan.py | 55 +++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index eebef6aad..8d802354e 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -1359,6 +1359,61 @@ class C: assert 'href="#var"' in url assert not capsys.readouterr().out +def test_EpydocLinker_xref_look_for_name_multiple_candidates(capsys:CapSys) -> None: + """ + When the linker use look_for_name(), if 'identifier' refers to more than one object, it complains. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('class C:...', modname='_one') + builder.addModuleString('class C:...', modname='_two') + builder.addModuleString('"L{C}"', modname='top') + builder.buildModules() + docstring2html(system.allobjects['top']) + assert capsys.readouterr().out == ( + 'top:1: ambiguous ref to C, could be _one.C, _two.C\n' + 'top:1: Cannot find link target for "C"\n') + +def test_EpydocLinker_xref_look_for_name_into_uncle_objects(capsys:CapSys) -> None: + """ + The linker walk up the object tree and see if 'identifier' refers to an + object in an "uncle" object. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('', modname='pack', is_package=True) + builder.addModuleString('class C:...', modname='mod2', parent_name='pack') + builder.addModuleString('class I:\n var=1;"L{C}"', modname='mod1', parent_name='pack') + builder.buildModules() + assert 'href="pack.mod2.C.html"' in docstring2html(system.allobjects['pack.mod1.I.var']) + assert capsys.readouterr().out == '' + +def test_EpydocLinker_xref_look_for_name_into_all_modules(capsys:CapSys) -> None: + """ + The linker examine every module and package in the system and see if 'identifier' + names an object in each one. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('class C:...', modname='_one') + builder.addModuleString('"L{C}"', modname='top') + builder.buildModules() + assert 'href="_one.C.html"' in docstring2html(system.allobjects['top']) + assert capsys.readouterr().out == '' + +def test_EpydocLinker_xref_walk_up_the_object_tree(capsys:CapSys) -> None: + """ + The linker walks up the object tree and see if 'identifier' refers + to an object by Python name resolution in each context. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('class C:...', modname='pack', is_package=True) + builder.addModuleString('class I:\n var=1;"L{C}"', modname='mod1', parent_name='pack') + builder.buildModules() + assert 'href="pack.C.html"' in docstring2html(system.allobjects['pack.mod1.I.var']) + assert capsys.readouterr().out == '' + def test_xref_not_found_epytext(capsys: CapSys) -> None: """ When a link in an epytext docstring cannot be resolved, the reference From 31144e50c694dafd8bd119f68cfb19e10e38a496 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 13 Jul 2023 12:27:11 -0400 Subject: [PATCH 09/12] Introduce Class.parsed_bases, Function.parsed_decorators, Function.parsed_annotations, FunctionOverload.parsed_decorators, Attribute.parsed_decorators, Attribute.parsed_value. Fixes links in presentation of these components. --- pydoctor/astbuilder.py | 8 +- pydoctor/epydoc/markup/_types.py | 1 + pydoctor/epydoc2stan.py | 150 ++++++++++++++++------ pydoctor/model.py | 9 +- pydoctor/templatewriter/pages/__init__.py | 31 +---- pydoctor/test/test_epydoc2stan.py | 24 +++- 6 files changed, 150 insertions(+), 73 deletions(-) diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index a9199c922..d663cfeee 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -981,12 +981,12 @@ class _ValueFormatter: """ def __init__(self, value: ast.expr, ctx: model.Documentable): - self._colorized = colorize_inline_pyval(value) + self.parsed = colorize_inline_pyval(value) """ The colorized value as L{ParsedDocstring}. """ - self._linker = ctx.docstring_linker + self.linker = ctx.docstring_linker """ Linker. """ @@ -999,7 +999,7 @@ def __repr__(self) -> str: # Using node2stan.node2html instead of flatten(to_stan()). # This avoids calling flatten() twice, # but potential XML parser errors caused by XMLString needs to be handled later. - return ''.join(node2stan.node2html(self._colorized.to_node(), self._linker)) + return ''.join(node2stan.node2html(self.parsed.to_node(), self.linker)) class _AnnotationValueFormatter(_ValueFormatter): """ @@ -1007,7 +1007,7 @@ class _AnnotationValueFormatter(_ValueFormatter): """ def __init__(self, value: ast.expr, ctx: model.Function): super().__init__(value, ctx) - self._linker = linker._AnnotationLinker(ctx) + self.linker = linker._AnnotationLinker(ctx) def __repr__(self) -> str: """ diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index b221ccb87..8b5f00f6f 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -13,6 +13,7 @@ from docutils import nodes from twisted.web.template import Tag, tags +# TODO: this class should support to_node() like others. class ParsedTypeDocstring(TypeDocstring, ParsedDocstring): """ Add L{ParsedDocstring} interface on top of L{TypeDocstring} and diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index d0407f3b2..1da415e46 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -4,12 +4,13 @@ from collections import defaultdict import enum +from functools import partial import inspect import builtins from itertools import chain from typing import ( TYPE_CHECKING, Any, Callable, ClassVar, DefaultDict, Dict, Generator, - Iterator, List, Mapping, Optional, Sequence, Tuple, Union, + Iterator, List, Mapping, Optional, Sequence, Tuple, TypeVar, Union, ) import ast import re @@ -271,18 +272,19 @@ def __init__(self, obj: model.Documentable): self.sinces: List[Field] = [] self.unknowns: DefaultDict[str, List[FieldDesc]] = defaultdict(list) - def set_param_types_from_annotations( - self, annotations: Mapping[str, Optional[ast.expr]] - ) -> None: + def set_param_types_from_annotations(self) -> None: + if not isinstance(self.obj, model.Function): + return + annotations = self.obj.annotations _linker = linker._AnnotationLinker(self.obj) formatted_annotations = { - name: None if value is None - else ParamType(safe_to_stan(colorize_inline_pyval(value), _linker, + name: None if parsed_annotation is None + else ParamType(safe_to_stan(parsed_annotation, _linker, self.obj, fallback=colorized_pyval_fallback, section='annotation', report=False), # don't spam the log, invalid annotation are going to be reported when the signature gets colorized origin=FieldOrigin.FROM_AST) - for name, value in annotations.items() + for name, parsed_annotation in get_parsed_annotations(self.obj).items() } ret_type = formatted_annotations.pop('return', None) @@ -799,8 +801,7 @@ def format_docstring(obj: model.Documentable) -> Tag: ret(unwrap_docstring_stan(stan)) fh = FieldHandler(obj) - if isinstance(obj, model.Function): - fh.set_param_types_from_annotations(obj.annotations) + fh.set_param_types_from_annotations() if source is not None: assert obj.parsed_docstring is not None, "ensure_parsed_docstring() did not do it's job" for field in obj.parsed_docstring.fields: @@ -879,23 +880,87 @@ def type2stan(obj: model.Documentable) -> Optional[Tag]: return safe_to_stan(parsed_type, _linker, obj, fallback=colorized_pyval_fallback, section='annotation') +_T = TypeVar('_T') +def _memoize(o:object, attrname:str, getter:Callable[[], _T]) -> _T: + parsed = getattr(o, attrname, None) + if parsed is not None: + return parsed #type:ignore + parsed = getter() + setattr(o, attrname, parsed) + return parsed + def get_parsed_type(obj: model.Documentable) -> Optional[ParsedDocstring]: """ Get the type of this attribute as parsed docstring. """ - parsed_type = obj.parsed_type - if parsed_type is not None: - return parsed_type - - # Only Attribute instances have the 'annotation' attribute. - annotation: Optional[ast.expr] = getattr(obj, 'annotation', None) - if annotation is not None: - parsed_type = colorize_inline_pyval(annotation) - # cache parsed_type attribute - obj.parsed_type = parsed_type - return parsed_type + def _get_parsed_type() -> Optional[ParsedDocstring]: + annotation = getattr(obj, 'annotation', None) + if annotation is not None: + v = colorize_inline_pyval(annotation) + reportWarnings(obj, v.warnings, section='colorize annotation') + return v + return None + return _memoize(obj, 'parsed_type', _get_parsed_type) - return None +def get_parsed_decorators(obj: Union[model.Attribute, model.Function, + model.FunctionOverload]) -> Optional[Sequence[ParsedDocstring]]: + """ + Get the decorators of this function as parsed docstring. + """ + def _get_parsed_decorators() -> Optional[Sequence[ParsedDocstring]]: + v = [colorize_inline_pyval(dec) for dec in obj.decorators] if \ + obj.decorators is not None else None + for c in v or (): + reportWarnings(obj, c.warnings, section='colorize decorators') + return v + return _memoize(obj, 'parsed_decorators', _get_parsed_decorators) + +def get_parsed_value(obj:model.Attribute) -> Optional[ParsedDocstring]: + """ + Get the value of this constant as parsed docstring. + """ + def _get_parsed_value() -> Optional[ParsedDocstring]: + v = colorize_pyval(obj.value, + linelen=obj.system.options.pyvalreprlinelen, + maxlines=obj.system.options.pyvalreprmaxlines) if obj.value is not None else None + # Report eventual warnings. + reportWarnings(obj, v.warnings, section='colorize constant') + return v + return _memoize(obj, 'parsed_value', _get_parsed_value) + +def get_parsed_annotations(obj:model.Function) -> Mapping[str, Optional[ParsedDocstring]]: + """ + Get the annotations of this function as dict from str to parsed docstring. + """ + def _get_parsed_annotations() -> Mapping[str, Optional[ParsedDocstring]]: + return {name:colorize_inline_pyval(ann) if ann else None for \ + (name, ann) in obj.annotations.items()} + # do not warn here + return _memoize(obj, 'parsed_annotations', _get_parsed_annotations) + +def get_parsed_bases(obj:model.Class) -> Sequence[ParsedDocstring]: + """ + Get the bases of this class as a seqeunce of parsed docstrings. + """ + def _get_parsed_bases() -> Sequence[ParsedDocstring]: + r = [] + for (str_base, base_node), base_obj in zip(obj.rawbases, obj.baseobjects): + # Make sure we bypass the linker’s resolver process for base object, + # because it has been resolved already (with two passes). + # Otherwise, since the class declaration wins over the imported names, + # a class with the same name as a base class confused pydoctor and it would link + # to it self: https://github.com/twisted/pydoctor/issues/662 + refmap = None + if base_obj is not None: + refmap = {str_base:base_obj.fullName()} + + # link to external class, using the colorizer here + # to link to classes with generics (subscripts and other AST expr). + p = colorize_inline_pyval(base_node, refmap=refmap) + r.append(p) + reportWarnings(obj, p.warnings, section='colorize bases') + return r + return _memoize(obj, 'parsed_bases', _get_parsed_bases) def format_toc(obj: model.Documentable) -> Optional[Tag]: # Load the parsed_docstring if it's not already done. @@ -990,23 +1055,19 @@ def colorized_pyval_fallback(_: List[ParseError], doc:ParsedDocstring, __:model. return Tag('code')(node2stan.gettext(doc.to_node())) def _format_constant_value(obj: model.Attribute) -> Iterator["Flattenable"]: - + doc = get_parsed_value(obj) + if doc is None: + return + # yield the table title, "Value" row = tags.tr(class_="fieldStart") row(tags.td(class_="fieldName")("Value")) # yield the first row. yield row - doc = colorize_pyval(obj.value, - linelen=obj.system.options.pyvalreprlinelen, - maxlines=obj.system.options.pyvalreprmaxlines) - value_repr = safe_to_stan(doc, obj.docstring_linker, obj, fallback=colorized_pyval_fallback, section='rendering of constant') - # Report eventual warnings. It warns when a regex failed to parse. - reportWarnings(obj, doc.warnings, section='colorize constant') - # yield the value repr. row = tags.tr() row(tags.td(tags.pre(class_='constant-value')(value_repr))) @@ -1243,23 +1304,34 @@ def transform_parsed_names(node:'model.Module') -> None: for p in ob.signature.parameters.values(): ann = p.annotation if p.annotation is not inspect.Parameter.empty else None if isinstance(ann, astbuilder._ValueFormatter): - _apply_reference_transform(ann._colorized, ob, is_annotation=True) + _apply_reference_transform(ann.parsed, ob, is_annotation=True) default = p.default if p.default is not inspect.Parameter.empty else None if isinstance(default, astbuilder._ValueFormatter): - _apply_reference_transform(default._colorized, ob) - # TODO: resolve function's annotations, they are currently presented twice - # we can only change signature, annotations in param table must be handled by - # introducing attribute parsed_annotations + _apply_reference_transform(default.parsed, ob) + for _,ann in get_parsed_annotations(ob).items(): + if ann: + _apply_reference_transform(ann, ob, is_annotation=True) + for dec in get_parsed_decorators(ob) or (): + if dec: + _apply_reference_transform(dec, ob) + for overload in ob.overloads: + for dec in get_parsed_decorators(overload) or (): + if dec: + _apply_reference_transform(dec, ob) elif isinstance(ob, model.Attribute): # resolve attribute annotation with parsed_type attribute parsed_type = get_parsed_type(ob) if parsed_type: _apply_reference_transform(parsed_type, ob, is_annotation=True) - # TODO: resolve parsed_value - # TODO: resolve parsed_decorators + if ob.kind in ob.system.show_attr_value: + parsed_value = get_parsed_value(ob) + if parsed_value: + _apply_reference_transform(parsed_value, ob) + for dec in get_parsed_decorators(ob) or (): + if dec: + _apply_reference_transform(dec, ob) elif isinstance(ob, model.Class): - # TODO: resolve parsed_class_signature - # TODO: resolve parsed_decorators - pass + for base in get_parsed_bases(ob): + _apply_reference_transform(base, ob) # do one test with parsed type docstrings \ No newline at end of file diff --git a/pydoctor/model.py b/pydoctor/model.py index 1e0bae1f9..a52232dff 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -646,7 +646,9 @@ def setup(self) -> None: """ self._initialbases: List[str] = [] self._initialbaseobjects: List[Optional['Class']] = [] - + + self.parsed_bases:Optional[List[ParsedDocstring]] = None + def _init_mro(self) -> None: """ Compute the correct value of the method resolution order returned by L{mro()}. @@ -843,6 +845,8 @@ def setup(self) -> None: self.kind = DocumentableKind.METHOD self.signature = None self.overloads = [] + self.parsed_decorators:Optional[Sequence[ParsedDocstring]] = None + self.parsed_annotations:Optional[Dict[str, Optional[ParsedDocstring]]] = None @attr.s(auto_attribs=True) class FunctionOverload: @@ -852,6 +856,7 @@ class FunctionOverload: primary: Function signature: Signature decorators: Sequence[ast.expr] + parsed_decorators:Optional[Sequence[ParsedDocstring]] = None class Attribute(Inheritable): kind: Optional[DocumentableKind] = DocumentableKind.ATTRIBUTE @@ -863,6 +868,8 @@ class Attribute(Inheritable): None value means the value is not initialized at the current point of the the process. """ + parsed_decorators:Optional[Sequence[ParsedDocstring]] = None + parsed_value:Optional[ParsedDocstring] = None # Work around the attributes of the same name within the System class. _ModuleT = Module diff --git a/pydoctor/templatewriter/pages/__init__.py b/pydoctor/templatewriter/pages/__init__.py index 272239605..68fd5425f 100644 --- a/pydoctor/templatewriter/pages/__init__.py +++ b/pydoctor/templatewriter/pages/__init__.py @@ -50,7 +50,8 @@ def format_decorators(obj: Union[model.Function, model.Attribute, model.Function # primary function for parts that requires an interface to Documentable methods or attributes documentable_obj = obj if not isinstance(obj, model.FunctionOverload) else obj.primary - for dec in obj.decorators or (): + for dec, doc in zip(obj.decorators or (), + epydoc2stan.get_parsed_decorators(obj) or ()): if isinstance(dec, ast.Call): fn = node2fullname(dec.func, documentable_obj) # We don't want to show the deprecated decorator; @@ -58,15 +59,9 @@ def format_decorators(obj: Union[model.Function, model.Attribute, model.Function if fn in ("twisted.python.deprecate.deprecated", "twisted.python.deprecate.deprecatedProperty"): break - - # Colorize decorators! - doc = colorize_inline_pyval(dec) stan = epydoc2stan.safe_to_stan(doc, documentable_obj.docstring_linker, documentable_obj, fallback=epydoc2stan.colorized_pyval_fallback, section='rendering of decorators') - - # Report eventual warnings. It warns when we can't colorize the expression for some reason. - epydoc2stan.reportWarnings(documentable_obj, doc.warnings, section='colorize decorator') yield '@', stan.children, tags.br() def format_signature(func: Union[model.Function, model.FunctionOverload]) -> "Flattenable": @@ -90,29 +85,17 @@ def format_class_signature(cls: model.Class) -> "Flattenable": """ r: List["Flattenable"] = [] # the linker will only be used to resolve the generic arguments of the base classes, - # it won't actually resolve the base classes (see comment few lines below). + # it won't actually resolve the base classes (see comment in epydoc2stan.get_parsed_bases). # this is why we're using the annotation linker. _linker = linker._AnnotationLinker(cls) - if cls.rawbases: + parsed_bases = epydoc2stan.get_parsed_bases(cls) + if parsed_bases: r.append('(') - for idx, ((str_base, base_node), base_obj) in enumerate(zip(cls.rawbases, cls.baseobjects)): + for idx, parsed_base in enumerate(parsed_bases): if idx != 0: r.append(', ') - - # Make sure we bypass the linker’s resolver process for base object, - # because it has been resolved already (with two passes). - # Otherwise, since the class declaration wins over the imported names, - # a class with the same name as a base class confused pydoctor and it would link - # to it self: https://github.com/twisted/pydoctor/issues/662 - - refmap = None - if base_obj is not None: - refmap = {str_base:base_obj.fullName()} - - # link to external class, using the colorizer here - # to link to classes with generics (subscripts and other AST expr). - stan = epydoc2stan.safe_to_stan(colorize_inline_pyval(base_node, refmap=refmap), _linker, cls, + stan = epydoc2stan.safe_to_stan(parsed_base, _linker, cls, fallback=epydoc2stan.colorized_pyval_fallback, section='rendering of class signature') r.extend(stan.children) diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 8d802354e..ea72bc824 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -2294,13 +2294,15 @@ def test_reparented_builtins_confusion() -> None: declares a name shadowing a builtin. """ src = ''' - class C: + class C(int): var: list C = print('one') + @stuff(auto=object) + def __init__(self, v:bytes=bytes): + "L{str}" ''' top = ''' - list = object - print = partial(print, flush=True) + list = object = int = print = str = bytes = True from src import C __all__=["C"] @@ -2311,9 +2313,17 @@ class C: builder.addModuleString(top, modname='top') builder.buildModules() clsvar = system.allobjects['top.C.var'] + C = system.allobjects['top.C'] + Ci = system.allobjects['top.C.C'] + __init__ = system.allobjects['top.C.__init__'] assert 'refuri="builtins.list"' in clsvar.parsed_type.to_node().pformat() #type: ignore - # does not work for constant values at the moment + assert 'refuri="builtins.print"' in Ci.parsed_value.to_node().pformat() #type: ignore + assert 'refuri="builtins.int"' in C.parsed_bases[0].to_node().pformat() #type: ignore + assert 'refuri="builtins.object"' in __init__.parsed_decorators[0].to_node().pformat() #type: ignore + assert 'refuri="builtins.bytes"' in __init__.signature.parameters['v'].default.parsed.to_node().pformat() #type: ignore + assert 'refuri="builtins.bytes"' in __init__.signature.parameters['v'].annotation.parsed.to_node().pformat() #type: ignore + assert 'refuri="builtins.bytes"' in __init__.parsed_annotations['v'].to_node().pformat() #type: ignore def test_link_resolving_unbound_names() -> None: """ @@ -2331,4 +2341,8 @@ class C: assert 'refuri="builtins.list"' in clsvar.parsed_type.to_node().pformat() #type: ignore assert 'refuri="unknown"' in clsvar.parsed_type.to_node().pformat() #type: ignore - # does not work for constant values at the moment \ No newline at end of file + # does not work for constant values at the moment + +# what to do with inherited documentation of reparented class attribute part of an +# import cycle? We can't set the value of parsed_docstring from the astbuilder because +# we havnen't resolved the mro yet. \ No newline at end of file From e39b77198685202070899fb1cc43410fb2be1c7a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 13 Jul 2023 13:59:29 -0400 Subject: [PATCH 10/12] Add tests --- pydoctor/test/test_epydoc2stan.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index ea72bc824..7bfaf4511 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -2324,6 +2324,8 @@ def __init__(self, v:bytes=bytes): assert 'refuri="builtins.bytes"' in __init__.signature.parameters['v'].default.parsed.to_node().pformat() #type: ignore assert 'refuri="builtins.bytes"' in __init__.signature.parameters['v'].annotation.parsed.to_node().pformat() #type: ignore assert 'refuri="builtins.bytes"' in __init__.parsed_annotations['v'].to_node().pformat() #type: ignore + assert __init__.parsed_docstring is None # should not be none, actually :/ + # assert 'refuri="builtins.bytes"' in __init__.parsed_docstring.to_node().pformat() #type: ignore def test_link_resolving_unbound_names() -> None: """ @@ -2343,6 +2345,29 @@ class C: assert 'refuri="unknown"' in clsvar.parsed_type.to_node().pformat() #type: ignore # does not work for constant values at the moment +def test_reference_transform_in_type_docstring() -> None: + """ + It will fail with ParsedTypeDocstring at the moment. + """ + src = ''' + __docformat__='google' + class C: + """ + Args: + a (list): the list + """ + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='src') + builder.addModuleString('from src import C;__all__=["C"];list=True', modname='top') + builder.buildModules() + clsvar = system.allobjects['top.C'] + + with pytest.raises(NotImplementedError): + assert 'refuri="builtins.list"' in clsvar.parsed_docstring.fields[1].body().to_node().pformat() #type: ignore + # what to do with inherited documentation of reparented class attribute part of an # import cycle? We can't set the value of parsed_docstring from the astbuilder because -# we havnen't resolved the mro yet. \ No newline at end of file +# we havnen't resolved the mro yet. + From f0f694d4d05d4a142cbed89ff420970d80e9fabd Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 13 Jul 2023 14:02:31 -0400 Subject: [PATCH 11/12] Fix mypy --- pydoctor/epydoc2stan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index 1da415e46..a85c9ed69 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -910,8 +910,10 @@ def get_parsed_decorators(obj: Union[model.Attribute, model.Function, def _get_parsed_decorators() -> Optional[Sequence[ParsedDocstring]]: v = [colorize_inline_pyval(dec) for dec in obj.decorators] if \ obj.decorators is not None else None + documentable_obj = obj if not isinstance(obj, model.FunctionOverload) else obj.primary for c in v or (): - reportWarnings(obj, c.warnings, section='colorize decorators') + if c: + reportWarnings(documentable_obj, c.warnings, section='colorize decorators') return v return _memoize(obj, 'parsed_decorators', _get_parsed_decorators) From dd2cf17a911a27b64417f073506082654df760bb Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 13 Jul 2023 14:06:15 -0400 Subject: [PATCH 12/12] fix static checks --- pydoctor/epydoc2stan.py | 5 ++--- pydoctor/templatewriter/pages/__init__.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index a85c9ed69..6d7371966 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -4,7 +4,6 @@ from collections import defaultdict import enum -from functools import partial import inspect import builtins from itertools import chain @@ -12,7 +11,6 @@ TYPE_CHECKING, Any, Callable, ClassVar, DefaultDict, Dict, Generator, Iterator, List, Mapping, Optional, Sequence, Tuple, TypeVar, Union, ) -import ast import re import attr @@ -926,7 +924,8 @@ def _get_parsed_value() -> Optional[ParsedDocstring]: linelen=obj.system.options.pyvalreprlinelen, maxlines=obj.system.options.pyvalreprmaxlines) if obj.value is not None else None # Report eventual warnings. - reportWarnings(obj, v.warnings, section='colorize constant') + if v: + reportWarnings(obj, v.warnings, section='colorize constant') return v return _memoize(obj, 'parsed_value', _get_parsed_value) diff --git a/pydoctor/templatewriter/pages/__init__.py b/pydoctor/templatewriter/pages/__init__.py index 68fd5425f..77c737b65 100644 --- a/pydoctor/templatewriter/pages/__init__.py +++ b/pydoctor/templatewriter/pages/__init__.py @@ -17,7 +17,6 @@ from pydoctor.templatewriter import util, TemplateLookup, TemplateElement from pydoctor.templatewriter.pages.table import ChildTable from pydoctor.templatewriter.pages.sidebar import SideBar -from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval if TYPE_CHECKING: from typing_extensions import Final