diff --git a/.gitignore b/.gitignore index 312718ab..9e1c0653 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ Manifest.toml *.jl.cov *.jl.*.cov *.jl.mem +LocalPreferences.toml diff --git a/Project.toml b/Project.toml index 26e74b7a..58ceed0d 100644 --- a/Project.toml +++ b/Project.toml @@ -7,15 +7,18 @@ version = "2.7.9" CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" FoldingTrees = "1eca21be-9b9b-4ed8-839a-6d8ae26b1781" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" Preferences = "21216c6a-2e73-6563-6e65-726566657250" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c" +TypedSyntax = "d265eb64-f81a-44ad-a842-4247ee1503de" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [compat] CodeTracking = "0.5, 1" FoldingTrees = "1" +JuliaSyntax = "0.3.2" Preferences = "1" SnoopPrecompile = "1" julia = "1.7" diff --git a/README.md b/README.md index f4966c9b..da418bd1 100644 --- a/README.md +++ b/README.md @@ -8,35 +8,27 @@ :warning: The latest stable version is only compatible with Julia v1.7 and higher. Cthulhu can help you debug type inference issues by recursively showing the -`code_typed` output until you find the exact point where inference gave up, -messed up, or did something unexpected. Using the Cthulhu interface you can +type-inferred code until you find the exact point where inference gave up, +messed up, or did something unexpected. Using the Cthulhu interface, you can debug type inference problems faster. -Looking at type-inferred code can be a bit daunting initially, but you grow more -comfortable with practice. Consider starting with a [tutorial on "lowered" representation](https://juliadebug.github.io/JuliaInterpreter.jl/stable/ast/), -which introduces most of the new concepts. Type-inferrred code differs mostly -by having additional type annotation and (depending on whether you're looking -at optimized or non-optimized code) may incorporate inlining and other fairly -significant transformations of the original code as written by the programmer. - Cthulhu's main tool, `descend`, can be invoked like this: ```julia -descend(f, tt) # function and argument types +descend(f, tt) # function `f` and Tuple `tt` of argument types @descend f(args) # normal call ``` -`descend` allows you to interactively explore the output of -`code_typed` by descending into `invoke` and `call` statements. (`invoke` -statements correspond to static dispatch, whereas `call` statements correspond -to dynamic dispatch.) Press enter to select an `invoke` or `call` to descend -into, select ↩ to ascend, and press q or control-c to quit. - -### JuliaCon 2019 Talk and Demo -[Watch on YouTube](https://www.youtube.com/watch?v=qf9oA09wxXY) -[![Click to watch video](https://img.youtube.com/vi/qf9oA09wxXY/0.jpg)](https://www.youtube.com/watch?v=qf9oA09wxXY) - -The version of Cthulhu in the demo is a little outdated, without the newest features, but largely it has not changed too much. +`descend` allows you to interactively explore the type-annotated source +code by descending into the callees of `f`. +Press enter to select a call to descend into, select ↩ to ascend, +and press q or control-c to quit. +You can also toggle various aspect of the view, for example to suppress +"type-stable" (concretely inferred) annotations or view non-concrete +types in red. +Currently-active options are highlighted with color; press the corresponding +key to toggle these options. Below we walk through a simple example of +these interactive features. ## Usage: descend @@ -46,10 +38,45 @@ function foo() sum(rand(T, 100)) end -descend(foo, Tuple{}) -@descend foo() +descend(foo, Tuple{}) # option 1: specify by function name and argument types +@descend foo() # option 2: apply `@descend` to a working execution of the function ``` +If you do this, you'll see quite a bit of text output. Let's break it down and +see it section-by-section. At the top, you may see something like this: + +![source-section-all](images_readme/descend_source_show_all.png) + +This shows your original source code (together with line numbers, which here were in the REPL). +The cyan annotations are the types of the variables: `Union{Float64, Int64}` means "either a `Float64` +or an `Int64`". +Small *concrete* unions (where all the possibilities are known exactly) are generally are not a problem +for type inference, unless there are so many that Julia stops trying to work +out all the different combinations (see [this blog post](https://julialang.org/blog/2018/08/union-splitting/) +for more information). + +In the next section you may see something like + +![toggles](images_readme/descend_toggles.png) + +This section shows you some interactive options you have for controlling the display. +Normal text inside `[]` generally indicates "off", and color is used for "on" or specific options. +For example, if you hit `w` to turn on warnings, now you should see something like this: + +![warn](images_readme/descend_source_toggles_warn.png) + +Now you can see small concrete unions in yellow, and concretely inferred code in cyan. +Serious forms of poor inferrability are colored in red (of which there are none in this example); +these generally hurt runtime performance and may make compiled code more vulnerable to being invalidated. + +In the final section, you see: + +![calls](images_readme/descend_calls.png) + +This is a menu of calls that you can further descend into. Move the dot `•` with the up and down +arrow keys, and hit Enter to descend into a particular call. + + ## Methods: descend - `@descend_code_typed` @@ -117,7 +144,7 @@ The calls that appear on the same line separated by `=>` represent inlined metho you enter at the final (topmost) call on that line. By default, -- `descend` views optimized code without "warn" coloration of types +- `descend` views non-optimized code without "warn" coloration of types - `ascend` views non-optimized code with "warn" coloration You can toggle between these with `o` and `w`. @@ -152,29 +179,19 @@ Then invoke: Cthulhu.@descend foo(5) ``` -Now, descend: - -``` -%22 = call bar(::Union{Float64, Int64},::Union{Float64, Int64},::Union{Float64, Int64})::String -``` - -which shows (after typing `w`) +Now, descend into `bar`: move the cursor down (or wrap around by hitting the up arrow) until +the dot is next to the `bar` call: ``` -∘ ── %0 = invoke bar(::Union{Float64, Int64},::Union{Float64, Int64},::Union{Float64, Int64})::String -Variables - #self#::Core.Const(bar) - x::Union{Float64, Int64} - y::Union{Float64, Int64} - z::Union{Float64, Int64} -[...] + ⋮ + 4 (4.5 * n::Int64)::Float64 + • 6 bar(x, y, z) + ↩ ``` -The text of `Union{Float64, Int64}` will be colored in red indicating there are type-instabilities, -but they are unlikely to be problem in actual execution, -because `bar` here serves as a ["function barrier"](https://docs.julialang.org/en/v1/manual/performance-tips/#kernel-functions) and -`bar` will be called with fully concrete runtime types via dynamic dispatch. +and then hit Enter. Then you will see the code for `bar` with its type annotations. +Notice that many variables are annotated as `Union`. To give Cthulhu more complete type information, we have to actually run some Julia code. There are many ways to do this. In this example, we use [`Infiltrator.jl`](https://github.com/JuliaDebug/Infiltrator.jl). Add an `@infiltrate`: @@ -203,21 +220,35 @@ Infiltrating foo(n::Int64) at ex.jl:10: infil> ``` -Enter `@descend bar(x, y, z)` and type `w`: +Enter `@descend bar(x, y, z)` you can see that, for `foo(4)`, the types within `bar` are fully inferred. -``` -infil> @descend bar(x, y, z) - -∘ ── %0 = invoke bar(::Float64,::Float64,::Int64)::String -Variables - #self#::Core.Const(bar) - x::Float64 - y::Float64 - z::Int64 -[...] -``` +## Viewing the internal representation of Julia code + +Anyone using Cthulhu to investigate the behavior of Julia's compiler will +prefer to examine the +While Cthulhu tries to place type-annotations on the source code, this obscures +detail and can occassionally go awry (see details [here](TypedSyntax/README.md)). +For anyone who needs more direct insight, it can be better to look directly at Julia's +internal representations of type-inferred code. +Looking at type-inferred code can be a bit daunting initially, but you grow more +comfortable with practice. Consider starting with a +[tutorial on "lowered" representation](https://juliadebug.github.io/JuliaInterpreter.jl/stable/ast/), +which introduces most of the new concepts. Type-inferrred code differs from +lowered representation by having additional type annotation. +Moreover, `call` statements that can be inferred are converted to `invoke`s +(these correspond to static dispatch), whereas dynamic dispatch is indicated by the +remaining `call` statements. +Depending on whether you're looking at optimized or non-optimized code, +it may also incorporate inlining and other fairly significant transformations +of the original code as written by the programmer. + +This video demonstrates Cthulhu for viewing "raw" type-inferred code: +[Watch on YouTube](https://www.youtube.com/watch?v=qf9oA09wxXY) +[![Click to watch video](https://img.youtube.com/vi/qf9oA09wxXY/0.jpg)](https://www.youtube.com/watch?v=qf9oA09wxXY) + +The version of Cthulhu in the demo is a little outdated, without the newest features, +but may still be relevant for users who want to view code at this level of detail. -You can see that, for `foo(4)`, the types within `bar` are fully inferred. ## Customization diff --git a/TypedSyntax/LICENSE b/TypedSyntax/LICENSE new file mode 100644 index 00000000..0f26201c --- /dev/null +++ b/TypedSyntax/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Tim Holy and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/TypedSyntax/Project.toml b/TypedSyntax/Project.toml new file mode 100644 index 00000000..d9e675d2 --- /dev/null +++ b/TypedSyntax/Project.toml @@ -0,0 +1,19 @@ +name = "TypedSyntax" +uuid = "d265eb64-f81a-44ad-a842-4247ee1503de" +authors = ["Tim Holy and contributors"] +version = "0.1.0" + +[deps] +CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" + +[compat] +CodeTracking = "1" +JuliaSyntax = "0.3.2" +julia = "1" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/TypedSyntax/README.md b/TypedSyntax/README.md new file mode 100644 index 00000000..84659f46 --- /dev/null +++ b/TypedSyntax/README.md @@ -0,0 +1,133 @@ +# TypedSyntax + +This package aims to map types, as determined via type-inference, back to the source code as written by the developer. It can be used to understand program behavior and identify causes of "type instability" (inference failures) without the need to read [intermediate representations](https://docs.julialang.org/en/v1/devdocs/ast/) of Julia code. + +This package is built on [JuliaSyntax](https://github.com/JuliaLang/JuliaSyntax.jl) and extends it by attaching type annotations to the nodes of its syntax trees. Here's a demo: + +```julia +julia> using TypedSyntax + +julia> f(x, y, z) = x + y * z; + +julia> node = TypedSyntaxNode(f, (Float64, Int, Float32)) +line:col│ tree │ type + 1:1 │[=] │Float64 + 1:1 │ [call] + 1:1 │ f + 1:3 │ x │Float64 + 1:6 │ y │Int64 + 1:9 │ z │Float32 + 1:13 │ [call-i] │Float64 + 1:14 │ x │Float64 + 1:16 │ + + 1:17 │ [call-i] │Float32 + 1:18 │ y │Int64 + 1:20 │ * + 1:22 │ z │Float32 +``` + +The right hand column is the new information added by `TypedSyntaxNode`, indicating the type assigned to each variable or function call. + +You can also display this in a form closer to the original source code, but with type-annotations: + +```julia +julia> printstyled(stdout, node; hide_type_stable=false) +f(x::Float64, y::Int64, z::Float32)::Float64 = (x::Float64 + (y::Int64 * z::Float32)::Float32)::Float64 +``` + +`hide_type_stable=true` (which is the default) will suppress printing of concrete types, so you need to set it to `false` if you want to see all the types. + +The default is aimed at identifying sources of "type instability" (poor inferrability): + +```julia +julia> printstyled(stdout, TypedSyntaxNode(f, (Float64, Int, Real))) +``` + +which produces + +f(x, y, z::Real)::Any = (x + (y * z::Real)::Any)::Any + +The boldfaced text above is typically printed in color in the REPL: + +- red indicates non-concrete types +- yellow indicates a "small union" of concrete types. These usually pose no issues, unless there are too many combinations of such unions. + +Printing with color can be suppressed with the keyword argument `iswarn=false`. + +## Caveats + +TypedSyntax aims for accuracy, but there are a number of factors that pose challenges. +First, anonymous and internal functions appear as part of the source text, but internally Julia handles these as separate type-inferred methods, and these are hidden from the annotator. +Therefore, in + +```julia +julia> sumfirst(c) = sum(x -> first(x), c); # better to use `sum(first, c)` but this is just an illustration + +julia> printstyled(stdout, TypedSyntaxNode(sumfirst, (Vector{Any},))) +sumfirst(c)::Any = sum(x -> first(x), c)::Any +``` + +`x` and `first(x)` both have type `Any`, but they are not annotated as such because they are hidden inside the anonymous function. + +Second, this package works by attempting to "reconstruct history": starting from the type-inferred code, it tries to map calls back to the source. It would be much safer to instead keep track of the source during inference, but at present this is not possible (see [this Julia issue](https://github.com/JuliaLang/julia/issues/31162)). There are cases where this mapping fails: for example, with + +```julia +julia> function summer(list) + s = 0 + for x in list + s += x + end + return s + end; +``` +then (on Julia 1.9) +``` +julia> tsn, mappings = TypedSyntax.tsn_and_mappings(summer, (Vector{Float64},)); + +julia> hcat(tsn.typedsource.code, mappings) +16×2 Matrix{Any}: + :(_4 = 0) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(_2) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[list] + :(_3 = Base.iterate(%2)) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[(= x list)] + :(_3 === nothing) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(Base.not_int(%4)) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(goto %16 if not %5) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(_3::Tuple{Float64, Int64}) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(_5 = Core.getfield(%7, 1)) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(Core.getfield(%7, 2)) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(_4 = _4 + _5) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[(+= s x)] + :(_3 = Base.iterate(%2, %9)) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(_3 === nothing) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(Base.not_int(%12)) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(goto %16 if not %13) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(goto %7) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[] + :(return _4) Union{TreeNode{SyntaxData}, TreeNode{TypedSyntaxData}}[s] +``` +The left column contains the statements of the type-inferred code, the right column the mappings back to the source. +You can see that the majority of these mappings are empty, indicating either no good match or that there were multiple possible matches. This is because lowering changes the implementation so significantly that there are few calls that relate directly to the source. + +Nevertheless, many statements in the source can be annotated: + +```julia +julia> tsn +line:col│ tree │ type + 1:1 │[function] │Union{Float64, Int64} + 1:10 │ [call] + 1:10 │ summer + 1:17 │ list │Vector{Float64} + 1:22 │ [block] + 2:5 │ [=] + 2:5 │ s + 2:9 │ 0 + 3:5 │ [for] + 3:8 │ [=] │Union{Nothing, Tuple{Float64, Int64}} + 3:9 │ x + 3:14 │ list │Vector{Float64} + 3:18 │ [block] + 4:9 │ [+=] │Float64 + 4:9 │ s │Float64 + 4:14 │ x │Float64 + 6:5 │ [return] │Union{Float64, Int64} + 6:12 │ s │Union{Float64, Int64} +``` +This is largely because just the named-variables provide considerable information. diff --git a/TypedSyntax/src/TypedSyntax.jl b/TypedSyntax/src/TypedSyntax.jl new file mode 100644 index 00000000..1c33325c --- /dev/null +++ b/TypedSyntax/src/TypedSyntax.jl @@ -0,0 +1,16 @@ +module TypedSyntax + +using Core: CodeInfo, MethodInstance, SlotNumber, SSAValue +using Core.Compiler: TypedSlot +using JuliaSyntax: JuliaSyntax, TreeNode, AbstractSyntaxData, SyntaxData, SyntaxNode, GreenNode, AbstractSyntaxNode, SyntaxHead, SourceFile, + head, kind, child, children, haschildren, untokenize, first_byte, last_byte, source_line, source_location, + sourcetext, @K_str, @KSet_str, is_infix_op_call, is_prefix_op_call, is_prec_assignment, is_operator, is_literal +using Base.Meta: isexpr +using CodeTracking + +export TypedSyntaxNode + +include("node.jl") +include("show.jl") + +end diff --git a/TypedSyntax/src/node.jl b/TypedSyntax/src/node.jl new file mode 100644 index 00000000..9cc270a8 --- /dev/null +++ b/TypedSyntax/src/node.jl @@ -0,0 +1,469 @@ + +mutable struct TypedSyntaxData <: AbstractSyntaxData + source::SourceFile + typedsource::CodeInfo + raw::GreenNode{SyntaxHead} + position::Int + val::Any + typ::Any # can either be a Type or `nothing` +end +TypedSyntaxData(sd::SyntaxData, src::CodeInfo, typ=nothing) = TypedSyntaxData(sd.source, src, sd.raw, sd.position, sd.val, typ) + +const TypedSyntaxNode = TreeNode{TypedSyntaxData} +const MaybeTypedSyntaxNode = Union{SyntaxNode,TypedSyntaxNode} + +struct NotFound end +# struct Unmatched end + +# Call these if you want both the TypedSyntaxNode and the `mappings` list, +# where `mappings[i]` corresponds to `(src::CodeInfo).code[i]`. +function tsn_and_mappings(@nospecialize(f), @nospecialize(t); kwargs...) + m = which(f, t) + src, rt = getsrc(f, t) + tsn_and_mappings(m, src, rt; kwargs...) +end + +function tsn_and_mappings(m::Method, src::CodeInfo, @nospecialize(rt); warn::Bool=true, strip_macros::Bool=false, kwargs...) + def = definition(String, m) + if isnothing(def) + warn && @warn "couldn't retrieve source of $m" + return nothing, nothing + end + sourcetext, lineno = def + rootnode = JuliaSyntax.parse(SyntaxNode, sourcetext; filename=string(m.file), first_line=lineno, kwargs...) + if strip_macros + rootnode = get_function_def(rootnode) + if !is_function_def(rootnode) + warn && @warn "couldn't retrieve source of $m" + return nothing, nothing + end + end + Δline = lineno - m.line # offset from original line number (Revise) + mappings, symtyps = map_ssas_to_source(src, rootnode, Δline) + node = TypedSyntaxNode(rootnode, src, mappings, symtyps) + node.typ = rt + return node, mappings +end + +TypedSyntaxNode(@nospecialize(f), @nospecialize(t); kwargs...) = tsn_and_mappings(f, t; kwargs...)[1] + +TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, Δline::Integer=0) = + TypedSyntaxNode(rootnode, src, map_ssas_to_source(src, rootnode, Δline)...) + +function TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, mappings, symtyps) + # There may be ambiguous assignments back to the source; preserve just the unambiguous ones + node2ssa = IdDict{SyntaxNode,Int}(only(list) => i for (i, list) in pairs(mappings) if length(list) == 1) + # Copy `rootnode`, adding type annotations + trootnode = TreeNode(nothing, nothing, TypedSyntaxData(rootnode.data, src, gettyp(node2ssa, rootnode, src))) + addchildren!(trootnode, rootnode, src, node2ssa, symtyps, mappings) + # Add argtyps to signature + fnode = get_function_def(trootnode) + if is_function_def(fnode) + sig, body = children(fnode) + if kind(sig) == K"where" + sig = child(sig, 1) + end + @assert kind(sig) == K"call" + i = 1 + for arg in Iterators.drop(children(sig), 1) + kind(arg) == K"parameters" && break # kw args + if kind(arg) == K"..." + arg = only(children(arg)) + end + if kind(arg) == K"::" + nchildren = length(children(arg)) + if nchildren == 1 + # unnamed argument + found = false + while i <= length(src.slotnames) + if src.slotnames[i] == Symbol("#unused#") + arg.typ = src.slottypes[i] + i += 1 + found = true + break + end + i += 1 + end + @assert found + continue + elseif nchildren == 2 + arg = child(arg, 1) # extract the name + else + error("unexpected number of children: ", children(arg)) + end + end + @assert kind(arg) == K"Identifier" + argname = arg.val + while i <= length(src.slotnames) + if src.slotnames[i] == argname + arg.typ = src.slottypes[i] + i += 1 + break + end + i += 1 + end + end + end + # foreach(mappings) do mapped + # if any(n -> !isa(n, TypedSyntaxNode), mapped) + # display(src.parent) + # display(rootnode) + # error("hoped that all would be typed, got ", mapped) + # end + # end + return trootnode +end + +function addchildren!(tparent, parent, src::CodeInfo, node2ssa, symtyps, mappings) + if haschildren(parent) && tparent.children === nothing + tparent.children = TypedSyntaxNode[] + end + for child in children(parent) + tnode = TreeNode(tparent, nothing, TypedSyntaxData(child.data, src, gettyp(node2ssa, child, src))) + if tnode.typ === nothing && kind(child) == K"Identifier" + tnode.typ = get(symtyps, child, nothing) + end + push!(tparent, tnode) + addchildren!(tnode, child, src, node2ssa, symtyps, mappings) + end + # In `return f(args..)`, copy any types assigned to `f(args...)` up to the `[return]` node + if kind(tparent) == K"return" && haschildren(tparent) + tparent.typ = only(children(tparent)).typ + end + # Replace the entry in `mappings` to be the typed node + i = get(node2ssa, parent, nothing) + if i !== nothing + @assert length(mappings[i]) == 1 + mappings[i][1] = tparent + end + return tparent +end + +function gettyp(node2ssa, node, src) + i = get(node2ssa, node, nothing) + i === nothing && return nothing + stmt = src.code[i] + if isa(stmt, Core.ReturnNode) + arg = stmt.val + isa(arg, SSAValue) && return src.ssavaluetypes[arg.id] + is_slot(arg) && return src.slottypes[arg.id] + end + return src.ssavaluetypes[i] +end + +Base.copy(tsd::TypedSyntaxData) = TypedSyntaxData(tsd.source, tsd.typedsource, tsd.raw, tsd.position, tsd.val, tsd.typ) + +gettyp(node::AbstractSyntaxNode) = gettyp(node.data) +gettyp(::JuliaSyntax.SyntaxData) = nothing +gettyp(data::TypedSyntaxData) = data.typ + +# function find_codeloc(src, lineno) +# clidx = searchsortedfirst(src.linetable, lineno; lt=(linenode, line) -> linenode.line < line) +# clidx = min(clidx, length(src.linetable)) +# if src.linetable[clidx].line > lineno +# clidx -= 1 # handle multiline statements +# end +# return clidx +# end +# function find_coderange(src, lineno) +# clidx = find_codeloc(src, lineno) +# ibegin = searchsortedfirst(src.codelocs, clidx) +# ibegin += src.codelocs[ibegin] > lineno +# iend = searchsortedlast(src.codelocs, clidx) +# return ibegin:iend +# end + +function sparam_name(mi::MethodInstance, i::Int) + sig = (mi.def::Method).sig::UnionAll + while true + i == 1 && break + sig = sig.body::UnionAll + i -= 1 + end + return sig.var.name +end + +# # Get the name of the function being called +# function extract_call_name(stmt, mi)::Union{Symbol,Nothing} +# stmt.head ∈ (:call, :invoke) || return nothing +# f = stmt.args[1] +# if isa(f, GlobalRef) && f.mod === Core && f.name == :_apply_iterate # handle vararg calls +# # Sanity check +# fiter = stmt.args[2] +# @assert isa(fiter, GlobalRef) && fiter.name == :iterate +# f = stmt.args[3] # get the actual call +# end +# if is_slot(f) +# return src.slotnames[f.id] +# end +# if isa(f, SSAValue) +# f = src.ssavaluetypes[f.id] +# if isa(fname, Core.Const) +# f = fname.val +# end +# end +# if isexpr(f, :static_parameter) +# return sparam_name(mi, f.args[1]::Int) +# end +# if isa(f, QuoteNode) +# f = f.val +# end +# isa(f, GlobalRef) && return f.name +# return nothing +# end + +function getsrc(@nospecialize(f), @nospecialize(t)) + srcrts = code_typed(f, t; debuginfo=:source, optimize=false) + return only(srcrts) +end + +function is_function_def(node) # this is not `Base.is_function_def` + kind(node) == K"function" && return true + kind(node) == K"=" && kind(child(node, 1)) ∈ KSet"call where" && return true + return false +end + +function get_function_def(rootnode) + while kind(rootnode) == K"macrocall" + idx = findlast(node -> is_function_def(node) || kind(node) == K"macrocall", children(rootnode)) + idx === nothing && break + rootnode = child(rootnode, idx) + end + return rootnode +end + +function collect_symbol_nodes!(symlocs::AbstractDict, node) + kind(node) == K"->" && return symlocs # skip inner functions (including `do` blocks below) + is_function_def(node) && return symlocs + if kind(node) == K"Identifier" || is_operator(node) + name = node.val + if isa(name, Symbol) + locs = get!(Vector{typeof(node)}, symlocs, name) + push!(locs, node) + end + end + if is_literal(node) && node.val !== nothing # FIXME: distinguish literal `nothing` from source `nothing` + locs = get!(Vector{typeof(node)}, symlocs, node.val) + push!(locs, node) + end + if haschildren(node) + for c in (kind(node) == K"do" ? (child(node, 1),) : children(node)) # process only `g(args...)` in `g(args...) do ... end` + collect_symbol_nodes!(symlocs, c) + end + end + return symlocs +end + +# Find all places in the source code where a symbol is used +function collect_symbol_nodes(rootnode) + rootnode = get_function_def(rootnode) + is_function_def(rootnode) || error("expected function definition, got ", sourcetext(rootnode)) + symlocs = Dict{Any,Vector{typeof(rootnode)}}() + return collect_symbol_nodes!(symlocs, child(rootnode, 2)) +end + +function map_ssas_to_source(src, rootnode, Δline) + # Find all leaf-nodes for a given symbol + symlocs = collect_symbol_nodes(rootnode) # symlocs = Dict(:name => [node1, node2, ...]) + # Initialize the type-assignment of each slot at each use location + symtyps = IdDict{typeof(rootnode),Any}() # symtyps = IdDict(node => typ) + # Initialize the (possibly ambiguous) attributions for each stmt in `src` (`stmt = src.code[i]`) + mappings = [Union{SyntaxNode,TypedSyntaxNode}[] for _ in eachindex(src.code)] # mappings[i] = [node1, node2, ...] + + # Append (to `mapped`) all nodes in `targets` that are consistent with the line number of the `i`th stmt + function append_targets_for_line!(mapped, i, targets) + j = src.codelocs[i] + linerange = src.linetable[j].line + Δline : ( + j < length(src.linetable) ? src.linetable[j+1].line - 1 + Δline : typemax(Int)) + for t in targets + source_line(t) ∈ linerange && push!(mapped, t) + end + return mapped + end + # For a call argument `arg`, find all source statements that match + function append_targets_for_arg!(mapped, i, arg) + targets = if is_slot(arg) + # If `arg` is a variable, e.g., the `x` in `f(x)` + get(symlocs, src.slotnames[arg.id], nothing) # find all places this variable is used + elseif isa(arg, GlobalRef) + get(symlocs, arg.name, nothing) # find all places this name is used + elseif isa(arg, SSAValue) + # If `arg` is the result from a call, e.g., the `g(x)` in `f(g(x))` + mappings[arg.id] + elseif isa(arg, Core.Const) + get(symlocs, arg.val, nothing) # FIXME: distinguish this `nothing` from a literal `nothing` + elseif is_src_literal(arg) + get(symlocs, arg, nothing) # FIXME: distinguish this `nothing` from a literal `nothing` + end + if targets !== nothing + append_targets_for_line!(mapped, i, targets) # select the subset consistent with the line number + end + return mapped + end + + argmapping = typeof(rootnode)[] + for (i, mapped, stmt) in zip(eachindex(mappings), mappings, src.code) + empty!(argmapping) + if is_slot(stmt) || isa(stmt, SSAValue) + append_targets_for_arg!(mapped, i, stmt) + elseif isa(stmt, Core.ReturnNode) + append_targets_for_line!(mapped, i, append_targets_for_arg!(argmapping, i, stmt.val)) + elseif isa(stmt, Expr) + if stmt.head == :(=) + # We defer setting up `symtyps` for the LHS because processing the RHS first might eliminate ambiguities + # # Update `symtyps` for this assignment + lhs = stmt.args[1] + @assert is_slot(lhs) + # For `mappings` we're interested only in the right hand side of this assignment + stmt = stmt.args[2] + if is_slot(stmt) || isa(stmt, SSAValue) # can we just look up the answer? + append_targets_for_arg!(mapped, i, stmt) + length(mapped) > 1 && filter!(mapped) do argnode + if kind(argnode.parent) == K"tuple" + is_prec_assignment(argnode.parent.parent) && any(==(argnode), children(child(argnode.parent.parent, 2))) + else + is_prec_assignment(argnode.parent) && argnode == child(argnode.parent, 2) # rhs + end + end + if length(mapped) == 1 + symtyps[only(mapped)] = is_slot(stmt) ? src.slottypes[stmt.id] : src.ssavaluetypes[stmt.id] + end + append_targets_for_arg!(argmapping, i, lhs) + filter!(argmapping) do argnode + if kind(argnode.parent) == K"tuple" + is_prec_assignment(argnode.parent.parent) && any(==(argnode), children(child(argnode.parent.parent, 1))) + else + is_prec_assignment(argnode.parent) && argnode == child(argnode.parent, 1) + end + end + if length(argmapping) == 1 + symtyps[only(argmapping)] = src.ssavaluetypes[i] + end + empty!(argmapping) + continue + end + isa(stmt, Expr) || continue + # The right hand side was an expression. Fall through to the generic `call` analysis. + end + if stmt.head == :call && is_indexed_iterate(stmt.args[1]) + id = stmt.args[2] + @assert isa(id, SSAValue) + append!(mapped, mappings[id.id]) + continue + end + # When analyzing calls, we start with the symbols. For any that have been attributed to one or more + # nodes in the source, we make a consistency argument: which *parent* nodes take all of these as arguments? + # In many cases this allows unique assignment. + # Let's take a simple example: `x + sin(x + π / 4)`: in this case, `x + ` appears in two places but + # you can disambiguate it by noting that `x + π / 4` only occurs in one place. + # Note that the function name (e.g., `:sin`) is not special, we can effectively treat all as + # `invoke(f, args...)` and consider `f` just like any other argument. + # TODO?: handle gensymmed names, e.g., kw bodyfunctions? + # The advantage of this approach is precision: we don't depend on ordering of statements, + # so when it works you know you are correct. + stmtmapping = Set{typeof(rootnode)}() + for arg in stmt.args + # Collect all source-nodes that use this argument + append_targets_for_arg!(argmapping, i, arg) + if !isempty(argmapping) + if isempty(stmtmapping) + # First matched argument + # For each candidate source-node, push its parent-node into `stmtmapping`. + # The true call-node should be among these. + foreach(argmapping) do t + push!(stmtmapping, t.parent) + end + else + # Second or later matched argument + # The matching caller needs to be used by all `stmt.args`, + # so we `intersect` to find nodes that use all args + intersect!(stmtmapping, map(t->t.parent, argmapping)) + end + end + empty!(argmapping) + end + append!(mapped, stmtmapping) + sort!(mapped; by=t->t.position) # since they went into a set, best to order them within the source + stmt = src.code[i] # re-get the statement so we process slot-assignment + if length(mapped) == 1 && isa(stmt, Expr) + # We've mapped the call uniquely (i.e., we found the right match) + node = only(mapped) + # Final step: set up symtyps for all the user-visible variables + # Because lowering can build methods that take a different number of arguments than appear in the + # source text, don't try to count arguments. Instead, find a symbol that is part of + # `node` or, for the LHS of a `slot = callexpr` statement, one that shares a parent with `node`. + if stmt.head == :(=) + # Tag the LHS of this expression + arg = stmt.args[1] + @assert is_slot(arg) + sym = src.slotnames[arg.id] + if !isempty(string(sym)) + lhsnode = node + while !is_prec_assignment(lhsnode) + lhsnode = lhsnode.parent + end + lhsnode = child(lhsnode, 1) + if kind(lhsnode) == K"tuple" # tuple destructuring + found = false + for child in children(lhsnode) + if kind(child) == K"Identifier" + if child.val == sym + lhsnode = child + found = true + break + end + end + end + @assert found + end + symtyps[lhsnode] = src.ssavaluetypes[i] + end + # Now process the RHS + stmt = stmt.args[2] + end + # Process the call expr + if isa(stmt, Expr) + for arg in stmt.args + # For arguments that are slots, follow them backwards. + # (We're not assigning type to node, we're assigning nodes to ssavalues.) + # Arguments can locally be SSAValues but ultimately map back to slots + j = 0 + while isa(arg, SSAValue) + j = arg.id + arg = src.ssavaluetypes[j] # keep trying in case it maps back to a slot + end + if is_slot(arg) + sym = src.slotnames[arg.id] + sym == Symbol("") && continue + for t in symlocs[sym] + haskey(symtyps, t) && continue + if t.parent == node + is_prec_assignment(node) && t == child(node, 1) && continue + symtyps[t] = if j > 0 + src.ssavaluetypes[j] + else + # We failed to find it as an SSAValue, it must have type assigned at function entry + j = findfirst(==(sym), src.slotnames) + src.slottypes[j] + end + break + end + end + end + end + end + end + end + end + return mappings, symtyps +end + +function is_indexed_iterate(arg) + isa(arg, GlobalRef) || return false + arg.mod == Base || return false + return arg.name == :indexed_iterate +end + +is_slot(@nospecialize(arg)) = isa(arg, SlotNumber) || isa(arg, TypedSlot) + +is_src_literal(x) = isa(x, Integer) || isa(x, AbstractFloat) || isa(x, String) || isa(x, Char) || isa(x, Symbol) diff --git a/TypedSyntax/src/show.jl b/TypedSyntax/src/show.jl new file mode 100644 index 00000000..9d3b9ef4 --- /dev/null +++ b/TypedSyntax/src/show.jl @@ -0,0 +1,165 @@ +function Base.show(io::IO, ::MIME"text/plain", node::TypedSyntaxNode; show_byte_offsets=false) + println(io, "line:col│$(show_byte_offsets ? " byte_range │" : "") tree │ type") + JuliaSyntax._show_syntax_node(io, Ref{Union{Nothing,String}}(nothing), node, "", show_byte_offsets) +end + +function JuliaSyntax._show_syntax_node(io, current_filename, node::TypedSyntaxNode, indent, show_byte_offsets) + fname = node.source.filename + line, col = source_location(node.source, node.position) + posstr = "$(lpad(line, 4)):$(rpad(col,3))│" + if show_byte_offsets + posstr *= "$(lpad(first_byte(node),6)):$(rpad(last_byte(node),6))│" + end + val = node.val + nodestr = haschildren(node) ? "[$(untokenize(head(node)))]" : + isa(val, Symbol) ? string(val) : repr(val) + treestr = string(indent, nodestr) + if node.typ !== nothing + treestr = string(rpad(treestr, 40), "│$(node.typ)") + end + println(io, posstr, treestr) + if haschildren(node) + new_indent = indent*" " + for n in children(node) + JuliaSyntax._show_syntax_node(io, current_filename, n, new_indent, show_byte_offsets) + end + end +end + + +function Base.printstyled(io::IO, rootnode::MaybeTypedSyntaxNode; + type_annotations::Bool=true, iswarn::Bool=true, hide_type_stable::Bool=true, + idxend = last_byte(rootnode), kwargs...) + rt = gettyp(rootnode) + rootnode = get_function_def(rootnode) + position = first_byte(rootnode) - 1 + if is_function_def(rootnode) + # We're printing a MethodInstance + @assert length(children(rootnode)) == 2 + sig, body = children(rootnode) + position = show_src_expr(io, sig, position; type_annotations, iswarn, hide_type_stable) + type_annotations && maybe_show_annotation(io, rt; iswarn, hide_type_stable) + rootnode = body + end + position = show_src_expr(io, rootnode, position; type_annotations, iswarn, hide_type_stable) + println(io, rootnode.source[position+1:idxend]) + return nothing +end +Base.printstyled(rootnode::MaybeTypedSyntaxNode; kwargs...) = printstyled(stdout, rootnode; kwargs...) + +function show_src_expr(io::IO, node::MaybeTypedSyntaxNode, lastidx::Int; type_annotations::Bool=true, iswarn::Bool=false, hide_type_stable::Bool=false) + lastidx = catchup(io, node, lastidx) + _lastidx = last_byte(node) + if kind(node) == K"Identifier" || (kind(node) == K"::" && is_prefix_op_call(node)) + print_with_linenumber(io, node, lastidx+1:_lastidx) + type_annotations && maybe_show_annotation(io, gettyp(node); iswarn, hide_type_stable) + return _lastidx + end + # We only handle "call" nodes. For anything else, just print the node (recursing into children) + if kind(node) ∉ KSet"call ref" + for child in children(node) + lastidx = show_src_expr(io, child, lastidx; type_annotations, iswarn, hide_type_stable) + end + print_with_linenumber(io, node, lastidx+1:_lastidx) + return _lastidx + end + pre = prepost = post = "" + if is_infix_op_call(node) # wrap infix calls in parens before type-annotating + pre, post = "(", ")" + lastidx = catchup(io, first(children(node)), lastidx) + elseif is_prefix_op_call(node) # insert parens after prefix op and before type-annotating + prepost, post = "(", ")" + end + T = gettyp(node) + # should we print a type-annotation? + type_annotate = type_annotations & (isa(T, Vector{Int}) || (isa(T, Type) && (!hide_type_stable || is_type_unstable(T)))) + type_annotate && print(io, pre) + for (childid, child) in enumerate(children(node)) + childid == 2 && type_annotate && print(io, prepost) + lastidx = show_src_expr(io, child, lastidx; type_annotations, iswarn, hide_type_stable) + end + print(io, node.source[lastidx+1:_lastidx]) + if type_annotate && T !== nothing + show_annotation(io, T, post; iswarn) + end + return _lastidx +end + +function maybe_show_annotation(io, @nospecialize(T); iswarn, hide_type_stable) + T === nothing && return + if isa(T, Core.Const) + T = typeof(T.val) + end + if isa(T, Type) && (!hide_type_stable || is_type_unstable(T)) # should we print a type-annotation? + show_annotation(io, T; iswarn) + end +end + +function show_annotation(io, @nospecialize(T), post=""; iswarn::Bool) + print(io, post) + if isa(T, Vector{Int}) + isempty(T) && return + if iswarn + printstyled(io, "::NF"; color=:yellow) + else + print(io, "::NF") + end + return + end + if iswarn + color = !is_type_unstable(T) ? :cyan : + is_small_union_or_tunion(T) ? :yellow : :red + printstyled(io, "::", T; color) + else + printstyled(io, "::", T; color=:cyan) + end +end + +function catchup(io::IO, node::MaybeTypedSyntaxNode, lastidx::Int) + # Do any "overdue" printing now. Mostly, this catches whitespace + firstidx = first_byte(node) + if lastidx + 1 < firstidx + print_with_linenumber(io, node, lastidx+1:firstidx-1) + lastidx = firstidx-1 + end + return lastidx +end + +function print_with_linenumber(io::IO, node::AbstractSyntaxNode, byterange) + nd = ndigits(node.source.first_line + nlines(node.source) - 1) + offset = first(byterange) - 1 + if offset == 0 + # This is the first line, print the line number first + printstyled(io, lpad(source_line(node.source, 1), nd), " "; color=:light_black) + end + for (i, c) in pairs(node.source[byterange]) + print(io, c) + if c == '\n' + printstyled(io, lpad(source_line(node.source, i + offset + 1), nd), " "; color=:light_black) + end + end +end + +nlines(source) = length(source.line_starts) + +is_type_unstable(@nospecialize(type)) = type isa Type && (!Base.isdispatchelem(type) || type == Core.Box) +function is_small_union_or_tunion(@nospecialize(T)) + Base.isvarargtype(T) && return false + if T <: Tuple # is it Tuple{U} + return all(is_small_union_or_tunion, Base.unwrap_unionall(T).parameters) + end + isa(T, Union) || return false + n, isc = countconcrete(T) + return isc & (n <= 3) +end + +function countconcrete(@nospecialize(T)) + if Base.isdispatchelem(T) + return 1, true + elseif isa(T, Union) + na, isca = countconcrete(T.a) + nb, iscb = countconcrete(T.b) + return na + nb, isca & iscb + end + return 0, false +end diff --git a/TypedSyntax/test/runtests.jl b/TypedSyntax/test/runtests.jl new file mode 100644 index 00000000..636ece76 --- /dev/null +++ b/TypedSyntax/test/runtests.jl @@ -0,0 +1,268 @@ +using JuliaSyntax: JuliaSyntax, SyntaxNode, children, child, sourcetext, kind, @K_str +using TypedSyntax: TypedSyntax, TypedSyntaxNode, NotFound, getsrc +using Test + +hastype(@nospecialize(T)) = isa(T, Type) && T !== NotFound +has_name_typ(node, name::Symbol, @nospecialize(T)) = kind(node) == K"Identifier" && node.val === name && node.typ === T + +module TSN + +function has2xa(x) + x &= x +end +function has2xb(x) + x -= x + return x +end + +# This is taken from the definition of `sin(::Int)` in Base, copied here for testing purposes +# in case the implementation changes +for f in (:mysin,) + @eval function ($f)(x::Real) + xf = float(x) + x === xf && throw(MethodError($f, (x,))) + return ($f)(xf) + end +end + +function summer(list) + s = 0 # deliberately ::Int to test type-changes + for x in list + s += x + end + return s +end + +zerowhere(::AbstractArray{T}) where T<:Real = zero(T) + +# with two uses of the same slot in the same call +function simplef(a, b) + z = a * a + return z + b + y +end + +add2(x) = x[1] + x[2] + +likevect(X::T...) where {T} = T[ X[i] for i = 1:length(X) ] + +end + +@testset "TypedSyntax.jl" begin + st = """ + f(x, y, z) = x * y + z + """ + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN1.jl") + TSN.eval(Expr(rootnode)) + src, _ = getsrc(TSN.f, (Float32, Int, Float64)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + @test children(sig)[2].typ === Float32 + @test children(sig)[3].typ === Int + @test children(sig)[4].typ === Float64 + @test body.typ === Float64 # aggregate output + @test children(body)[1].typ === Float32 + + # Multiline + st = """ + function g(a, b, c) + x = a + b + return x + c + end + """ + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN2.jl") + TSN.eval(Expr(rootnode)) + src, _ = getsrc(TSN.g, (Int16, Int16, Int32)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + @test length(children(sig)) == 4 + @test children(body)[2].typ === Int32 + # Check that `x` gets an assigned type + nodex = child(body, 1, 1) + @test nodex.typ === Int16 + + # Target ambiguity + st = "math(x) = x + sin(x + π / 4)" + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN2.jl") + TSN.eval(Expr(rootnode)) + src, _ = getsrc(TSN.math, (Int,)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + @test has_name_typ(child(body, 1), :x, Int) + @test has_name_typ(child(body, 3, 2, 1), :x, Int) + pi4 = child(body, 3, 2, 3) + @test kind(pi4) == K"call" && pi4.typ == Core.Const(π / 4) + tsn = TypedSyntaxNode(TSN.has2xa, (Real,)) + @test tsn.typ === Any + sig, body = children(tsn) + @test has_name_typ(child(sig, 2), :x, Real) + @test has_name_typ(child(body, 1, 2), :x, Real) + @test has_name_typ(child(body, 1, 1), :x, Any) + tsn = TypedSyntaxNode(TSN.has2xb, (Real,)) + @test tsn.typ === Any + sig, body = children(tsn) + @test has_name_typ(child(sig, 2), :x, Real) + @test has_name_typ(child(body, 1, 2), :x, Real) + @test has_name_typ(child(body, 1, 1), :x, Any) + + # Target duplication + st = "math2(x) = sin(x) + sin(x)" + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN2.jl") + TSN.eval(Expr(rootnode)) + src, _ = getsrc(TSN.math2, (Int,)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + @test body.typ === Float64 + @test_broken child(body, 1).typ === Float64 + tsn = TypedSyntaxNode(TSN.simplef, Tuple{Float32, Int32}) + sig, body = children(tsn) + @test has_name_typ(child(body, 1, 2, 1), :a, Float32) + @test has_name_typ(child(body, 1, 2, 3), :a, Float32) + + # Inner functions + for (st, idxsinner, idxsouter) in ( + ("firstfirst(c) = map(x -> first(x), first(c))", (2, 2), (3,)), + (""" + firstfirst(c) = map(first(c)) do x + first(x) + end + """, (3, 1), (1, 2)) + ) + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN3.jl") + TSN.eval(Expr(rootnode)) + src, _ = getsrc(TSN.firstfirst, (Vector{Vector{Real}},)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + @test !hastype(child(body, idxsinner...).typ) # first(x) is hidden in anonymous function and not assignable + @test child(body, idxsouter...).typ === Vector{Real} + end + + # `ref` indexing + st = """ + function setlist!(listset, listget, i, j) + listset[i+1][j+1] = listget[i][j] + end + """ + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN4.jl") + TSN.eval(Expr(rootnode)) + src, rt = getsrc(TSN.setlist!, (Vector{Vector{Float32}}, Vector{Vector{UInt8}}, Int, Int)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + nodelist = child(body, 1, 2, 1, 1) # `listget` + @test sourcetext(nodelist) == "listget" && nodelist.typ === Vector{Vector{UInt8}} + @test nodelist.parent.typ === Vector{UInt8} # `listget[i]` + @test sourcetext(child(nodelist.parent, 2)) == "i" + @test nodelist.parent.parent.typ === UInt8 # `listget[i][j]` + + nodelist = child(body, 1, 1, 1, 1) # `listset` + @test sourcetext(nodelist) == "listset" && nodelist.typ === Vector{Vector{Float32}} + @test sourcetext(child(nodelist.parent, 2)) == "i+1" + @test nodelist.parent.typ === Vector{Float32} # `listset[i+1]` + @test kind(nodelist.parent.parent.parent) == K"=" # the `setindex!` call + + # tuple-destructuring + st = """ + function callfindmin(list) + val, idx = findmin(list) + x, y = idx, val + return y + end + """ + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN5.jl") + TSN.eval(Expr(rootnode)) + src, rt = getsrc(TSN.callfindmin, (Vector{Float64},)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + t = child(body, 1, 1) + @test kind(t) == K"tuple" + @test has_name_typ(child(t, 1), :val, Float64) + @test has_name_typ(child(t, 2), :idx, Int) + t = child(body, 2, 1) + @test kind(t) == K"tuple" + @test has_name_typ(child(t, 1), :x, Int) + @test has_name_typ(child(t, 2), :y, Float64) + + # kwfuncs + st = """ + function avoidzero(x; avoid_zero=true) + fx = float(x) + return iszero(x) ? oftype(fx, NaN) : fx + end + """ + rootnode = JuliaSyntax.parse(SyntaxNode, st; filename="TSN6.jl") + TSN.eval(Expr(rootnode)) + src, rt = getsrc(TSN.avoidzero, (Int,)) + # src looks like this: + # %1 = Main.TSN.:(var"#avoidzero#6")(true, #self#, x)::Float64 + # return %1 + # Consequently there is nothing to match, but at least we shouldn't error + tsn = TypedSyntaxNode(rootnode, src) + @test isa(tsn, TypedSyntaxNode) + @test rt === Float64 + # Try the kwbodyfunc + m = which(TSN.avoidzero, (Int,)) + src, rt = getsrc(Base.bodyfunction(m), (Bool, typeof(TSN.avoidzero), Int,)) + tsn = TypedSyntaxNode(rootnode, src) + sig, body = children(tsn) + isz = child(body, 2, 1, 1) + @test kind(isz) == K"call" && child(isz, 1).val == :iszero + @test isz.typ === Bool + @test child(body, 2, 1, 2).typ == Core.Const(NaN) + + # macros in function definition + tsn = TypedSyntaxNode(TSN.mysin, (Int,)) + @test kind(tsn) == K"macrocall" + sig, body = children(child(tsn, 2)) + @test has_name_typ(child(sig, 2, 1), :x, Int) + @test has_name_typ(child(body, 1, 1), :xf, Float64) + + # `for` loops + tsn = TypedSyntaxNode(TSN.summer, (Vector{Float64},)) + @test tsn.typ == Union{Int,Float64} + sig, body = children(tsn) + @test has_name_typ(child(sig, 2), :list, Vector{Float64}) + @test_broken has_name_typ(child(body, 1, 1), :s, Int) + @test_broken has_name_typ(child(body, 2, 1, 1), :x, Float64) + node = child(body, 2, 2, 1) + @test kind(node) == K"+=" + @test has_name_typ(child(node, 1), :s, Float64) # if this line runs, the LHS now has type `Float64` + @test has_name_typ(child(node, 2), :x, Float64) + @test has_name_typ(child(body, 3, 1), :s, Union{Float64, Int}) + + # `where` and unnamed arguments + tsn = TypedSyntaxNode(TSN.zerowhere, (Vector{Int16},)) + sig, body = children(tsn) + @test child(sig, 1, 2).typ === Vector{Int16} + @test body.typ === Core.Const(Int16(0)) + + # varargs + tsn = TypedSyntaxNode(TSN.likevect, (Int, Int)) + sig, body = children(tsn) + nodeva = child(sig, 1, 2) + @test kind(nodeva) == K"..." + @test has_name_typ(child(nodeva, 1, 1), :X, Tuple{Int,Int}) + + # Display + tsn = TypedSyntaxNode(TSN.summer, (Vector{Any},)) + str = sprint(tsn; context=:color=>true) do io, obj + printstyled(io, obj; iswarn=true, hide_type_stable=false) + end + @test occursin("summer(list\e[36m::Vector{Any}\e[39m)\e[31m::Any", str) + @test occursin("s\e[31m::Any\e[39m += x\e[31m::Any\e[39m", str) + str = sprint(tsn; context=:color=>true) do io, obj + printstyled(io, obj; type_annotations=false) + end + @test occursin("summer(list)", str) + @test occursin("s += x", str) + tsn = TypedSyntaxNode(TSN.zerowhere, (Vector{Int16},)) + str = sprint(tsn; context=:color=>true) do io, obj + printstyled(io, obj; iswarn=true, hide_type_stable=false) + end + @test occursin("AbstractArray{T}\e[36m::Vector{Int16}\e[39m", str) + @test occursin("Real\e[36m::Int16\e[39m", str) + tsn = TypedSyntaxNode(TSN.add2, (Vector{Float32},)) + str = sprint(tsn; context=:color=>true) do io, obj + printstyled(io, obj; iswarn=true, hide_type_stable=false) + end + @test occursin("[1]\e[36m::Float32\e[39m", str) + @test occursin("[2]\e[36m::Float32\e[39m", str) +end diff --git a/images_readme/descend_calls.png b/images_readme/descend_calls.png new file mode 100644 index 00000000..25d1f633 Binary files /dev/null and b/images_readme/descend_calls.png differ diff --git a/images_readme/descend_source_show_all.png b/images_readme/descend_source_show_all.png new file mode 100644 index 00000000..fca107c0 Binary files /dev/null and b/images_readme/descend_source_show_all.png differ diff --git a/images_readme/descend_source_toggles_warn.png b/images_readme/descend_source_toggles_warn.png new file mode 100644 index 00000000..6e27b54d Binary files /dev/null and b/images_readme/descend_source_toggles_warn.png differ diff --git a/images_readme/descend_toggles.png b/images_readme/descend_toggles.png new file mode 100644 index 00000000..32151c77 Binary files /dev/null and b/images_readme/descend_toggles.png differ diff --git a/src/Cthulhu.jl b/src/Cthulhu.jl index 7e2a7d8f..0b9abcdb 100644 --- a/src/Cthulhu.jl +++ b/src/Cthulhu.jl @@ -6,6 +6,9 @@ using CodeTracking: definition, whereis using InteractiveUtils using UUIDs using REPL: REPL, AbstractTerminal +using JuliaSyntax +using JuliaSyntax: SyntaxNode +using TypedSyntax import Core: MethodInstance const CC = Core.Compiler @@ -49,6 +52,7 @@ Base.@kwdef mutable struct CthulhuConfig with_effects::Bool = false inline_cost::Bool = false type_annotations::Bool = true + annotate_source::Bool = true # overrides optimize, although the current setting is preserved end """ @@ -75,6 +79,7 @@ end - `with_effects::Bool` Intial state of "effects" toggle. Defaults to `false`. - `inline_cost::Bool` Initial state of "inlining costs" toggle. Defaults to `false`. - `type_annotations::Bool` Initial state of "type annnotations" toggle. Defaults to `true`. +- `annotate_source::Bool` Initial state of "Source". Defaults to `true`. """ const CONFIG = CthulhuConfig() @@ -416,7 +421,8 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs remarks::Bool = CONFIG.remarks&!CONFIG.optimize, # default is false with_effects::Bool = CONFIG.with_effects, # default is false inline_cost::Bool = CONFIG.inline_cost&CONFIG.optimize, # default is false - type_annotations::Bool = CONFIG.type_annotations # default is true + type_annotations::Bool = CONFIG.type_annotations, # default is true + annotate_source::Bool = CONFIG.annotate_source, # default is true ) if isnothing(hide_type_stable) @@ -427,7 +433,7 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs debuginfo = getfield(DInfo, debuginfo)::DebugInfo end - is_cached(key::MethodInstance) = can_descend(interp, key, optimize) + is_cached(key::MethodInstance) = can_descend(interp, key, optimize & !annotate_source) menu_options = (; cursor = '•', scroll_wrap = true) display_CI = true @@ -438,7 +444,7 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs do_typeinf!(new_interp, new_mi) _descend(term, new_interp, new_mi; debuginfo, optimize, interruptexc, iswarn, hide_type_stable, remarks, - with_effects, inline_cost, type_annotations) + with_effects, inline_cost, type_annotations, annotate_source) end custom_toggles = Cthulhu.custom_toggles(interp) if !(custom_toggles isa Vector{CustomToggle}) @@ -449,11 +455,11 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs end while true if isa(override, InferenceResult) - (; src, rt, infos, slottypes, codeinf, effects) = lookup_constproped(interp, curs, override, optimize) + (; src, rt, infos, slottypes, codeinf, effects) = lookup_constproped(interp, curs, override, optimize & !annotate_source) elseif isa(override, SemiConcreteCallInfo) - (; src, rt, infos, slottypes, codeinf, effects) = lookup_semiconcrete(interp, curs, override, optimize) + (; src, rt, infos, slottypes, codeinf, effects) = lookup_semiconcrete(interp, curs, override, optimize & !annotate_source) else - if optimize + if optimize && !annotate_source codeinst = get_optimized_codeinst(interp, curs) if codeinst.inferred === nothing if isdefined(codeinst, :rettype_const) @@ -478,16 +484,16 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs end end end - (; src, rt, infos, slottypes, effects, codeinf) = lookup(interp, curs, optimize) + (; src, rt, infos, slottypes, effects, codeinf) = lookup(interp, curs, optimize & !annotate_source) end mi = get_mi(curs) - src = preprocess_ci!(src, mi, optimize, CONFIG) - if optimize || isa(src, IRCode) # optimization might have deleted some statements + src = preprocess_ci!(src, mi, optimize & !annotate_source, CONFIG) + if (optimize & !annotate_source) || isa(src, IRCode) # optimization might have deleted some statements infos = src.stmts.info else @assert length(src.code) == length(infos) end - callsites = find_callsites(interp, src, infos, mi, slottypes, optimize) + callsites, sourcenodes = find_callsites(interp, src, infos, mi, slottypes, optimize & !annotate_source, annotate_source) if display_CI pc2remarks = remarks ? get_remarks(interp, override !== nothing ? override : mi) : nothing @@ -506,7 +512,7 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs cthulhu_typed(lambda_io, debuginfo, src, rt, effects, mi; iswarn, hide_type_stable, pc2remarks, pc2effects, - inline_cost, type_annotations, + inline_cost, type_annotations, annotate_source, interp) end end @@ -523,7 +529,7 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs cthulhu_typed(lambda_io, debuginfo, src, rt, effects, mi; iswarn, hide_type_stable, pc2remarks, pc2effects, - inline_cost, type_annotations, + inline_cost, type_annotations, annotate_source, interp) end view_cmd = cthulhu_typed @@ -533,8 +539,9 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs @label show_menu - menu = CthulhuMenu(callsites, with_effects, optimize, iswarn&get(iostream, :color, false)::Bool, custom_toggles; menu_options...) - usg = usage(view_cmd, optimize, iswarn, hide_type_stable, debuginfo, remarks, with_effects, inline_cost, type_annotations, CONFIG.enable_highlighter, custom_toggles) + shown_callsites = annotate_source ? sourcenodes : callsites + menu = CthulhuMenu(shown_callsites, with_effects, optimize & !annotate_source, iswarn&get(iostream, :color, false)::Bool, hide_type_stable, custom_toggles; menu_options...) + usg = usage(view_cmd, annotate_source, optimize, iswarn, hide_type_stable, debuginfo, remarks, with_effects, inline_cost, type_annotations, CONFIG.enable_highlighter, custom_toggles) cid = request(term, usg, menu) toggle = menu.toggle @@ -546,10 +553,11 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs interruptexc ? throw(InterruptException()) : break end callsite = callsites[cid] + sourcenode = !isempty(sourcenodes) ? sourcenodes[cid] : nothing info = callsite.info if info isa MultiCallInfo - sub_callsites = let callsite=callsite + show_sub_callsites = sub_callsites = let callsite=callsite map(ci->Callsite(callsite.id, ci, callsite.head), info.callinfos) end if isempty(sub_callsites) @@ -561,7 +569,23 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs @error "Expected multiple callsites, but found none. Please fill an issue with a reproducing example." continue end - menu = CthulhuMenu(sub_callsites, with_effects, optimize, false, custom_toggles; sub_menu=true, menu_options...) + if sourcenode !== nothing + show_sub_callsites = let callsite=callsite + map(info.callinfos) do ci + p = Base.unwrap_unionall(get_mi(ci).specTypes).parameters + if length(p) == length(JuliaSyntax.children(sourcenode)) + 1 + newnode = copy(sourcenode) + for (i, child) in enumerate(JuliaSyntax.children(newnode)) + child.typ = p[i+1] + end + newnode + else + Callsite(callsite.id, ci, callsite.head) + end + end + end + end + menu = CthulhuMenu(show_sub_callsites, with_effects, optimize & !annotate_source, false, false, custom_toggles; sub_menu=true, menu_options...) cid = request(term, "", menu) if cid == length(sub_callsites) + 1 continue @@ -601,7 +625,7 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs override = get_override(info), debuginfo, optimize, interruptexc, iswarn, hide_type_stable, - remarks, with_effects, inline_cost, type_annotations) + remarks, with_effects, inline_cost, type_annotations, annotate_source) elseif toggle === :warn iswarn ⊻= true @@ -667,8 +691,13 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs display_CI = false elseif toggle === :typed view_cmd = cthulhu_typed + annotate_source = false + display_CI = true + elseif toggle === :source + view_cmd = cthulhu_typed + annotate_source = true display_CI = true - elseif toggle === :ast || toggle === :llvm || toggle === :native || toggle === :source + elseif toggle === :ast || toggle === :llvm || toggle === :native view_cmd = CODEVIEWS[toggle] println(iostream) view_cmd(iostream, mi, optimize, debuginfo, interp, CONFIG) @@ -792,7 +821,7 @@ function ascend(term, mi; interp::AbstractInterpreter=NativeInterpreter(), kwarg # warn highlighting is useful. interp′ = CthulhuInterpreter(interp) do_typeinf!(interp′, mi) - browsecodetyped && _descend(term, interp′, mi; iswarn=true, optimize=false, interruptexc=false, kwargs...) + browsecodetyped && _descend(term, interp′, mi; annotate_source=true, iswarn=true, optimize=false, interruptexc=false, kwargs...) end end end diff --git a/src/callsite.jl b/src/callsite.jl index b3d76039..cad02c7e 100644 --- a/src/callsite.jl +++ b/src/callsite.jl @@ -186,13 +186,21 @@ function Base.print(io::TextWidthLimiter, s::String) io.width += width return end + seen_termchar = in_termchar = false for c in graphemes(s) + if c == "\e" + in_termchar = true + end cwidth = textwidth(c) - if has_space(io, cwidth) + if in_termchar || has_space(io, cwidth) print(io.io, c) io.width += cwidth else - break + seen_termchar || break + end + if in_termchar && c == "m" + in_termchar = false + seen_termchar = !seen_termchar end end print(io.io, '…') @@ -209,6 +217,8 @@ function Base.print(io::TextWidthLimiter, c::Char) end return end +Base.print(io::TextWidthLimiter, sym::Symbol) = print(io, string(sym)) +Base.write(io::TextWidthLimiter, x::UInt8) = print(io, Char(x)) function Base.take!(io::TextWidthLimiter) io.width = 0 diff --git a/src/codeview.jl b/src/codeview.jl index 15cb5cbc..9bae3917 100644 --- a/src/codeview.jl +++ b/src/codeview.jl @@ -56,16 +56,6 @@ function cthulhu_ast(io::IO, mi, optimize, debuginfo, ::CthulhuInterpreter, conf end end -function cthulhu_source(io::IO, mi, optimize, debuginfo, ::CthulhuInterpreter, config::CthulhuConfig) - meth = mi.def::Method - def = definition(String, meth) - if isnothing(def) - return @warn "couldn't retrieve source of $meth" - end - src, line = def - highlight(io, src, "julia", config) -end - using Base.IRShow: IRShow, _stmt, _type, should_print_ssa_type, IRShowConfig, show_ir const __debuginfo = merge(IRShow.__debuginfo, Dict( @@ -108,7 +98,7 @@ function cthulhu_typed(io::IO, debuginfo::Symbol, src::Union{CodeInfo,IRCode}, @nospecialize(rt), effects::Effects, mi::Union{Nothing,MethodInstance}; iswarn::Bool=false, hide_type_stable::Bool=false, pc2remarks::Union{Nothing,PC2Remarks}=nothing, pc2effects::Union{Nothing,PC2Effects}=nothing, - inline_cost::Bool=false, type_annotations::Bool=true, + inline_cost::Bool=false, type_annotations::Bool=true, annotate_source::Bool=false, interp::AbstractInterpreter=CthulhuInterpreter()) debuginfo = IRShow.debuginfo(debuginfo) @@ -116,6 +106,15 @@ function cthulhu_typed(io::IO, debuginfo::Symbol, rettype = ignorelimited(rt) lambda_io = IOContext(io, :limit=>true) + if annotate_source && isa(src, CodeInfo) + tsn, _ = get_typed_sourcetext(mi, src, rt) + if tsn !== nothing + printstyled(lambda_io, tsn; type_annotations, iswarn, hide_type_stable, idxend=lastindex(tsn.source)) + println(lambda_io) + return nothing + end + end + if isa(src, CodeInfo) # we're working on pre-optimization state, need to ignore `LimitedAccuracy` src = copy(src) @@ -273,7 +272,6 @@ const CODEVIEWS = (; llvm=cthulhu_llvm, native=cthulhu_native, ast=cthulhu_ast, - source=cthulhu_source, ) """ diff --git a/src/reflection.jl b/src/reflection.jl index ae41b90d..ea02c2f0 100644 --- a/src/reflection.jl +++ b/src/reflection.jl @@ -20,11 +20,12 @@ end function find_callsites(interp::AbstractInterpreter, CI::Union{Core.CodeInfo, IRCode}, stmt_infos::Union{Vector{CCCallInfo}, Nothing}, mi::Core.MethodInstance, - slottypes::Vector{Any}, optimize::Bool=true) + slottypes::Vector{Any}, optimize::Bool=true, annotate_source::Bool=false) sptypes = sptypes_from_meth_instance(mi) - callsites = Callsite[] + callsites, sourcenodes = Callsite[], Union{TypedSyntaxNode,SyntaxNode,Callsite}[] stmts = isa(CI, IRCode) ? CI.stmts.inst : CI.code nstmts = length(stmts) + _, mappings = annotate_source ? get_typed_sourcetext(mi, CI, nothing; warn=false) : (nothing, nothing) for id = 1:nstmts stmt = stmts[id] @@ -93,9 +94,17 @@ function find_callsites(interp::AbstractInterpreter, CI::Union{Core.CodeInfo, IR end push!(callsites, callsite) + if annotate_source + if mappings !== nothing + mapped = mappings[id] + push!(sourcenodes, length(mapped) == 1 ? mapped[1] : callsite) + else + push!(sourcenodes, callsite) + end + end end end - return callsites + return callsites, sourcenodes end function process_const_info(interp::AbstractInterpreter, @nospecialize(thisinfo), @@ -293,7 +302,7 @@ function find_caller_of(interp::AbstractInterpreter, callee::MethodInstance, cal for optimize in (true, false) (; src, rt, infos, slottypes) = lookup(interp′, caller, optimize) src = preprocess_ci!(src, caller, optimize, CONFIG) - callsites = find_callsites(interp′, src, infos, caller, slottypes, optimize) + callsites, _ = find_callsites(interp′, src, infos, caller, slottypes, optimize) callsites = allow_unspecialized ? filter(cs->maybe_callsite(cs, callee), callsites) : filter(cs->is_callsite(cs, callee), callsites) foreach(cs -> add_sourceline!(locs, src, cs.id), callsites) @@ -331,3 +340,22 @@ function add_sourceline!(locs, CI, stmtidx::Int) end return locs end + +function get_typed_sourcetext(mi, src, rt; warn::Bool=true) + meth = mi.def::Method + tsn, mappings = TypedSyntax.tsn_and_mappings(meth, src, rt; warn, strip_macros=true) + # If we're filling in keyword args, just show the signature + if meth.name == :kwcall || !isempty(Base.kwarg_decl(meth)) + _, body = TypedSyntax.children(tsn) + # eliminate the body node + raw, bodyraw = tsn.raw, body.raw + idx = findfirst(==(bodyraw), raw.args) + if idx !== nothing + rawargs = raw.args[1:idx-1] + tsn.raw = typeof(raw)(raw.head, sum(nd -> nd.span, rawargs), rawargs) + body.raw = typeof(bodyraw)(bodyraw.head, UInt32(0), ()) + empty!(TypedSyntax.children(body)) + end + end + return tsn, mappings +end diff --git a/src/ui.jl b/src/ui.jl index 420dfa7f..7725e18a 100644 --- a/src/ui.jl +++ b/src/ui.jl @@ -27,9 +27,9 @@ function show_as_line(callsite::Callsite, with_effects::Bool, optimize::Bool, is end end -function CthulhuMenu(callsites, with_effects::Bool, optimize::Bool, iswarn::Bool, custom_toggles::Vector{CustomToggle}; - pagesize::Int=10, sub_menu = false, kwargs...) - options = vcat(map(callsite->show_as_line(callsite, with_effects, optimize, iswarn), callsites), ["↩"]) +function CthulhuMenu(callsites, with_effects::Bool, optimize::Bool, iswarn::Bool, hide_type_stable::Bool, + custom_toggles::Vector{CustomToggle}; pagesize::Int=10, sub_menu = false, kwargs...) + options = build_options(callsites, with_effects, optimize, iswarn, hide_type_stable) length(options) < 1 && error("CthulhuMenu must have at least one option") # if pagesize is -1, use automatic paging @@ -47,6 +47,35 @@ function CthulhuMenu(callsites, with_effects::Bool, optimize::Bool, iswarn::Bool return CthulhuMenu(options, pagesize, pageoffset, selected, nothing, sub_menu, config, custom_toggles) end +build_options(callsites::Vector{Callsite}, with_effects::Bool, optimize::Bool, iswarn::Bool, ::Bool) = + vcat(map(callsite->show_as_line(callsite, with_effects, optimize, iswarn), callsites), ["↩"]) +function build_options(callsites, with_effects::Bool, optimize::Bool, iswarn::Bool, hide_type_stable::Bool) + reduced_displaysize = (displaysize(stdout)::Tuple{Int,Int})[2] - 3 + nd = nothing + + shown_callsites = map(callsites) do node + if isa(node, Callsite) + show_as_line(node, with_effects, optimize, iswarn) + else + if nd === nothing + nd = ndigits(node.source.first_line + TypedSyntax.nlines(node.source) - 1) + reduced_displaysize -= nd + 1 + end + string( + sprint(lpad(TypedSyntax.source_line(node.source, TypedSyntax.first_byte(node)), nd); context=:color=>true) do io, ln + printstyled(io, ln; color=:light_black) + end, + " ", + chomp(sprint(node; context=:color=>true) do io, node + printstyled(TextWidthLimiter(io, reduced_displaysize), node; iswarn, hide_type_stable) + end) + ) + end + end + push!(shown_callsites, "↩") + return shown_callsites +end + TerminalMenus.options(m::CthulhuMenu) = m.options TerminalMenus.cancel(m::CthulhuMenu) = m.selected = -1 @@ -59,7 +88,7 @@ function stringify(@nospecialize(f), context::IOContext) end const debugcolors = (:nothing, :light_black, :yellow) -function usage(@nospecialize(view_cmd), optimize, iswarn, hide_type_stable, debuginfo, remarks, with_effects, inline_cost, type_annotations, highlight, +function usage(@nospecialize(view_cmd), annotate_source, optimize, iswarn, hide_type_stable, debuginfo, remarks, with_effects, inline_cost, type_annotations, highlight, custom_toggles::Vector{CustomToggle}) colorize(use_color::Bool, c::Char) = stringify() do io use_color ? printstyled(io, c; color=:cyan) : print(io, c) @@ -70,17 +99,20 @@ function usage(@nospecialize(view_cmd), optimize, iswarn, hide_type_stable, debu println(ioctx, "Select a call to descend into or ↩ to ascend. [q]uit. [b]ookmark.") print(ioctx, "Toggles: [", - colorize(optimize, 'o'), "]ptimize, [", colorize(iswarn, 'w'), "]arn, [", colorize(hide_type_stable, 'h'), "]ide type-stable statements, [", - stringify() do io - printstyled(io, 'd'; color=debugcolors[Int(debuginfo)+1]) - end, "]ebuginfo, [", - colorize(remarks, 'r'), "]emarks, [", - colorize(with_effects, 'e'), "]ffects, [", - colorize(inline_cost, 'i'), "]nlining costs, [", colorize(type_annotations, 't'), "]ype annotations, [", colorize(highlight, 's'), "]yntax highlight for Source/LLVM/Native") + if !annotate_source + print(ioctx, ", [", + colorize(optimize, 'o'), "]ptimize, [", + stringify() do io + printstyled(io, 'd'; color=debugcolors[Int(debuginfo)+1]) + end, "]ebuginfo, [", + colorize(remarks, 'r'), "]emarks, [", + colorize(with_effects, 'e'), "]ffects, [", + colorize(inline_cost, 'i'), "]nlining costs") + end for i = 1:length(custom_toggles) ct = custom_toggles[i] print(ioctx, ", [", colorize(ct.onoff, Char(ct.key)), ']', ct.description) @@ -89,15 +121,19 @@ function usage(@nospecialize(view_cmd), optimize, iswarn, hide_type_stable, debu print(ioctx, '.') println(ioctx) println(ioctx, "Show: [", - colorize(view_cmd === cthulhu_source, 'S'), "]ource code, [", + colorize(annotate_source, 'S'), "]ource code, [", colorize(view_cmd === cthulhu_ast, 'A'), "]ST, [", - colorize(view_cmd === cthulhu_typed, 'T'), "]yped code, [", + colorize(!annotate_source && view_cmd === cthulhu_typed, 'T'), "]yped code, [", colorize(view_cmd === cthulhu_llvm, 'L'), "]LVM IR, [", colorize(view_cmd === cthulhu_native, 'N'), "]ative code") print(ioctx, """ - Actions: [E]dit source code, [R]evise and redisplay - Advanced: dump [P]arams cache.""") + Actions: [E]dit source code, [R]evise and redisplay""") + if !annotate_source + print(ioctx, + """ + Advanced: dump [P]arams cache.""") + end return String(take!(io)) end diff --git a/test/FakeTerminals.jl b/test/FakeTerminals.jl index 1e59753d..37c42fc8 100644 --- a/test/FakeTerminals.jl +++ b/test/FakeTerminals.jl @@ -18,7 +18,7 @@ function fake_terminal(f; options::REPL.Options=REPL.Options(confirm_exit=false) term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb") term = REPL.Terminals.TTYTerminal(term_env, input.out, IOContext(output.in, :color=>get(stdout, :color, false)), err.in) - f(term, input.in, output.out) + f(term, input.in, output.out, err) t = @async begin close(input.in) close(output.in) diff --git a/test/setup.jl b/test/setup.jl index 479586d0..533d89c7 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -12,7 +12,7 @@ end function find_callsites_by_ftt(@nospecialize(f), @nospecialize(TT=Tuple{}); optimize=true) (; interp, src, infos, mi, slottypes) = cthulhu_info(f, TT; optimize) src === nothing && return Cthulhu.Callsite[] - callsites = Cthulhu.find_callsites(interp, src, infos, mi, slottypes, optimize) + callsites, _ = Cthulhu.find_callsites(interp, src, infos, mi, slottypes, optimize) @test all(c -> Cthulhu.get_effects(c) isa Cthulhu.Effects, callsites) return callsites end diff --git a/test/test_terminal.jl b/test/test_terminal.jl index 232dda27..9addebd3 100644 --- a/test/test_terminal.jl +++ b/test/test_terminal.jl @@ -45,8 +45,8 @@ end end # Write a file that we track with Revise. Creating it programmatically allows us to rewrite it with # different content - fn = tempname() - open(fn, "w") do io + revisedfile = tempname() + open(revisedfile, "w") do io println(io, """ function simplef(a, b) @@ -55,7 +55,7 @@ end end """) end - includet(@__MODULE__, fn) + includet(@__MODULE__, revisedfile) # Copy the user's current settings and set up the defaults CONFIG = deepcopy(Cthulhu.CONFIG) @@ -65,9 +65,9 @@ end end try - fake_terminal() do term, in, out + fake_terminal() do term, in, out, _ t = @async begin - @with_try_stderr out descend(simplef, Tuple{Float32, Int32}; interruptexc=false, terminal=term) + @with_try_stderr out descend(simplef, Tuple{Float32, Int32}; annotate_source=false, interruptexc=false, terminal=term) end lines = cread(out) @test occursin("invoke simplef(::Float32,::Int32)::Float32", lines) @@ -119,12 +119,7 @@ end # Source view write(in, 'S') lines = cread(out) - @test occursin(""" - \n\nfunction simplef(a, b) - z = a*a - return z + b - end - """, lines) + @test occursin("z\e[36m::Float32\e[39m = (a\e[36m::Float32\e[39m*a\e[36m::Float32\e[39m)\e[36m::Float32\e[39m", lines) @test occursin('[' * colorize(true, 'S') * "]ource", lines) # turn on syntax highlighting write(in, 's'); cread(out) @@ -134,7 +129,8 @@ end @test occursin("\u001B", first(split(lines, '\n'))) @test occursin('[' * colorize(true, 's') * "]yntax", lines) write(in, 's'); cread(out) # off again - # Toggling 'o' goes back to typed code, make sure it also updates the selector status + # Back to typed code + write(in, 'T'); cread(out) write(in, 'o') lines = cread(out) @test occursin('[' * colorize(true, 'T') * "]yped", lines) @@ -171,7 +167,7 @@ end @test occursin(r"\(z \+ b\)\u001B\[\d\dm::Float32\u001B\[39m", lines) @test occursin('[' * colorize(true, 'T') * "]yped", lines) # Revise - open(fn, "w") do io + open(revisedfile, "w") do io println(io, """ function simplef(a, b) @@ -190,9 +186,9 @@ end wait(t) end # Multicall & iswarn=true - fake_terminal() do term, in, out + fake_terminal() do term, in, out, _ t = @async begin - @with_try_stderr out descend_code_warntype(MultiCall.callfmulti, Tuple{Any}; interruptexc=false, optimize=false, terminal=term) + @with_try_stderr out descend_code_warntype(MultiCall.callfmulti, Tuple{Any}; annotate_source=false, interruptexc=false, optimize=false, terminal=term) end lines = cread(out) @test occursin("\nBody\e[", lines) @@ -214,9 +210,9 @@ end end # Tasks (see the special handling in `_descend`) ftask() = @sync @async show(io, "Hello") - fake_terminal() do term, in, out + fake_terminal() do term, in, out, _ t = @async begin - @with_try_stderr out @descend terminal=term ftask() + @with_try_stderr out @descend terminal=term annotate_source=false ftask() end lines = cread(out) @test occursin(r"• %\d\d = task", lines) @@ -230,24 +226,42 @@ end end # descend with MethodInstances mi = Cthulhu.get_specialization(MultiCall.callfmulti, Tuple{typeof(Ref{Any}(1))}) - fake_terminal() do term, in, out + fake_terminal() do term, in, out, _ t = @async begin - @with_try_stderr out descend(mi; interruptexc=false, optimize=false, terminal=term) + @with_try_stderr out descend(mi; annotate_source=false, interruptexc=false, optimize=false, terminal=term) end lines = cread(out) @test occursin("fmulti(::Any)", lines) write(in, 'q') wait(t) end - fake_terminal() do term, in, out + fake_terminal() do term, in, out, _ t = @async begin - @with_try_stderr out descend_code_warntype(mi; interruptexc=false, optimize=false, terminal=term) + @with_try_stderr out descend_code_warntype(mi; annotate_source=false, interruptexc=false, optimize=false, terminal=term) end lines = cread(out) @test occursin("Base.getindex(c)\e[91m\e[1m::Any\e[22m\e[39m", lines) write(in, 'q') wait(t) end + # Fallback to typed code + fake_terminal() do term, in, out, err + t = @async @with_try_stderr out begin + redirect_stderr(err) do + descend((Int,); annotate_source=true, interruptexc=false, optimize=false, terminal=term) do x + [x] + end + end + end + lines = cread1(out) + wlines = readuntil(err, '\n') + @test occursin("couldn't retrieve source", wlines) + @test occursin("%1 = Base.vect(x)", lines) + @test occursin("(::$Int)::Vector{$Int}", lines) + write(in, 'q') + readuntil(err, '\n') # Clear out any extra output + wait(t) + end # ascend @noinline inner3(x) = 3x @@ -255,7 +269,7 @@ end inner1(x) = -1*inner2(x) inner1(0x0123) mi = Cthulhu.get_specialization(inner3, Tuple{UInt16}) - fake_terminal() do term, in, out + fake_terminal() do term, in, out, _ t = @async begin @with_try_stderr out ascend(term, mi) end @@ -269,7 +283,6 @@ end ln = occursin(r"caller.*inner3", lines[6]) ? 6 : occursin(r"caller.*inner3", lines[8]) ? 8 : error("not found") @test occursin("inner2", lines[ln+1]) - @test any(str -> occursin("Variables", str), lines[ln+2:end]) write(in, 'q') write(in, 'q') wait(t) @@ -279,7 +292,7 @@ end for fn in fieldnames(Cthulhu.CthulhuConfig) setfield!(Cthulhu.CONFIG, fn, getfield(CONFIG, fn)) end - rm(fn) + rm(revisedfile) end end