From c4086aa57c16b9ca68ba5dccdcd2fb90c868d8a4 Mon Sep 17 00:00:00 2001 From: Stefan Krastanov Date: Sun, 31 Dec 2023 17:04:24 -0500 Subject: [PATCH] benchmark cleanup and selfhosting (#210) --- .github/workflows/benchmark-comment.yml | 1 + .github/workflows/benchmark.yml | 2 +- Project.toml | 2 +- benchmark/plotbenchmarks.jl | 29 ++++++++++++------- benchmark/plotttfxbenchmarks.jl | 29 ++++++++++++++----- benchmark/run_benchmark_matrix.fish | 12 ++++++++ .../{run_bechmarks.sh => run_benchmarks.sh} | 0 ...fx_bechmarks.sh => run_ttfx_benchmarks.sh} | 0 8 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 benchmark/run_benchmark_matrix.fish rename benchmark/{run_bechmarks.sh => run_benchmarks.sh} (100%) rename benchmark/{run_ttfx_bechmarks.sh => run_ttfx_benchmarks.sh} (100%) diff --git a/.github/workflows/benchmark-comment.yml b/.github/workflows/benchmark-comment.yml index 66a250789..ffb2d3ff3 100644 --- a/.github/workflows/benchmark-comment.yml +++ b/.github/workflows/benchmark-comment.yml @@ -18,6 +18,7 @@ on: jobs: comment: runs-on: ubuntu-latest + #runs-on: self-hosted if: > ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index aa1040df1..5218fc733 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,11 +1,11 @@ name: Performance tracking - on: pull_request: jobs: performance-tracking: runs-on: ubuntu-latest + #runs-on: self-hosted steps: # setup - uses: actions/checkout@v4 diff --git a/Project.toml b/Project.toml index 93d8f6df4..58b0dc9ef 100644 --- a/Project.toml +++ b/Project.toml @@ -45,7 +45,7 @@ ILog2 = "0.2.3" InteractiveUtils = "1" LinearAlgebra = "1" MacroTools = "0.5.9" -Makie = "0.19.7" +Makie = "0.20" Nemo = "0.38" Plots = "1.38.0" PrecompileTools = "1.2" diff --git a/benchmark/plotbenchmarks.jl b/benchmark/plotbenchmarks.jl index ff3f330b1..dfc178287 100644 --- a/benchmark/plotbenchmarks.jl +++ b/benchmark/plotbenchmarks.jl @@ -24,6 +24,8 @@ function results_to_dataframe(res) DataFrame(recs) end +## +cd("QuantumCliffordBenchmarksLog/logs") ## files = glob("*.benchmarkresults") @@ -35,22 +37,29 @@ for f in files continue end sdf = results_to_dataframe(readresults(f)) - sdf[!,:tag] .= m.captures[1] + #sdf[!,:tag] .= m.captures[1] + sdf[!,:version] .= VersionNumber(m.captures[1][2:end]) sdf[!,:threads] .= m.captures[2] - sdf[!,:julia] .= m.captures[3] + #sdf[!,:julia] .= m.captures[3] + sdf[!,:julia] .= m.captures[3]=="nightly" ? v"1.999" : VersionNumber(m.captures[3]) push!(subframes, sdf) end ## df = vcat(subframes...) +maxjversion = maximum(df[df.julia .!= v"1.999",:julia]) +df[df.julia .== v"1.999", :julia] .= VersionNumber(maxjversion.major, maxjversion.minor+1, 0) df[!,:fullname] .= df[:,:taxon1] .* "-" .* df[:,:taxon2] .* "-" .* df[:,:taxon3] df[!,:groupname] .= df[:,:taxon1] .* "-" .* df[:,:taxon2] df[!,:logtime] .= log10.(df[:,:time]) +df_min = combine(groupby(df, [:taxon3, :taxon2, :taxon1, :julia, :threads]), :logtime=>minimum=>:minlogtime) +df = outerjoin(df, df_min, on=[:taxon3, :taxon2, :taxon1, :julia, :threads]) +df[!,:normlogtime] .= df[:,:logtime] .- df[:,:minlogtime] -df2 = combine(groupby(df, [:groupname, :taxon1, :tag, :julia, :threads]), :logtime=>mean=>:logtime) -df1 = combine(groupby(df2, [:taxon1, :tag, :julia, :threads]), :logtime=>mean=>:logtime) -df0 = combine(groupby(df1, [:tag, :julia, :threads]), :logtime=>mean=>:logtime) +df2 = combine(groupby(df, [:groupname, :taxon1, :version, :julia, :threads]), :normlogtime=>mean=>:logtime) +df1 = combine(groupby(df2, [:taxon1, :version, :julia, :threads]), :logtime=>mean=>:logtime) +df0 = combine(groupby(df1, [:version, :julia, :threads]), :logtime=>mean=>:logtime) layers = visual(Stairs, step=:center) axis = (;width = 500, height = 400, xticklabelrotation=-pi/3) @@ -59,25 +68,25 @@ facet = (;linkyaxes=:minimal) ## benchmark_data3 = data(df) -mappings3 = mapping(:tag, :time, color=:julia, row=:fullname, col=:threads) +mappings3 = mapping(:version, :time, color=:julia, row=:fullname, col=:threads) f3 = draw(benchmark_data3 * layers * mappings3; facet, axis) save("../benchmarks3.png",f3) benchmark_data2 = data(df2) -mappings2 = mapping(:tag, :logtime, color=:julia, row=:groupname, col=:threads) +mappings2 = mapping(:version, :logtime, color=:julia, row=:groupname, col=:threads) f2 = draw(benchmark_data2 * layers * mappings2; facet, axis) save("../benchmarks2.png",f2) benchmark_data1 = data(df1) -mappings1 = mapping(:tag, :logtime, color=:julia, row=:taxon1, col=:threads) +mappings1 = mapping(:version, :logtime, color=:julia, row=:taxon1, col=:threads) f1 = draw(benchmark_data1 * layers * mappings1; facet, axis) save("../benchmarks1.png",f1) benchmark_data0 = data(df0) -mappings0 = mapping(:tag, :logtime, color=:julia, col=:threads) +mappings0 = mapping(:version, :logtime, color=:julia, col=:threads) f0 = draw(benchmark_data0 * layers * mappings0; facet, axis) save("../benchmarks0.png",f0) -mappings3a = mapping(:tag, :allocs, color=:julia, row=:fullname, col=:threads) +mappings3a = mapping(:version, :allocs, color=:julia, row=:fullname, col=:threads) f3a = draw(benchmark_data3 * layers * mappings3a; facet, axis) save("../benchmarks3_allocs.png",f3a) diff --git a/benchmark/plotttfxbenchmarks.jl b/benchmark/plotttfxbenchmarks.jl index 248e5bf80..68b906405 100644 --- a/benchmark/plotttfxbenchmarks.jl +++ b/benchmark/plotttfxbenchmarks.jl @@ -8,7 +8,8 @@ using Statistics using Glob ## - +cd("QuantumCliffordBenchmarksLog/ttfxlogs") +## files = glob("using_*.ttfxresults") records = [] for f in files @@ -18,26 +19,38 @@ for f in files continue end tag = m.captures[1] + version = VersionNumber(tag[2:end]) threads = m.captures[2] - julia = m.captures[3] - pkgimport = minimum(parse.(Float64, readlines(f))) - task = minimum(parse.(Float64, readlines("task_tag=$(tag)-nthreads=$(threads)-julia=$(julia).ttfxresults"))) - push!(records, (;tag, threads, julia, pkgimport, task)) + juliastr = m.captures[3] + julia = juliastr=="nightly" ? v"1.999" : VersionNumber(m.captures[3]) + pkgimportsamples = parse.(Float64, readlines(f)) + pkgimport = isempty(pkgimportsamples) ? missing : minimum(parse.(Float64, readlines(f))) + tasksamples = parse.(Float64, readlines("task_tag=$(tag)-nthreads=$(threads)-julia=$(juliastr).ttfxresults")) + task = isempty(tasksamples) ? missing : minimum(tasksamples) + push!(records, (;version, threads, julia, pkgimport, task)) end ## df = DataFrame(records) +maxjversion = maximum(df[df.julia .!= v"1.999",:julia]) +df[df.julia .== v"1.999", :julia] .= VersionNumber(maxjversion.major, maxjversion.minor+1, 0) df[!,:total] .= df[:,:task] .+ df[:,:pkgimport] df = DataFrames.stack(df, [:total, :task, :pkgimport], variable_name=:x, value_name=:time) +sort!(df) # by first column (version) + +df = df[df.time .!== missing,:] # otherwise the ticks are messed up, probably a bug layers = visual(Stairs, step=:center) -axis = (;width = 500, height = 400, xticklabelrotation=-pi/3) -facet = (;linkyaxes=:minimal) +maxy = ceil(maximum(skipmissing(df.time))) +axis = (;width = 800, height = 400, xticklabelrotation=-pi/2, yticks=(0:0.5:maxy)) +facet = (; + linkyaxes=:minimal, + ) ## benchmark_data = data(df) -mappings = mapping(:tag, :time, color=:julia, row=:x, col=:threads) +mappings = mapping(:version, :time, color=:julia, row=:x, col=:threads) f = draw(benchmark_data * layers * mappings; facet, axis) save("../benchmarks_ttfx.png",f) diff --git a/benchmark/run_benchmark_matrix.fish b/benchmark/run_benchmark_matrix.fish new file mode 100644 index 000000000..3d62a611b --- /dev/null +++ b/benchmark/run_benchmark_matrix.fish @@ -0,0 +1,12 @@ +#!/usr/bin/env fish +for jlversion in nightly 1.10.0 1.9.0 1.8.0 1.7.0 1.6.0 + for tag in (git for-each-ref --sort=creatordate --format '%(tag)' refs/tags | tail -n+15) + for nthreads in 1 4 + for retune in true + ./run_ttfx_bechmarks.sh -v$jlversion -t$tag -n$nthreads + ./run_bechmarks.sh -v$jlversion -t$tag -n$nthreads -r$retune + end + end + end +end + diff --git a/benchmark/run_bechmarks.sh b/benchmark/run_benchmarks.sh similarity index 100% rename from benchmark/run_bechmarks.sh rename to benchmark/run_benchmarks.sh diff --git a/benchmark/run_ttfx_bechmarks.sh b/benchmark/run_ttfx_benchmarks.sh similarity index 100% rename from benchmark/run_ttfx_bechmarks.sh rename to benchmark/run_ttfx_benchmarks.sh