Skip to content

Commit

Permalink
DNM: Try out jemalloc leak profiling
Browse files Browse the repository at this point in the history
Motiviation is to find memory leaks in CI, want to evaluate if this
works at all first

See https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Leak-Checking
  • Loading branch information
def- committed Jan 20, 2025
1 parent 8e66d24 commit 450dbaf
Show file tree
Hide file tree
Showing 4 changed files with 11 additions and 1 deletion.
2 changes: 2 additions & 0 deletions ci/plugins/mzcompose/hooks/command
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ if [ -n "${CI_COVERAGE_ENABLED:-}" ]; then
find . -name '*.profraw' -delete # Remove remaining profraw files from coverage runs
fi

find . -name 'jeprof*.heap' -delete

ci_collapsed_heading ":docker: Purging all existing docker containers and volumes, regardless of origin"
docker ps --all --quiet | xargs --no-run-if-empty docker rm --force --volumes
rm -f services.log
Expand Down
4 changes: 4 additions & 0 deletions ci/plugins/mzcompose/hooks/pre-exit
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ if [ -n "${CI_COVERAGE_ENABLED:-}" ] && [ -z "${BUILDKITE_MZCOMPOSE_PLUGIN_SKIP_
fi
fi

find . -name 'jeprof*.heap' | while read -r i; do
buildkite-agent artifact upload "$i"
done

if [ "$CI_ANNOTATE_ERRORS_RESULT" -ne 0 ]; then
echo "+++ ci-annotate-errors failed, which indicates that an unknown error was found"
exit "$CI_ANNOTATE_ERRORS_RESULT"
Expand Down
1 change: 1 addition & 0 deletions misc/python/materialize/cli/ci_annotate_errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
| (FAIL|TIMEOUT)\s+\[\s*\d+\.\d+s\]
# parallel-workload
| worker_.*\ still\ running: [\s\S]* Threads\ have\ not\ stopped\ within\ 5\ minutes,\ exiting\ hard
| Leak\ approximation\ summary:
)
.* $
""",
Expand Down
5 changes: 4 additions & 1 deletion src/prof/src/jemalloc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,12 @@ use mz_ore::metrics::{MetricsRegistry, UIntGauge};
use pprof_util::ProfStartTime;
use tikv_jemalloc_ctl::{epoch, stats};

// TODO: Only set leak profiliing in CI, not in production, need to have a flag, or can we
// overwrite the malloc_conf from the outside?
#[allow(non_upper_case_globals)]
#[export_name = "malloc_conf"]
pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0";
pub static malloc_conf: &[u8] =
b"prof_prefix:/mzdata/jeprof,prof_leak:true,prof_final:true,prof:true,prof_active:true,lg_prof_sample:19\0";

#[derive(Copy, Clone, Debug)]
pub struct JemallocProfMetadata {
Expand Down

0 comments on commit 450dbaf

Please sign in to comment.