[POC] per test code coverage using rb_thread_add_event_hook in C extension

[anmarchenko]

Proof of concept that uses native C function rb_thread_add_event_hook to add an event hook for RUBY_EVENT_LINE event and track executed lines independently from Ruby's built-in Coverage module. Additionally, it is optimized by tracking only files that are located under the git repository root.

Performance evaluation

Projects used to evaluate performance:

• sidekiq: very small and fast test suite
• rubocop: very big but quite fast test suite (20 000 tests under 2 minutes)
• vagrant: very big and slow test suite (5 000 tests in about 14 minutes)

In the following table are durations of the whole test session in seconds, in brackets overhead percentage compared to run with datadog-ci.

Coverage type	Sidekiq	Rubocop	Vagrant
No coverage	~6,8	111	827
Simplecov (total coverage)	~7	137	864
Per test coverage, files	7,5 (10%)	178 (60%)	869 (5%)
Per test coverage, lines	8,5 (25%)	187 (68%)	886 (7%)
Per test coverage, lines AND total coverage with simplecov*	8,7 (24%)	210 (53%)	928 (7%)

* overhead here is compared to "Simplecov (total coverage)"

Relevant code

Native extension

static void dd_cov_update_line_coverage(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass)
{
  struct dd_cov_data *dd_cov_data;
  TypedData_Get_Struct(data, struct dd_cov_data, &dd_cov_data_type, dd_cov_data);

  const char *filename = rb_sourcefile();
  if (filename == 0)
  {
    return;
  }

  if (!prefix(dd_cov_data->root, filename))
  {
    return;
  }

  unsigned long len_filename = strlen(filename);

  VALUE rb_str_source_file = rb_str_new(filename, len_filename);

  if (dd_cov_data->mode == DD_COVERAGE_TARGET_FILES)
  {
    rb_hash_aset(dd_cov_data->coverage, rb_str_source_file, Qtrue);
    return;
  }

  if (dd_cov_data->mode == DD_COVERAGE_TARGET_LINES)
  {
    VALUE rb_lines = rb_hash_aref(dd_cov_data->coverage, rb_str_source_file);
    if (rb_lines == Qnil)
    {
      rb_lines = rb_ary_new();
      rb_hash_aset(dd_cov_data->coverage, rb_str_source_file, rb_lines);
    }

    VALUE line_number = INT2FIX(rb_sourceline());
    if (rb_ary_includes(rb_lines, line_number) == Qfalse)
    {
      rb_ary_push(rb_lines, line_number);
    }
  }
}

static VALUE dd_cov_start(VALUE self)
{
  // get current thread
  VALUE thval = rb_thread_current();

  // add event hook
  rb_thread_add_event_hook(thval, dd_cov_update_line_coverage, RUBY_EVENT_LINE, self);

  return self;
}

static VALUE dd_cov_stop(VALUE self)
{
  // get current thread
  VALUE thval = rb_thread_current();
  // remove event hook for the current thread
  rb_thread_remove_event_hook(thval, dd_cov_update_line_coverage);

  struct dd_cov_data *dd_cov_data;
  TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data);

  VALUE cov = dd_cov_data->coverage;

  dd_cov_data->coverage = rb_hash_new();

  return cov;
}

Usage in Ruby

mode = :files # or :lines
@ddcov = DDCov.new(root: Utils::Git.root, mode: mode)

# when test starts
@ddcov.start

# when test ends
@ddcov.stop # returns coverage hash