From 2d24951a0eb89f3482b01c75d8f2aebfbcdf5a34 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 21 Feb 2024 13:14:22 +0100 Subject: [PATCH 1/9] sinmplest code coverage with resume/suspend POC --- lib/datadog/ci/itr/coverage/collector.rb | 47 ++++++++++++++++++++++ lib/datadog/ci/itr/coverage/filter.rb | 29 +++++++++++++ lib/datadog/ci/test_visibility/recorder.rb | 25 ++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 lib/datadog/ci/itr/coverage/collector.rb create mode 100644 lib/datadog/ci/itr/coverage/filter.rb diff --git a/lib/datadog/ci/itr/coverage/collector.rb b/lib/datadog/ci/itr/coverage/collector.rb new file mode 100644 index 00000000..ac6ed911 --- /dev/null +++ b/lib/datadog/ci/itr/coverage/collector.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "coverage" + +require_relative "filter" + +module Datadog + module CI + module Itr + module Coverage + class Collector + def initialize + # Do not run code coverage if someone else is already running it. + # It means that user is running the test with coverage and ITR would mess it up. + @coverage_supported = !::Coverage.running? + # @coverage_supported = false + end + + def setup + if @coverage_supported + p "RUNNING WITH CODE COVERAGE ENABLED!" + ::Coverage.setup(lines: true) + else + p "RUNNING WITH CODE COVERAGE DISABLED!" + end + end + + def start + return unless @coverage_supported + + # if execution is threaded then coverage might already be running + ::Coverage.resume unless ::Coverage.running? + end + + def stop + return nil unless @coverage_supported + + result = ::Coverage.result(stop: false, clear: true) + ::Coverage.suspend if ::Coverage.running? + + Filter.call(result) + end + end + end + end + end +end diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb new file mode 100644 index 00000000..d81ad969 --- /dev/null +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require_relative "../../utils/git" + +module Datadog + module CI + module Itr + module Coverage + class Filter + def self.call(raw_result) + new.call(raw_result) + end + + def initialize(root: Utils::Git.root) + @regex = /\A#{Regexp.escape(root + File::SEPARATOR)}/i.freeze + end + + def call(raw_result) + return nil if raw_result.nil? + + raw_result.select do |path, coverage| + path =~ @regex && coverage[:lines].any? { |count| count && count > 0 } + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index c274590f..bd36e11e 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -20,6 +20,8 @@ require_relative "../test_module" require_relative "../test_suite" +require_relative "../itr/coverage/collector" + module Datadog module CI module TestVisibility @@ -38,6 +40,13 @@ def initialize( @local_context = Context::Local.new @global_context = Context::Global.new @codeowners = codeowners + + @coverage_collector = Itr::Coverage::Collector.new + begin + @coverage_collector.setup + rescue => e + Datadog.logger.debug("Coverage collector setup failed: #{e.message}") + end end def start_test_session(service: nil, tags: {}) @@ -103,12 +112,15 @@ def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block) {resource: test_name, continue_from: Datadog::Tracing::TraceDigest.new} ) + @coverage_collector.start if block start_datadog_tracer_span(test_name, span_options) do |tracer_span| test = build_test(tracer_span, tags) @local_context.activate_test(test) do block.call(test) + + on_test_end(test) end end else @@ -160,6 +172,8 @@ def active_test_suite(test_suite_name) end def deactivate_test + on_test_end(active_test) if active_test + @local_context.deactivate_test end @@ -314,6 +328,17 @@ def validate_test_suite_level_visibility_correctness(test) end end end + + # event system? + def on_test_end(test) + coverage = @coverage_collector.stop + if coverage + files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } + test.set_tag("_test.coverage", files_covered) + + # p test.get_tag("_test.coverage") + end + end end end end From ab0257e224d8cfcf539a6fd320deb0d486aa1654 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 21 Feb 2024 19:20:45 +0100 Subject: [PATCH 2/9] construct line coverage bitmaps while filtering --- lib/datadog/ci/itr/coverage/filter.rb | 28 ++++++++++++++++++++-- lib/datadog/ci/test_visibility/recorder.rb | 7 +++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb index d81ad969..879b4b12 100644 --- a/lib/datadog/ci/itr/coverage/filter.rb +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -18,9 +18,33 @@ def initialize(root: Utils::Git.root) def call(raw_result) return nil if raw_result.nil? - raw_result.select do |path, coverage| - path =~ @regex && coverage[:lines].any? { |count| count && count > 0 } + raw_result.filter_map do |path, coverage| + next unless path =~ @regex + next unless coverage[:lines].any? { |line| !line.nil? && line > 0 } + + [path, convert_lines_to_bitmap(coverage[:lines])] + end + end + + private + + def convert_lines_to_bitmap(lines) + bitmap = [] + current = 0 + bit = 1 << 63 + lines.each do |line| + if !line.nil? && line > 0 + current |= bit + end + bit >>= 1 + if bit == 0 + bitmap << current + current = 0 + bit = 1 << 63 + end end + bitmap << current + lines end end end diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index bd36e11e..30cfc712 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -333,10 +333,9 @@ def validate_test_suite_level_visibility_correctness(test) def on_test_end(test) coverage = @coverage_collector.stop if coverage - files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } - test.set_tag("_test.coverage", files_covered) - - # p test.get_tag("_test.coverage") + # move this to the code coverage transport + # files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } + test.set_tag("_test.coverage", coverage) end end end From 16700d0eaa5a1b6ae191431b4766b8b4459550e2 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 22 Feb 2024 11:27:32 +0100 Subject: [PATCH 3/9] wip --- lib/datadog/ci/itr/coverage/filter.rb | 4 ++++ lib/datadog/ci/test_visibility/recorder.rb | 2 ++ 2 files changed, 6 insertions(+) diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb index 879b4b12..e03fbc5e 100644 --- a/lib/datadog/ci/itr/coverage/filter.rb +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -6,6 +6,7 @@ module Datadog module CI module Itr module Coverage + # not filter, but rather filter and transformer class Filter def self.call(raw_result) new.call(raw_result) @@ -18,6 +19,9 @@ def initialize(root: Utils::Git.root) def call(raw_result) return nil if raw_result.nil? + # p "RAW" + # p raw_result.count + raw_result.filter_map do |path, coverage| next unless path =~ @regex next unless coverage[:lines].any? { |line| !line.nil? && line > 0 } diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index 30cfc712..6ae2a4a9 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -333,6 +333,8 @@ def validate_test_suite_level_visibility_correctness(test) def on_test_end(test) coverage = @coverage_collector.stop if coverage + # p "FILTERED" + # p coverage.count # move this to the code coverage transport # files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } test.set_tag("_test.coverage", coverage) From 9c137b49d71149dd5ab6cdace861112712a477ab Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 22 Feb 2024 14:55:31 +0100 Subject: [PATCH 4/9] basic native extension setup --- .gitignore | 5 +++ Gemfile | 3 ++ Rakefile | 6 +++ datadog-ci.gemspec | 3 ++ ext/ddcov/ddcov.c | 83 ++++++++++++++++++++++++++++++++++++ ext/ddcov/extconf.rb | 5 +++ spec/datadog/ci/span_spec.rb | 29 +++++++++++++ 7 files changed, 134 insertions(+) create mode 100644 ext/ddcov/ddcov.c create mode 100644 ext/ddcov/extconf.rb diff --git a/.gitignore b/.gitignore index 68740aeb..81623e49 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,8 @@ Gemfile-*.lock .ruby-version .DS_Store /test.rb + +# Native extension binaries +lib/**/*.bundle +lib/**/*.so +lib/**/*.o diff --git a/Gemfile b/Gemfile index c3275d07..3c523e0b 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,9 @@ gem "pry" gem "rake" gem "os" +# To compile native extensions +gem "rake-compiler" + gem "climate_control" gem "rspec" diff --git a/Rakefile b/Rakefile index 2e9733b0..b8dbd30e 100644 --- a/Rakefile +++ b/Rakefile @@ -121,3 +121,9 @@ end desc "CI task; it runs all tests for current version of Ruby" task ci: "test:all" + +require "rake/extensiontask" + +Rake::ExtensionTask.new("ddcov") do |ext| + ext.lib_dir = "lib/ddcov" +end diff --git a/datadog-ci.gemspec b/datadog-ci.gemspec index 07157b82..2606176b 100644 --- a/datadog-ci.gemspec +++ b/datadog-ci.gemspec @@ -35,8 +35,11 @@ Gem::Specification.new do |spec| NOTICE README.md lib/**/* + ext/**/* ]].select { |fn| File.file?(fn) } # We don't want directories, only files + spec.extensions << "ext/ddcov/extconf.rb" + spec.require_paths = ["lib"] spec.add_dependency "msgpack" diff --git a/ext/ddcov/ddcov.c b/ext/ddcov/ddcov.c new file mode 100644 index 00000000..b8a55459 --- /dev/null +++ b/ext/ddcov/ddcov.c @@ -0,0 +1,83 @@ +#include + +static ID id_puts, id_each; + +static void kernel_puts(VALUE val) +{ + rb_funcall(rb_mKernel, id_puts, 1, val); +} + +static VALUE my_fixed_args_method(VALUE self, VALUE arg1, VALUE arg2) +{ + kernel_puts(self); + kernel_puts(arg1); + kernel_puts(arg2); + + return Qnil; +} + +static VALUE my_var_args_c_array_method(int argc, VALUE *argv, VALUE self) +{ + kernel_puts(self); + + for (int i = 0; i < argc; i++) + { + kernel_puts(argv[i]); + } + + return Qnil; +} + +static VALUE my_var_args_rb_array_method(VALUE self, VALUE args) +{ + kernel_puts(self); + kernel_puts(args); + + return Qnil; +} + +static VALUE my_method_with_required_block(VALUE self) +{ + VALUE block_ret = rb_yield_values(0); + kernel_puts(block_ret); + + return Qnil; +} + +static VALUE array_puts_every_other_i(VALUE yielded_arg, VALUE data, int argc, const VALUE *argv, VALUE blockarg) +{ + int *puts_cur_ptr = (int *)data; + int puts_cur = *puts_cur_ptr; + + if (puts_cur) + { + kernel_puts(yielded_arg); + } + + *puts_cur_ptr = !puts_cur; + + return Qnil; +} + +static VALUE array_puts_every_other(VALUE self) +{ + int puts_cur = 1; + + rb_block_call(self, id_each, 0, NULL, array_puts_every_other_i, (VALUE)&puts_cur); + + return Qnil; +} + +void Init_ddcov(void) +{ + id_puts = rb_intern("puts"); + + rb_define_method(rb_cObject, "my_fixed_args_method", my_fixed_args_method, 2); + rb_define_method(rb_cObject, "my_var_args_c_array_method", my_var_args_c_array_method, -1); + rb_define_method(rb_cObject, "my_var_args_rb_array_method", my_var_args_rb_array_method, -2); + rb_define_method(rb_cObject, "my_method_with_required_block", my_method_with_required_block, 0); + + id_each = rb_intern("each"); + + rb_define_method(rb_cArray, "puts_every_other", array_puts_every_other, 0); +} diff --git a/ext/ddcov/extconf.rb b/ext/ddcov/extconf.rb new file mode 100644 index 00000000..f3914b7e --- /dev/null +++ b/ext/ddcov/extconf.rb @@ -0,0 +1,5 @@ +require "mkmf" + +extension_name = "ddcov" +dir_config(extension_name) +create_makefile(extension_name) diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index 897b779a..268ef1bb 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -1,7 +1,36 @@ +require_relative "../../../lib/ddcov/ddcov" + RSpec.describe Datadog::CI::Span do let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, name: "span_name", type: "test") } subject(:span) { described_class.new(tracer_span) } + describe "ddcov" do + puts "----- Testing Object#my_fixed_args_method -----" + + "I am self".my_fixed_args_method("Hi from argument 1", "Hi from argument 2") + + puts + puts "----- Testing Object#my_var_args_c_array_method -----" + + "Hi from self".my_var_args_c_array_method("1", "2", "3", "4") + + puts + puts "----- Testing Object#my_var_args_rb_array_method -----" + + "Hi from self".my_var_args_rb_array_method("1", "2") + + puts + puts "----- Testing Object#my_method_with_required_block -----" + + my_method_with_required_block do + "foo" + end + + puts "---- Test Array#puts_every_other ----" + + [1, 2, 3, 4, 5, 6, 7].puts_every_other + end + describe "#name" do it "returns the span name" do expect(span.name).to eq("span_name") From 663a4b70c4bbdbe23b2d6119e426552cdcf5f3b9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 22 Feb 2024 16:17:40 +0100 Subject: [PATCH 5/9] DDCov tracks RUBY_EVENT_LINE --- ext/ddcov/ddcov.c | 86 ++++++++++++++---------------------- spec/datadog/ci/span_spec.rb | 27 ----------- 2 files changed, 32 insertions(+), 81 deletions(-) diff --git a/ext/ddcov/ddcov.c b/ext/ddcov/ddcov.c index b8a55459..ce420311 100644 --- a/ext/ddcov/ddcov.c +++ b/ext/ddcov/ddcov.c @@ -1,83 +1,61 @@ #include +#include +#include -static ID id_puts, id_each; +static ID id_puts; static void kernel_puts(VALUE val) { rb_funcall(rb_mKernel, id_puts, 1, val); } -static VALUE my_fixed_args_method(VALUE self, VALUE arg1, VALUE arg2) -{ - kernel_puts(self); - kernel_puts(arg1); - kernel_puts(arg2); - - return Qnil; -} - -static VALUE my_var_args_c_array_method(int argc, VALUE *argv, VALUE self) -{ - kernel_puts(self); - - for (int i = 0; i < argc; i++) - { - kernel_puts(argv[i]); - } - - return Qnil; -} +VALUE DDCovClass = Qnil; -static VALUE my_var_args_rb_array_method(VALUE self, VALUE args) +VALUE dd_cov_initialize(VALUE self) { - kernel_puts(self); - kernel_puts(args); - - return Qnil; + rb_iv_set(self, "@var", rb_hash_new()); + return self; } -static VALUE my_method_with_required_block(VALUE self) +void dd_cov_update_line_coverage(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass) { - VALUE block_ret = rb_yield_values(0); - kernel_puts(block_ret); - - return Qnil; + printf("EVENT HOOK FIRED\n"); + printf("FILE: %s\n", rb_sourcefile()); + printf("LINE: %d\n", rb_sourceline()); + kernel_puts(klass); + // kernel_puts(event); + // kernel_puts(data); + // kernel_puts(self); + // kernel_puts(id); } -static VALUE array_puts_every_other_i(VALUE yielded_arg, VALUE data, int argc, const VALUE *argv, VALUE blockarg) +VALUE dd_cov_start(VALUE self) { - int *puts_cur_ptr = (int *)data; - int puts_cur = *puts_cur_ptr; - - if (puts_cur) - { - kernel_puts(yielded_arg); - } + // get current thread + VALUE thval = rb_thread_current(); - *puts_cur_ptr = !puts_cur; + // add event hook + rb_thread_add_event_hook(thval, dd_cov_update_line_coverage, RUBY_EVENT_LINE, Qnil); - return Qnil; + return self; } -static VALUE array_puts_every_other(VALUE self) +VALUE dd_cov_stop(VALUE self) { - int puts_cur = 1; + // get current thread + VALUE thval = rb_thread_current(); - rb_block_call(self, id_each, 0, NULL, array_puts_every_other_i, (VALUE)&puts_cur); - - return Qnil; + // remove event hook + rb_thread_remove_event_hook(thval, dd_cov_update_line_coverage); + return self; } void Init_ddcov(void) { id_puts = rb_intern("puts"); - rb_define_method(rb_cObject, "my_fixed_args_method", my_fixed_args_method, 2); - rb_define_method(rb_cObject, "my_var_args_c_array_method", my_var_args_c_array_method, -1); - rb_define_method(rb_cObject, "my_var_args_rb_array_method", my_var_args_rb_array_method, -2); - rb_define_method(rb_cObject, "my_method_with_required_block", my_method_with_required_block, 0); - - id_each = rb_intern("each"); - - rb_define_method(rb_cArray, "puts_every_other", array_puts_every_other, 0); + DDCovClass = rb_define_class("DDCov", rb_cObject); + rb_define_method(DDCovClass, "initialize", dd_cov_initialize, 0); + rb_define_method(DDCovClass, "start", dd_cov_start, 0); + rb_define_method(DDCovClass, "stop", dd_cov_stop, 0); } diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index 268ef1bb..eafe2d73 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -4,33 +4,6 @@ let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, name: "span_name", type: "test") } subject(:span) { described_class.new(tracer_span) } - describe "ddcov" do - puts "----- Testing Object#my_fixed_args_method -----" - - "I am self".my_fixed_args_method("Hi from argument 1", "Hi from argument 2") - - puts - puts "----- Testing Object#my_var_args_c_array_method -----" - - "Hi from self".my_var_args_c_array_method("1", "2", "3", "4") - - puts - puts "----- Testing Object#my_var_args_rb_array_method -----" - - "Hi from self".my_var_args_rb_array_method("1", "2") - - puts - puts "----- Testing Object#my_method_with_required_block -----" - - my_method_with_required_block do - "foo" - end - - puts "---- Test Array#puts_every_other ----" - - [1, 2, 3, 4, 5, 6, 7].puts_every_other - end - describe "#name" do it "returns the span name" do expect(span.name).to eq("span_name") From 75161687b0b8d90edab54c422ebc71e9ec2be2b8 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 22 Feb 2024 17:51:37 +0100 Subject: [PATCH 6/9] track covered source files with DDCov --- ext/ddcov/ddcov.c | 34 +++++++++++----------- lib/datadog/ci/itr/coverage/collector.rb | 24 ++++----------- lib/datadog/ci/itr/coverage/filter.rb | 3 +- lib/datadog/ci/test_visibility/recorder.rb | 1 + spec/datadog/ci/span_spec.rb | 2 -- 5 files changed, 25 insertions(+), 39 deletions(-) diff --git a/ext/ddcov/ddcov.c b/ext/ddcov/ddcov.c index ce420311..9fe245a9 100644 --- a/ext/ddcov/ddcov.c +++ b/ext/ddcov/ddcov.c @@ -2,13 +2,6 @@ #include #include -static ID id_puts; - -static void kernel_puts(VALUE val) -{ - rb_funcall(rb_mKernel, id_puts, 1, val); -} - VALUE DDCovClass = Qnil; VALUE dd_cov_initialize(VALUE self) @@ -19,14 +12,20 @@ VALUE dd_cov_initialize(VALUE self) void dd_cov_update_line_coverage(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass) { - printf("EVENT HOOK FIRED\n"); - printf("FILE: %s\n", rb_sourcefile()); - printf("LINE: %d\n", rb_sourceline()); - kernel_puts(klass); - // kernel_puts(event); - // kernel_puts(data); - // kernel_puts(self); - // kernel_puts(id); + // printf("EVENT HOOK FIRED\n"); + // printf("FILE: %s\n", rb_sourcefile()); + // printf("LINE: %d\n", rb_sourceline()); + + const char *filename = rb_sourcefile(); + if (filename == 0) + { + return; + } + + unsigned long filename_length = strlen(filename); + + VALUE rb_str_source_file = rb_str_new(filename, filename_length); + rb_hash_aset(rb_iv_get(data, "@var"), rb_str_source_file, Qtrue); } VALUE dd_cov_start(VALUE self) @@ -35,7 +34,7 @@ VALUE dd_cov_start(VALUE self) VALUE thval = rb_thread_current(); // add event hook - rb_thread_add_event_hook(thval, dd_cov_update_line_coverage, RUBY_EVENT_LINE, Qnil); + rb_thread_add_event_hook(thval, dd_cov_update_line_coverage, RUBY_EVENT_LINE, self); return self; } @@ -47,7 +46,8 @@ VALUE dd_cov_stop(VALUE self) // remove event hook rb_thread_remove_event_hook(thval, dd_cov_update_line_coverage); - return self; + + return rb_iv_get(self, "@var"); } void Init_ddcov(void) diff --git a/lib/datadog/ci/itr/coverage/collector.rb b/lib/datadog/ci/itr/coverage/collector.rb index ac6ed911..f4093edb 100644 --- a/lib/datadog/ci/itr/coverage/collector.rb +++ b/lib/datadog/ci/itr/coverage/collector.rb @@ -3,6 +3,7 @@ require "coverage" require_relative "filter" +require_relative "../../../../ddcov/ddcov" module Datadog module CI @@ -10,33 +11,20 @@ module Itr module Coverage class Collector def initialize - # Do not run code coverage if someone else is already running it. - # It means that user is running the test with coverage and ITR would mess it up. - @coverage_supported = !::Coverage.running? - # @coverage_supported = false + @ddcov = DDCov.new end def setup - if @coverage_supported - p "RUNNING WITH CODE COVERAGE ENABLED!" - ::Coverage.setup(lines: true) - else - p "RUNNING WITH CODE COVERAGE DISABLED!" - end + p "RUNNING WITH CODE COVERAGE ENABLED" end def start - return unless @coverage_supported - - # if execution is threaded then coverage might already be running - ::Coverage.resume unless ::Coverage.running? + @ddcov.start end def stop - return nil unless @coverage_supported - - result = ::Coverage.result(stop: false, clear: true) - ::Coverage.suspend if ::Coverage.running? + result = @ddcov.stop + @ddcov.instance_variable_set(:@var, {}) Filter.call(result) end diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb index e03fbc5e..c2cf7bd8 100644 --- a/lib/datadog/ci/itr/coverage/filter.rb +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -24,9 +24,8 @@ def call(raw_result) raw_result.filter_map do |path, coverage| next unless path =~ @regex - next unless coverage[:lines].any? { |line| !line.nil? && line > 0 } - [path, convert_lines_to_bitmap(coverage[:lines])] + [path, coverage] end end diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index 6ae2a4a9..c306f960 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -335,6 +335,7 @@ def on_test_end(test) if coverage # p "FILTERED" # p coverage.count + # p coverage # move this to the code coverage transport # files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } test.set_tag("_test.coverage", coverage) diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index eafe2d73..897b779a 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -1,5 +1,3 @@ -require_relative "../../../lib/ddcov/ddcov" - RSpec.describe Datadog::CI::Span do let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, name: "span_name", type: "test") } subject(:span) { described_class.new(tracer_span) } From 522f590e5d3493f90b913ed4257d990409b83b17 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 23 Feb 2024 15:18:53 +0100 Subject: [PATCH 7/9] optimized version of ddcov --- ext/ddcov/ddcov.c | 107 +++++++++++++++++++---- lib/datadog/ci/itr/coverage/collector.rb | 14 ++- 2 files changed, 96 insertions(+), 25 deletions(-) diff --git a/ext/ddcov/ddcov.c b/ext/ddcov/ddcov.c index 9fe245a9..417949de 100644 --- a/ext/ddcov/ddcov.c +++ b/ext/ddcov/ddcov.c @@ -2,19 +2,80 @@ #include #include -VALUE DDCovClass = Qnil; +// Utils +static bool prefix(const char *pre, const char *str) +{ + return strncmp(pre, str, strlen(pre)) == 0; +} -VALUE dd_cov_initialize(VALUE self) +// Data structure +struct dd_cov_data { - rb_iv_set(self, "@var", rb_hash_new()); - return self; + char *root; + VALUE coverage; +}; + +static void dd_cov_mark(void *ptr) +{ + // printf("MARK\n"); + struct dd_cov_data *dd_cov_data = ptr; + rb_gc_mark_movable(dd_cov_data->coverage); +} + +static void dd_cov_free(void *ptr) +{ + // printf("FREE\n"); + struct dd_cov_data *dd_cov_data = ptr; + + xfree(dd_cov_data); +} + +static void dd_cov_compact(void *ptr) +{ + // printf("COMPACT\n"); + struct dd_cov_data *dd_cov_data = ptr; + dd_cov_data->coverage = rb_gc_location(dd_cov_data->coverage); +} + +const rb_data_type_t dd_cov_data_type = { + .wrap_struct_name = "dd_cov", + .function = { + .dmark = dd_cov_mark, + .dfree = dd_cov_free, + .dsize = NULL, + .dcompact = dd_cov_compact}, + .flags = RUBY_TYPED_FREE_IMMEDIATELY}; + +static VALUE dd_cov_allocate(VALUE klass) +{ + // printf("ALLOCATE\n"); + struct dd_cov_data *dd_cov_data; + VALUE obj = TypedData_Make_Struct(klass, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); + dd_cov_data->coverage = rb_hash_new(); + return obj; +} + +// DDCov methods +static VALUE dd_cov_initialize(VALUE self, VALUE rb_root) +{ + // printf("INITIALIZE\n"); + struct dd_cov_data *dd_cov_data; + TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); + + // printf("struct got\n"); + dd_cov_data->root = StringValueCStr(rb_root); + // printf("root set\n"); + + return Qnil; } -void dd_cov_update_line_coverage(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass) +static void dd_cov_update_line_coverage(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass) { // printf("EVENT HOOK FIRED\n"); // printf("FILE: %s\n", rb_sourcefile()); // printf("LINE: %d\n", rb_sourceline()); + struct dd_cov_data *dd_cov_data; + TypedData_Get_Struct(data, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); const char *filename = rb_sourcefile(); if (filename == 0) @@ -22,13 +83,18 @@ void dd_cov_update_line_coverage(rb_event_flag_t event, VALUE data, VALUE self, return; } - unsigned long filename_length = strlen(filename); + if (!prefix(dd_cov_data->root, filename)) + { + return; + } + + unsigned long len_filename = strlen(filename); - VALUE rb_str_source_file = rb_str_new(filename, filename_length); - rb_hash_aset(rb_iv_get(data, "@var"), rb_str_source_file, Qtrue); + VALUE rb_str_source_file = rb_str_new(filename, len_filename); + rb_hash_aset(dd_cov_data->coverage, rb_str_source_file, Qtrue); } -VALUE dd_cov_start(VALUE self) +static VALUE dd_cov_start(VALUE self) { // get current thread VALUE thval = rb_thread_current(); @@ -39,23 +105,30 @@ VALUE dd_cov_start(VALUE self) return self; } -VALUE dd_cov_stop(VALUE self) +static VALUE dd_cov_stop(VALUE self) { // get current thread VALUE thval = rb_thread_current(); - // remove event hook rb_thread_remove_event_hook(thval, dd_cov_update_line_coverage); - return rb_iv_get(self, "@var"); + struct dd_cov_data *dd_cov_data; + TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); + + VALUE cov = dd_cov_data->coverage; + + dd_cov_data->coverage = rb_hash_new(); + + return cov; } void Init_ddcov(void) { - id_puts = rb_intern("puts"); + VALUE cDDCov = rb_define_class("DDCov", rb_cObject); + + rb_define_alloc_func(cDDCov, dd_cov_allocate); - DDCovClass = rb_define_class("DDCov", rb_cObject); - rb_define_method(DDCovClass, "initialize", dd_cov_initialize, 0); - rb_define_method(DDCovClass, "start", dd_cov_start, 0); - rb_define_method(DDCovClass, "stop", dd_cov_stop, 0); + rb_define_method(cDDCov, "initialize", dd_cov_initialize, 1); + rb_define_method(cDDCov, "start", dd_cov_start, 0); + rb_define_method(cDDCov, "stop", dd_cov_stop, 0); } diff --git a/lib/datadog/ci/itr/coverage/collector.rb b/lib/datadog/ci/itr/coverage/collector.rb index f4093edb..b58b2125 100644 --- a/lib/datadog/ci/itr/coverage/collector.rb +++ b/lib/datadog/ci/itr/coverage/collector.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true -require "coverage" - -require_relative "filter" +require_relative "../../utils/git" require_relative "../../../../ddcov/ddcov" module Datadog @@ -11,7 +9,7 @@ module Itr module Coverage class Collector def initialize - @ddcov = DDCov.new + # TODO: make this thread local end def setup @@ -19,14 +17,14 @@ def setup end def start + @ddcov = DDCov.new(Utils::Git.root) @ddcov.start end def stop - result = @ddcov.stop - @ddcov.instance_variable_set(:@var, {}) - - Filter.call(result) + @ddcov.stop + # p "RAW" + # p result.count end end end From ee647e5adf28e5e9fe6c8ed090e1cf82238c3c90 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 26 Feb 2024 17:30:38 +0100 Subject: [PATCH 8/9] lines coverage support, DD_COV_REPORT support --- ext/ddcov/ddcov.c | 61 ++++++++++++++++++++-- lib/datadog/ci/itr/coverage/collector.rb | 24 ++++++--- lib/datadog/ci/test_visibility/recorder.rb | 16 +++++- 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/ext/ddcov/ddcov.c b/ext/ddcov/ddcov.c index 417949de..7dc2bc09 100644 --- a/ext/ddcov/ddcov.c +++ b/ext/ddcov/ddcov.c @@ -8,10 +8,15 @@ static bool prefix(const char *pre, const char *str) return strncmp(pre, str, strlen(pre)) == 0; } +// const +#define DD_COVERAGE_TARGET_FILES 1 +#define DD_COVERAGE_TARGET_LINES 2 + // Data structure struct dd_cov_data { char *root; + int mode; VALUE coverage; }; @@ -56,14 +61,39 @@ static VALUE dd_cov_allocate(VALUE klass) } // DDCov methods -static VALUE dd_cov_initialize(VALUE self, VALUE rb_root) +static VALUE dd_cov_initialize(int argc, VALUE *argv, VALUE self) { - // printf("INITIALIZE\n"); + VALUE opt; + int mode; + + rb_scan_args(argc, argv, "10", &opt); + VALUE rb_root = rb_hash_lookup(opt, ID2SYM(rb_intern("root"))); + if (!RTEST(rb_root)) + { + rb_raise(rb_eArgError, "root is required"); + } + + VALUE rb_mode = rb_hash_lookup(opt, ID2SYM(rb_intern("mode"))); + if (!RTEST(rb_mode) || rb_mode == ID2SYM(rb_intern("files"))) + { + mode = DD_COVERAGE_TARGET_FILES; + } + else if (rb_mode == ID2SYM(rb_intern("lines"))) + { + mode = DD_COVERAGE_TARGET_LINES; + } + else + { + rb_raise(rb_eArgError, "mode is invalid"); + } + + printf("MODE IS %d\n", mode); struct dd_cov_data *dd_cov_data; TypedData_Get_Struct(self, struct dd_cov_data, &dd_cov_data_type, dd_cov_data); // printf("struct got\n"); dd_cov_data->root = StringValueCStr(rb_root); + dd_cov_data->mode = mode; // printf("root set\n"); return Qnil; @@ -91,7 +121,28 @@ static void dd_cov_update_line_coverage(rb_event_flag_t event, VALUE data, VALUE unsigned long len_filename = strlen(filename); VALUE rb_str_source_file = rb_str_new(filename, len_filename); - rb_hash_aset(dd_cov_data->coverage, rb_str_source_file, Qtrue); + + if (dd_cov_data->mode == DD_COVERAGE_TARGET_FILES) + { + rb_hash_aset(dd_cov_data->coverage, rb_str_source_file, Qtrue); + return; + } + + if (dd_cov_data->mode == DD_COVERAGE_TARGET_LINES) + { + VALUE rb_lines = rb_hash_aref(dd_cov_data->coverage, rb_str_source_file); + if (rb_lines == Qnil) + { + rb_lines = rb_ary_new(); + rb_hash_aset(dd_cov_data->coverage, rb_str_source_file, rb_lines); + } + + VALUE line_number = INT2FIX(rb_sourceline()); + if (rb_ary_includes(rb_lines, line_number) == Qfalse) + { + rb_ary_push(rb_lines, line_number); + } + } } static VALUE dd_cov_start(VALUE self) @@ -109,7 +160,7 @@ static VALUE dd_cov_stop(VALUE self) { // get current thread VALUE thval = rb_thread_current(); - // remove event hook + // remove event hook for the current thread rb_thread_remove_event_hook(thval, dd_cov_update_line_coverage); struct dd_cov_data *dd_cov_data; @@ -128,7 +179,7 @@ void Init_ddcov(void) rb_define_alloc_func(cDDCov, dd_cov_allocate); - rb_define_method(cDDCov, "initialize", dd_cov_initialize, 1); + rb_define_method(cDDCov, "initialize", dd_cov_initialize, -1); rb_define_method(cDDCov, "start", dd_cov_start, 0); rb_define_method(cDDCov, "stop", dd_cov_stop, 0); } diff --git a/lib/datadog/ci/itr/coverage/collector.rb b/lib/datadog/ci/itr/coverage/collector.rb index b58b2125..0c4c4460 100644 --- a/lib/datadog/ci/itr/coverage/collector.rb +++ b/lib/datadog/ci/itr/coverage/collector.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "coverage" + require_relative "../../utils/git" require_relative "../../../../ddcov/ddcov" @@ -8,23 +10,31 @@ module CI module Itr module Coverage class Collector - def initialize + def initialize(mode: :files, enabled: true) # TODO: make this thread local + # modes available: :files, :lines + @mode = mode + @enabled = enabled + + if @enabled + @ddcov = DDCov.new(root: Utils::Git.root, mode: mode) + end end def setup - p "RUNNING WITH CODE COVERAGE ENABLED" + if @enabled + p "RUNNING WITH CODE COVERAGE ENABLED" + else + p "RUNNING WITH CODE COVERAGE DISABLED" + end end def start - @ddcov = DDCov.new(Utils::Git.root) - @ddcov.start + @ddcov.start if @enabled end def stop - @ddcov.stop - # p "RAW" - # p result.count + @ddcov.stop if @enabled end end end diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index c306f960..bf444e01 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -41,7 +41,7 @@ def initialize( @global_context = Context::Global.new @codeowners = codeowners - @coverage_collector = Itr::Coverage::Collector.new + @coverage_collector = Itr::Coverage::Collector.new(mode: :lines, enabled: true) begin @coverage_collector.setup rescue => e @@ -333,6 +333,20 @@ def validate_test_suite_level_visibility_correctness(test) def on_test_end(test) coverage = @coverage_collector.stop if coverage + if ENV["DD_COV_REPORT"] + # append to report.log file + File.open("report.log", "a") do |f| + f.write("#{test.name}\n") + f.write("---------------------------------------------------\n") + + coverage.each do |filename, lines| + f.write("#{filename}\n") + sorted_lines = lines.uniq.sort + f.write("#{sorted_lines}\n") + end + f.write("---------------------------------------------------\n") + end + end # p "FILTERED" # p coverage.count # p coverage From fb8b92f7512263720c99cded5c4cb272a6bf8bfb Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 27 Feb 2024 14:40:30 +0100 Subject: [PATCH 9/9] print mode --- lib/datadog/ci/itr/coverage/collector.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datadog/ci/itr/coverage/collector.rb b/lib/datadog/ci/itr/coverage/collector.rb index 0c4c4460..785c86ab 100644 --- a/lib/datadog/ci/itr/coverage/collector.rb +++ b/lib/datadog/ci/itr/coverage/collector.rb @@ -23,7 +23,7 @@ def initialize(mode: :files, enabled: true) def setup if @enabled - p "RUNNING WITH CODE COVERAGE ENABLED" + p "RUNNING WITH CODE COVERAGE ENABLED AND MODE #{@mode}" else p "RUNNING WITH CODE COVERAGE DISABLED" end