From 2d24951a0eb89f3482b01c75d8f2aebfbcdf5a34 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 21 Feb 2024 13:14:22 +0100 Subject: [PATCH 1/4] sinmplest code coverage with resume/suspend POC --- lib/datadog/ci/itr/coverage/collector.rb | 47 ++++++++++++++++++++++ lib/datadog/ci/itr/coverage/filter.rb | 29 +++++++++++++ lib/datadog/ci/test_visibility/recorder.rb | 25 ++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 lib/datadog/ci/itr/coverage/collector.rb create mode 100644 lib/datadog/ci/itr/coverage/filter.rb diff --git a/lib/datadog/ci/itr/coverage/collector.rb b/lib/datadog/ci/itr/coverage/collector.rb new file mode 100644 index 00000000..ac6ed911 --- /dev/null +++ b/lib/datadog/ci/itr/coverage/collector.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "coverage" + +require_relative "filter" + +module Datadog + module CI + module Itr + module Coverage + class Collector + def initialize + # Do not run code coverage if someone else is already running it. + # It means that user is running the test with coverage and ITR would mess it up. + @coverage_supported = !::Coverage.running? + # @coverage_supported = false + end + + def setup + if @coverage_supported + p "RUNNING WITH CODE COVERAGE ENABLED!" + ::Coverage.setup(lines: true) + else + p "RUNNING WITH CODE COVERAGE DISABLED!" + end + end + + def start + return unless @coverage_supported + + # if execution is threaded then coverage might already be running + ::Coverage.resume unless ::Coverage.running? + end + + def stop + return nil unless @coverage_supported + + result = ::Coverage.result(stop: false, clear: true) + ::Coverage.suspend if ::Coverage.running? + + Filter.call(result) + end + end + end + end + end +end diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb new file mode 100644 index 00000000..d81ad969 --- /dev/null +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require_relative "../../utils/git" + +module Datadog + module CI + module Itr + module Coverage + class Filter + def self.call(raw_result) + new.call(raw_result) + end + + def initialize(root: Utils::Git.root) + @regex = /\A#{Regexp.escape(root + File::SEPARATOR)}/i.freeze + end + + def call(raw_result) + return nil if raw_result.nil? + + raw_result.select do |path, coverage| + path =~ @regex && coverage[:lines].any? { |count| count && count > 0 } + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index c274590f..bd36e11e 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -20,6 +20,8 @@ require_relative "../test_module" require_relative "../test_suite" +require_relative "../itr/coverage/collector" + module Datadog module CI module TestVisibility @@ -38,6 +40,13 @@ def initialize( @local_context = Context::Local.new @global_context = Context::Global.new @codeowners = codeowners + + @coverage_collector = Itr::Coverage::Collector.new + begin + @coverage_collector.setup + rescue => e + Datadog.logger.debug("Coverage collector setup failed: #{e.message}") + end end def start_test_session(service: nil, tags: {}) @@ -103,12 +112,15 @@ def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block) {resource: test_name, continue_from: Datadog::Tracing::TraceDigest.new} ) + @coverage_collector.start if block start_datadog_tracer_span(test_name, span_options) do |tracer_span| test = build_test(tracer_span, tags) @local_context.activate_test(test) do block.call(test) + + on_test_end(test) end end else @@ -160,6 +172,8 @@ def active_test_suite(test_suite_name) end def deactivate_test + on_test_end(active_test) if active_test + @local_context.deactivate_test end @@ -314,6 +328,17 @@ def validate_test_suite_level_visibility_correctness(test) end end end + + # event system? + def on_test_end(test) + coverage = @coverage_collector.stop + if coverage + files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } + test.set_tag("_test.coverage", files_covered) + + # p test.get_tag("_test.coverage") + end + end end end end From ab0257e224d8cfcf539a6fd320deb0d486aa1654 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 21 Feb 2024 19:20:45 +0100 Subject: [PATCH 2/4] construct line coverage bitmaps while filtering --- lib/datadog/ci/itr/coverage/filter.rb | 28 ++++++++++++++++++++-- lib/datadog/ci/test_visibility/recorder.rb | 7 +++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb index d81ad969..879b4b12 100644 --- a/lib/datadog/ci/itr/coverage/filter.rb +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -18,9 +18,33 @@ def initialize(root: Utils::Git.root) def call(raw_result) return nil if raw_result.nil? - raw_result.select do |path, coverage| - path =~ @regex && coverage[:lines].any? { |count| count && count > 0 } + raw_result.filter_map do |path, coverage| + next unless path =~ @regex + next unless coverage[:lines].any? { |line| !line.nil? && line > 0 } + + [path, convert_lines_to_bitmap(coverage[:lines])] + end + end + + private + + def convert_lines_to_bitmap(lines) + bitmap = [] + current = 0 + bit = 1 << 63 + lines.each do |line| + if !line.nil? && line > 0 + current |= bit + end + bit >>= 1 + if bit == 0 + bitmap << current + current = 0 + bit = 1 << 63 + end end + bitmap << current + lines end end end diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index bd36e11e..30cfc712 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -333,10 +333,9 @@ def validate_test_suite_level_visibility_correctness(test) def on_test_end(test) coverage = @coverage_collector.stop if coverage - files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } - test.set_tag("_test.coverage", files_covered) - - # p test.get_tag("_test.coverage") + # move this to the code coverage transport + # files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } + test.set_tag("_test.coverage", coverage) end end end From 16700d0eaa5a1b6ae191431b4766b8b4459550e2 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 22 Feb 2024 11:27:32 +0100 Subject: [PATCH 3/4] wip --- lib/datadog/ci/itr/coverage/filter.rb | 4 ++++ lib/datadog/ci/test_visibility/recorder.rb | 2 ++ 2 files changed, 6 insertions(+) diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb index 879b4b12..e03fbc5e 100644 --- a/lib/datadog/ci/itr/coverage/filter.rb +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -6,6 +6,7 @@ module Datadog module CI module Itr module Coverage + # not filter, but rather filter and transformer class Filter def self.call(raw_result) new.call(raw_result) @@ -18,6 +19,9 @@ def initialize(root: Utils::Git.root) def call(raw_result) return nil if raw_result.nil? + # p "RAW" + # p raw_result.count + raw_result.filter_map do |path, coverage| next unless path =~ @regex next unless coverage[:lines].any? { |line| !line.nil? && line > 0 } diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index 30cfc712..6ae2a4a9 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -333,6 +333,8 @@ def validate_test_suite_level_visibility_correctness(test) def on_test_end(test) coverage = @coverage_collector.stop if coverage + # p "FILTERED" + # p coverage.count # move this to the code coverage transport # files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } test.set_tag("_test.coverage", coverage) From b3bacccfb2a75497731973e7c7d260c3a39b3d4b Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 26 Feb 2024 17:53:58 +0100 Subject: [PATCH 4/4] DD_COV_REPORT support --- lib/datadog/ci/itr/coverage/collector.rb | 9 ++++--- lib/datadog/ci/itr/coverage/filter.rb | 31 ++-------------------- lib/datadog/ci/test_visibility/recorder.rb | 25 ++++++++++++++--- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/lib/datadog/ci/itr/coverage/collector.rb b/lib/datadog/ci/itr/coverage/collector.rb index ac6ed911..5c532f45 100644 --- a/lib/datadog/ci/itr/coverage/collector.rb +++ b/lib/datadog/ci/itr/coverage/collector.rb @@ -9,15 +9,16 @@ module CI module Itr module Coverage class Collector - def initialize + def initialize(enabled: true) # Do not run code coverage if someone else is already running it. # It means that user is running the test with coverage and ITR would mess it up. + @enabled = enabled @coverage_supported = !::Coverage.running? # @coverage_supported = false end def setup - if @coverage_supported + if @coverage_supported && @enabled p "RUNNING WITH CODE COVERAGE ENABLED!" ::Coverage.setup(lines: true) else @@ -26,14 +27,14 @@ def setup end def start - return unless @coverage_supported + return if !@coverage_supported || !@enabled # if execution is threaded then coverage might already be running ::Coverage.resume unless ::Coverage.running? end def stop - return nil unless @coverage_supported + return if !@coverage_supported || !@enabled result = ::Coverage.result(stop: false, clear: true) ::Coverage.suspend if ::Coverage.running? diff --git a/lib/datadog/ci/itr/coverage/filter.rb b/lib/datadog/ci/itr/coverage/filter.rb index e03fbc5e..acc19419 100644 --- a/lib/datadog/ci/itr/coverage/filter.rb +++ b/lib/datadog/ci/itr/coverage/filter.rb @@ -19,36 +19,9 @@ def initialize(root: Utils::Git.root) def call(raw_result) return nil if raw_result.nil? - # p "RAW" - # p raw_result.count - - raw_result.filter_map do |path, coverage| - next unless path =~ @regex - next unless coverage[:lines].any? { |line| !line.nil? && line > 0 } - - [path, convert_lines_to_bitmap(coverage[:lines])] - end - end - - private - - def convert_lines_to_bitmap(lines) - bitmap = [] - current = 0 - bit = 1 << 63 - lines.each do |line| - if !line.nil? && line > 0 - current |= bit - end - bit >>= 1 - if bit == 0 - bitmap << current - current = 0 - bit = 1 << 63 - end + raw_result.select do |path, coverage| + path =~ @regex && coverage[:lines].any? { |line| !line.nil? && line > 0 } end - bitmap << current - lines end end end diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index 6ae2a4a9..278fe0e3 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -333,10 +333,27 @@ def validate_test_suite_level_visibility_correctness(test) def on_test_end(test) coverage = @coverage_collector.stop if coverage - # p "FILTERED" - # p coverage.count - # move this to the code coverage transport - # files_covered = coverage.keys.map { |filename| Utils::Git.relative_to_root(filename) } + if ENV["DD_COV_REPORT"] + # append to report.log file + File.open("report.log", "a") do |f| + f.write("#{test.name}\n") + f.write("---------------------------------------------------\n") + + coverage.each do |filename, cov| + f.write("#{filename}\n") + sorted_lines = [] + + cov[:lines].each_with_index do |count, line_number| + next if count.nil? || count.zero? + + sorted_lines << line_number + 1 + end + f.write("#{sorted_lines.sort}\n") + end + f.write("---------------------------------------------------\n") + end + end + test.set_tag("_test.coverage", coverage) end end