do not send coverage events for tests skipped by test framework or by…

… ITR

lib/datadog/ci/itr/runner.rb

@@ -89,8 +89,9 @@ def code_coverage?
           @code_coverage_enabled
         end
 
-        def start_coverage
+        def start_coverage(test)
           return if !enabled? || !code_coverage?
+          return if test.skipped_by_itr?
 
           coverage_collector&.start
         end
@@ -99,7 +100,9 @@ def stop_coverage(test)
           return if !enabled? || !code_coverage?
 
           coverage = coverage_collector&.stop
-          return if coverage.nil?
+          return if coverage.nil? || coverage.empty?
+
+          return if test.skipped? || test.skipped_by_itr?
 
           test_source_file = test.source_file
 
@@ -121,7 +124,7 @@ def stop_coverage(test)
         end
 
         def mark_if_skippable(test)
-          return unless enabled? && skipping_tests?
+          return if !enabled? || !skipping_tests?
 
           test_full_name = Utils::TestRun.test_full_name(test.name, test.test_suite_name)
 

lib/datadog/ci/test_visibility/recorder.rb

@@ -402,7 +402,7 @@ def on_test_finished(test)
 
         def on_test_started(test)
           @itr.mark_if_skippable(test)
-          @itr.start_coverage
+          @itr.start_coverage(test)
         end
       end
     end

sig/datadog/ci/itr/runner.rbs

@@ -27,7 +27,7 @@ module Datadog
 
         def code_coverage?: () -> bool
 
-        def start_coverage: () -> void
+        def start_coverage: (Datadog::CI::Test test) -> void
 
         def stop_coverage: (Datadog::CI::Test test) -> Datadog::CI::ITR::Coverage::Event?
 

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -189,12 +189,12 @@
     context "collecting coverage with features dir as root" do
       before { skip if PlatformHelpers.jruby? }
 
-      it "creates coverage events for each test" do
-        expect(coverage_events).to have(4).items
+      it "creates coverage events for each non-skipped test" do
+        expect(coverage_events).to have(1).item
 
         expect_coverage_events_belong_to_session(test_session_span)
         expect_coverage_events_belong_to_suite(first_test_suite_span)
-        expect_coverage_events_belong_to_tests(test_spans)
+        expect_coverage_events_belong_to_tests([test_spans.first])
         expect_non_empty_coverages
 
         feature_coverage = coverage_events.first.coverage

spec/datadog/ci/itr/runner_spec.rb

@@ -122,7 +122,7 @@
       it "does not start coverage" do
         expect(runner).not_to receive(:coverage_collector)
 
-        runner.start_coverage
+        runner.start_coverage(test_span)
         expect(runner.stop_coverage(test_span)).to be_nil
       end
     end
@@ -133,7 +133,7 @@
       it "does not start coverage" do
         expect(runner).not_to receive(:coverage_collector)
 
-        runner.start_coverage
+        runner.start_coverage(test_span)
         expect(runner.stop_coverage(test_span)).to be_nil
       end
     end
@@ -148,11 +148,21 @@
       it "starts coverage" do
         expect(runner).to receive(:coverage_collector).twice.and_call_original
 
-        runner.start_coverage
+        runner.start_coverage(test_span)
         expect(1 + 1).to eq(2)
         coverage_event = runner.stop_coverage(test_span)
         expect(coverage_event.coverage.size).to be > 0
       end
+
+      context "when test is skipped by ITR" do
+        it "does not start coverage" do
+          test_span.set_tag(Datadog::CI::Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
+
+          expect(runner).not_to receive(:coverage_collector)
+
+          runner.start_coverage(test_span)
+        end
+      end
     end
 
     context "when JRuby and code coverage is enabled" do
@@ -166,7 +176,7 @@
         expect(runner).not_to receive(:coverage_collector)
         expect(runner.code_coverage?).to be(false)
 
-        runner.start_coverage
+        runner.start_coverage(test_span)
         expect(runner.stop_coverage(test_span)).to be_nil
       end
     end
@@ -182,13 +192,13 @@
 
       configure
 
-      expect(test_span).to receive(:id).and_return(1)
-      expect(test_span).to receive(:test_suite_id).and_return(2)
-      expect(test_span).to receive(:test_session_id).and_return(3)
+      allow(test_span).to receive(:id).and_return(1)
+      allow(test_span).to receive(:test_suite_id).and_return(2)
+      allow(test_span).to receive(:test_session_id).and_return(3)
     end
 
     it "creates coverage event and writes it" do
-      runner.start_coverage
+      runner.start_coverage(test_span)
       expect(1 + 1).to eq(2)
       expect(runner.stop_coverage(test_span)).not_to be_nil
 
@@ -200,6 +210,37 @@
         expect(event.coverage.size).to be > 0
       end
     end
+
+    context "when test is skipped" do
+      it "does not write coverage event" do
+        runner.start_coverage(test_span)
+        expect(1 + 1).to eq(2)
+        test_span.set_tag(Datadog::CI::Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
+
+        expect(runner.stop_coverage(test_span)).to be_nil
+        expect(writer).not_to have_received(:write)
+      end
+    end
+
+    context "when test is skipped by ITR" do
+      it "does not write coverage event" do
+        runner.start_coverage(test_span)
+        expect(1 + 1).to eq(2)
+        test_span.skipped!
+
+        expect(runner.stop_coverage(test_span)).to be_nil
+        expect(writer).not_to have_received(:write)
+      end
+    end
+
+    context "when coverage was not collected" do
+      it "does not write coverage event" do
+        expect(1 + 1).to eq(2)
+
+        expect(runner.stop_coverage(test_span)).to be_nil
+        expect(writer).not_to have_received(:write)
+      end
+    end
   end
 
   describe "#mark_if_skippable" do
@@ -213,7 +254,7 @@
           fetch_skippable_tests: instance_double(
             Datadog::CI::ITR::Skippable::Response,
             correlation_id: "42",
-            tests: Set.new(["suite.test"])
+            tests: Set.new(["suite.test", "suite2.test", "suite.test3"])
           )
         )
       end
@@ -249,7 +290,7 @@
       context "when test is not skippable" do
         let(:test_span) do
           Datadog::CI::Test.new(
-            Datadog::Tracing::SpanOperation.new("test", tags: {"test.name" => "test", "test.suite" => "test"})
+            Datadog::Tracing::SpanOperation.new("test", tags: {"test.name" => "test2", "test.suite" => "suite"})
           )
         end