diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index 0d219b96..ecb184a5 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -26,6 +26,7 @@ module Test # ITR tags TAG_ITR_TEST_SKIPPING_ENABLED = "test.itr.tests_skipping.enabled" TAG_ITR_TEST_SKIPPING_TYPE = "test.itr.tests_skipping.type" + TAG_ITR_SKIPPED_BY_ITR = "test.itr.skipped_by_itr" # Code coverage tags TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled" diff --git a/lib/datadog/ci/itr/runner.rb b/lib/datadog/ci/itr/runner.rb index 44d72877..7e8ac877 100644 --- a/lib/datadog/ci/itr/runner.rb +++ b/lib/datadog/ci/itr/runner.rb @@ -2,6 +2,8 @@ require "pp" +require "datadog/core/utils/forking" + require_relative "../ext/test" require_relative "../ext/transport" @@ -19,7 +21,9 @@ module ITR # Integrates with backend to provide test impact analysis data and # skip tests that are not impacted by the changes class Runner - attr_reader :correlation_id, :skippable_tests + include Core::Utils::Forking + + attr_reader :correlation_id, :skippable_tests, :skipped_tests_count def initialize( dd_env:, @@ -39,6 +43,9 @@ def initialize( @correlation_id = nil @skippable_tests = [] + @skipped_tests_count = 0 + @mutex = Mutex.new + Datadog.logger.debug("ITR Runner initialized with enabled: #{@enabled}") end @@ -113,6 +120,32 @@ def stop_coverage(test) event end + def mark_if_skippable(test) + return unless enabled? && skipping_tests? + + test_full_name = Utils::TestRun.test_full_name(test.name, test.test_suite_name) + + if @skippable_tests.include?(test_full_name) + test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true") + increment_skipped_tests_counter + + Datadog.logger.debug { "Marked test as skippable: #{test_full_name}" } + else + Datadog.logger.debug { "Test is not skippable: #{test_full_name}" } + end + end + + def increment_skipped_tests_counter + if forked? + Datadog.logger.warn { "ITR is not supported for forking test runners yet" } + return + end + + @mutex.synchronize do + @skipped_tests_count += 1 + end + end + def shutdown! @coverage_writer&.stop end diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb index a72acddf..191528ce 100644 --- a/lib/datadog/ci/test.rb +++ b/lib/datadog/ci/test.rb @@ -62,6 +62,12 @@ def source_file get_tag(Ext::Test::TAG_SOURCE_FILE) end + # Returns "true" if the test is skipped by the ITR. + # @return [Boolean] true if the test is skipped by the ITR, false otherwise. + def skipped_by_itr? + get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true" + end + # Sets the status of the span to "pass". # @return [void] def passed! diff --git a/lib/datadog/ci/test_visibility/recorder.rb b/lib/datadog/ci/test_visibility/recorder.rb index d3a07414..52b759f9 100644 --- a/lib/datadog/ci/test_visibility/recorder.rb +++ b/lib/datadog/ci/test_visibility/recorder.rb @@ -127,21 +127,18 @@ def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block) test = build_test(tracer_span, tags) @local_context.activate_test(test) do - @itr.start_coverage - + on_test_started(test) res = block.call(test) on_test_finished(test) - res end end else tracer_span = start_datadog_tracer_span(test_name, span_options) - test = build_test(tracer_span, tags) @local_context.activate_test(test) - @itr.start_coverage + on_test_started(test) test end @@ -402,6 +399,11 @@ def validate_test_suite_level_visibility_correctness(test) def on_test_finished(test) @itr.stop_coverage(test) end + + def on_test_started(test) + @itr.mark_if_skippable(test) + @itr.start_coverage + end end end end diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index dc308af1..c5b954a3 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -36,6 +36,8 @@ module Datadog TAG_ITR_TEST_SKIPPING_TYPE: "test.itr.tests_skipping.type" + TAG_ITR_SKIPPED_BY_ITR: "test.itr.skipped_by_itr" + TAG_CODE_COVERAGE_ENABLED: "test.code_coverage.enabled" TAG_TEST_SESSION_ID: "_test.session_id" diff --git a/sig/datadog/ci/itr/runner.rbs b/sig/datadog/ci/itr/runner.rbs index 180d7aca..4da4dd1e 100644 --- a/sig/datadog/ci/itr/runner.rbs +++ b/sig/datadog/ci/itr/runner.rbs @@ -2,16 +2,21 @@ module Datadog module CI module ITR class Runner + include Datadog::Core::Utils::Forking + @enabled: bool @test_skipping_enabled: bool @code_coverage_enabled: bool @correlation_id: String - @skippable_tests: Array[Datadog::CI::ITR::Skippable::Test] + @skippable_tests: Array[String] @coverage_writer: Datadog::CI::ITR::Coverage::Writer? @api: Datadog::CI::Transport::Api::Base? @dd_env: String? + @skipped_tests_count: Integer + @mutex: Thread::Mutex + def initialize: (dd_env: String?, ?enabled: bool, coverage_writer: Datadog::CI::ITR::Coverage::Writer?, api: Datadog::CI::Transport::Api::Base?) -> void def configure: (Hash[String, untyped] remote_configuration, test_session: Datadog::CI::TestSession, git_tree_upload_worker: Datadog::CI::Worker) -> void @@ -26,6 +31,8 @@ module Datadog def stop_coverage: (Datadog::CI::Test test) -> Datadog::CI::ITR::Coverage::Event? + def mark_if_skippable: (Datadog::CI::Test test) -> void + def shutdown!: () -> void private @@ -39,6 +46,8 @@ module Datadog def ensure_test_source_covered: (String test_source_file, Hash[String, untyped] coverage) -> void def fetch_skippable_tests: (test_session: Datadog::CI::TestSession, git_tree_upload_worker: Datadog::CI::Worker) -> void + + def increment_skipped_tests_counter: () -> void end end end diff --git a/sig/datadog/ci/test.rbs b/sig/datadog/ci/test.rbs index ab77f7d0..c79be5ff 100644 --- a/sig/datadog/ci/test.rbs +++ b/sig/datadog/ci/test.rbs @@ -7,6 +7,7 @@ module Datadog def test_suite_name: () -> String? def test_module_id: () -> String? def test_session_id: () -> String? + def skipped_by_itr?: () -> bool def source_file: () -> String? private diff --git a/sig/datadog/ci/test_visibility/recorder.rbs b/sig/datadog/ci/test_visibility/recorder.rbs index 0363922b..a753203c 100644 --- a/sig/datadog/ci/test_visibility/recorder.rbs +++ b/sig/datadog/ci/test_visibility/recorder.rbs @@ -93,6 +93,8 @@ module Datadog def validate_test_suite_level_visibility_correctness: (Datadog::CI::Test test) -> void def on_test_finished: (Datadog::CI::Test test) -> void + + def on_test_started: (Datadog::CI::Test test) -> void end end end diff --git a/sig/datadog/ci/utils/test_run.rbs b/sig/datadog/ci/utils/test_run.rbs index 66fef457..887b5292 100644 --- a/sig/datadog/ci/utils/test_run.rbs +++ b/sig/datadog/ci/utils/test_run.rbs @@ -6,7 +6,7 @@ module Datadog def self.command: () -> String - def self.test_full_name: (String test_name, String test_suite) -> String + def self.test_full_name: (String test_name, String? test_suite) -> String end end end diff --git a/spec/datadog/ci/itr/runner_spec.rb b/spec/datadog/ci/itr/runner_spec.rb index 6fd53fd5..f9e6f028 100644 --- a/spec/datadog/ci/itr/runner_spec.rb +++ b/spec/datadog/ci/itr/runner_spec.rb @@ -201,4 +201,92 @@ end end end + + describe "#mark_if_skippable" do + subject { runner.mark_if_skippable(test_span) } + + context "when skipping tests" do + let(:remote_configuration) { {"itr_enabled" => true, "code_coverage" => true, "tests_skipping" => true} } + let(:skippable) do + instance_double( + Datadog::CI::ITR::Skippable, + fetch_skippable_tests: instance_double( + Datadog::CI::ITR::Skippable::Response, + correlation_id: "42", + tests: Set.new(["suite.test"]) + ) + ) + end + + before do + expect(Datadog::CI::ITR::Skippable).to receive(:new).and_return(skippable) + + configure + end + + context "when test is skippable" do + let(:test_span) do + Datadog::CI::Test.new( + Datadog::Tracing::SpanOperation.new("test", tags: {"test.name" => "test", "test.suite" => "suite"}) + ) + end + + it "marks test as skippable" do + expect { subject } + .to change { test_span.skipped_by_itr? } + .from(false) + .to(true) + end + + it "increments skipped tests count" do + expect { subject } + .to change { runner.skipped_tests_count } + .from(0) + .to(1) + end + end + + context "when test is not skippable" do + let(:test_span) do + Datadog::CI::Test.new( + Datadog::Tracing::SpanOperation.new("test", tags: {"test.name" => "test", "test.suite" => "test"}) + ) + end + + it "does not mark test as skippable" do + expect { subject } + .not_to change { test_span.skipped_by_itr? } + end + + it "does not increment skipped tests count" do + expect { subject } + .not_to change { runner.skipped_tests_count } + end + end + end + + context "when not skipping tests" do + let(:remote_configuration) { {"itr_enabled" => true, "code_coverage" => true, "tests_skipping" => false} } + + before do + configure + end + + let(:test_span) do + Datadog::CI::Test.new( + Datadog::Tracing::SpanOperation.new("test", tags: {"test.name" => "test", "test.suite" => "suite"}) + ) + end + + it "does not mark test as skippable" do + expect { subject } + .not_to change { test_span.skipped_by_itr? } + end + + it "does not increment skipped tests count" do + expect { subject } + .not_to change { runner.skipped_tests_count } + end + end + end end diff --git a/spec/datadog/ci/test_spec.rb b/spec/datadog/ci/test_spec.rb index f474145f..5f8053ef 100644 --- a/spec/datadog/ci/test_spec.rb +++ b/spec/datadog/ci/test_spec.rb @@ -99,6 +99,26 @@ it { is_expected.to eq("foo/bar.rb") } end + describe "#skipped_by_itr?" do + subject(:skipped_by_itr) { ci_test.skipped_by_itr? } + + context "when tag is set" do + before do + allow(tracer_span).to( + receive(:get_tag).with(Datadog::CI::Ext::Test::TAG_ITR_SKIPPED_BY_ITR).and_return("true") + ) + end + + it { is_expected.to be true } + end + + context "when tag is not set" do + before { allow(tracer_span).to receive(:get_tag).with(Datadog::CI::Ext::Test::TAG_ITR_SKIPPED_BY_ITR).and_return(nil) } + + it { is_expected.to be false } + end + end + describe "#set_parameters" do let(:parameters) { {"foo" => "bar", "baz" => "qux"} }