diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 7c6416d3..9ab584e2 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -114,6 +114,7 @@ def activate_ci!(settings) library_settings_client: build_library_settings_client(settings, test_visibility_api) ) @test_retries = TestRetries::Component.new( + retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled, retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts, retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit ) diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index 0479d182..b47c759f 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -88,6 +88,12 @@ def self.add_settings!(base) o.default true end + option :retry_failed_tests_enabled do |o| + o.type :bool + o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_ENABLED + o.default true + end + option :retry_failed_tests_max_attempts do |o| o.type :int o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS @@ -97,7 +103,7 @@ def self.add_settings!(base) option :retry_failed_tests_total_limit do |o| o.type :int o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT - o.default 100 + o.default 1000 end define_method(:instrument) do |integration_name, options = {}, &block| diff --git a/lib/datadog/ci/ext/settings.rb b/lib/datadog/ci/ext/settings.rb index 14cb9a7c..f422955f 100644 --- a/lib/datadog/ci/ext/settings.rb +++ b/lib/datadog/ci/ext/settings.rb @@ -15,8 +15,9 @@ module Settings ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH" ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE" ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING = "DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING" - ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_MAX_ATTEMPTS" - ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_TOTAL_LIMIT" + ENV_RETRY_FAILED_TESTS_ENABLED = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED" + ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT" + ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT" # Source: https://docs.datadoghq.com/getting_started/site/ DD_SITE_ALLOWLIST = %w[ diff --git a/lib/datadog/ci/ext/telemetry.rb b/lib/datadog/ci/ext/telemetry.rb index 22d15ac9..ed6c338c 100644 --- a/lib/datadog/ci/ext/telemetry.rb +++ b/lib/datadog/ci/ext/telemetry.rb @@ -62,6 +62,7 @@ module Telemetry TAG_IS_UNSUPPORTED_CI = "is_unsupported_ci" TAG_BROWSER_DRIVER = "browser_driver" TAG_IS_RUM = "is_rum" + TAG_IS_RETRY = "is_retry" TAG_LIBRARY = "library" TAG_ENDPOINT = "endpoint" TAG_ERROR_TYPE = "error_type" diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index f4f85bf2..b80b6b0b 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -14,11 +14,11 @@ class Component :retry_failed_tests_total_limit, :retry_failed_tests_count def initialize( + retry_failed_tests_enabled:, retry_failed_tests_max_attempts:, retry_failed_tests_total_limit: ) - # enabled only by remote settings - @retry_failed_tests_enabled = false + @retry_failed_tests_enabled = retry_failed_tests_enabled @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts @retry_failed_tests_total_limit = retry_failed_tests_total_limit # counter that store the current number of failed tests retried @@ -28,7 +28,7 @@ def initialize( end def configure(library_settings) - @retry_failed_tests_enabled = library_settings.flaky_test_retries_enabled? + @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled? end def with_retries(&block) diff --git a/lib/datadog/ci/test_visibility/telemetry.rb b/lib/datadog/ci/test_visibility/telemetry.rb index 8c4ff2cd..86d48e5e 100644 --- a/lib/datadog/ci/test_visibility/telemetry.rb +++ b/lib/datadog/ci/test_visibility/telemetry.rb @@ -55,6 +55,9 @@ def self.event_tags_from_span(span) # codeowner tag tags[Ext::Telemetry::TAG_HAS_CODEOWNER] = "true" if span.get_tag(Ext::Test::TAG_CODEOWNERS) + # set is_retry tag if span represents a retried test + tags[Ext::Telemetry::TAG_IS_RETRY] = "true" if span.get_tag(Ext::Test::TAG_IS_RETRY) + tags end diff --git a/sig/datadog/ci/ext/settings.rbs b/sig/datadog/ci/ext/settings.rbs index 92a17f02..d60d0200 100644 --- a/sig/datadog/ci/ext/settings.rbs +++ b/sig/datadog/ci/ext/settings.rbs @@ -12,6 +12,7 @@ module Datadog ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH: String ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE: String ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING: String + ENV_RETRY_FAILED_TESTS_ENABLED: String ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS: String ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT: String diff --git a/sig/datadog/ci/ext/telemetry.rbs b/sig/datadog/ci/ext/telemetry.rbs index 9bc3769f..34c4141b 100644 --- a/sig/datadog/ci/ext/telemetry.rbs +++ b/sig/datadog/ci/ext/telemetry.rbs @@ -94,6 +94,8 @@ module Datadog TAG_IS_RUM: "is_rum" + TAG_IS_RETRY: "is_retry" + TAG_LIBRARY: "library" TAG_ENDPOINT: "endpoint" diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 7a76adc6..1144687b 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -12,7 +12,7 @@ module Datadog @mutex: Thread::Mutex - def initialize: (retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void + def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void diff --git a/spec/datadog/ci/configuration/settings_spec.rb b/spec/datadog/ci/configuration/settings_spec.rb index 508985ed..e5b7be6b 100644 --- a/spec/datadog/ci/configuration/settings_spec.rb +++ b/spec/datadog/ci/configuration/settings_spec.rb @@ -390,6 +390,117 @@ def patcher end end + describe "#retry_failed_tests_enabled" do + subject(:retry_failed_tests_enabled) { settings.ci.retry_failed_tests_enabled } + + it { is_expected.to be true } + + context "when #{Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_ENABLED}" do + around do |example| + ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_ENABLED => enable) do + example.run + end + end + + context "is not defined" do + let(:enable) { nil } + + it { is_expected.to be true } + end + + context "is set to true" do + let(:enable) { "true" } + + it { is_expected.to be true } + end + + context "is set to false" do + let(:enable) { "false" } + + it { is_expected.to be false } + end + end + end + + describe "#retry_failed_tests_enabled=" do + it "updates the #retry_failed_tests_enabled setting" do + expect { settings.ci.retry_failed_tests_enabled = false } + .to change { settings.ci.retry_failed_tests_enabled } + .from(true) + .to(false) + end + end + + describe "#retry_failed_tests_max_attempts" do + subject(:retry_failed_tests_max_attempts) { settings.ci.retry_failed_tests_max_attempts } + + it { is_expected.to eq 5 } + + context "when #{Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS}" do + around do |example| + ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS => attempts) do + example.run + end + end + + context "is not defined" do + let(:attempts) { nil } + + it { is_expected.to eq 5 } + end + + context "is set to value" do + let(:attempts) { "10" } + + it { is_expected.to eq attempts.to_i } + end + end + end + + describe "#retry_failed_tests_max_attempts=" do + it "updates the #retry_failed_tests_max_attempts setting" do + expect { settings.ci.retry_failed_tests_max_attempts = 7 } + .to change { settings.ci.retry_failed_tests_max_attempts } + .from(5) + .to(7) + end + end + + describe "#retry_failed_tests_total_limit" do + subject(:retry_failed_tests_total_limit) { settings.ci.retry_failed_tests_total_limit } + + it { is_expected.to eq 1000 } + + context "when #{Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT}" do + around do |example| + ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT => attempts) do + example.run + end + end + + context "is not defined" do + let(:attempts) { nil } + + it { is_expected.to eq 1000 } + end + + context "is set to value" do + let(:attempts) { "10" } + + it { is_expected.to eq attempts.to_i } + end + end + end + + describe "#retry_failed_tests_total_limit=" do + it "updates the #retry_failed_tests_total_limit setting" do + expect { settings.ci.retry_failed_tests_total_limit = 42 } + .to change { settings.ci.retry_failed_tests_total_limit } + .from(1000) + .to(42) + end + end + describe "#instrument" do let(:integration_name) { :fake } diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index 5ab2fa86..27e6407d 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -2,11 +2,14 @@ RSpec.describe Datadog::CI::TestRetries::Component do let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings) } + + let(:retry_failed_tests_enabled) { true } let(:retry_failed_tests_max_attempts) { 1 } let(:retry_failed_tests_total_limit) { 12 } subject(:component) do described_class.new( + retry_failed_tests_enabled: retry_failed_tests_enabled, retry_failed_tests_max_attempts: retry_failed_tests_max_attempts, retry_failed_tests_total_limit: retry_failed_tests_total_limit ) @@ -38,6 +41,20 @@ expect(component.retry_failed_tests_enabled).to be false end end + + context "when flaky test retries are disabled in local settings" do + let(:retry_failed_tests_enabled) { false } + + before do + allow(library_settings).to receive(:flaky_test_retries_enabled?).and_return(true) + end + + it "disables retrying failed tests even if it's enabled remotely" do + subject + + expect(component.retry_failed_tests_enabled).to be false + end + end end describe "#retry_failed_tests_max_attempts" do diff --git a/spec/datadog/ci/test_visibility/telemetry_spec.rb b/spec/datadog/ci/test_visibility/telemetry_spec.rb index a4d9d968..7bcff7f7 100644 --- a/spec/datadog/ci/test_visibility/telemetry_spec.rb +++ b/spec/datadog/ci/test_visibility/telemetry_spec.rb @@ -205,6 +205,36 @@ it { event_finished } end + + context "test span with retry" do + let(:span) do + Datadog::Tracing::SpanOperation.new( + "test_session", + type: Datadog::CI::Ext::AppTypes::TYPE_TEST, + tags: { + Datadog::CI::Ext::Test::TAG_FRAMEWORK => "rspec", + Datadog::CI::Ext::Environment::TAG_PROVIDER_NAME => "gha", + Datadog::CI::Ext::Test::TAG_CODEOWNERS => "@owner", + Datadog::CI::Ext::Test::TAG_IS_RUM_ACTIVE => "true", + Datadog::CI::Ext::Test::TAG_BROWSER_DRIVER => "selenium", + Datadog::CI::Ext::Test::TAG_IS_RETRY => "true" + } + ) + end + + let(:expected_tags) do + { + Datadog::CI::Ext::Telemetry::TAG_EVENT_TYPE => Datadog::CI::Ext::Telemetry::EventType::TEST, + Datadog::CI::Ext::Telemetry::TAG_TEST_FRAMEWORK => "rspec", + Datadog::CI::Ext::Telemetry::TAG_HAS_CODEOWNER => "true", + Datadog::CI::Ext::Telemetry::TAG_IS_RUM => "true", + Datadog::CI::Ext::Telemetry::TAG_BROWSER_DRIVER => "selenium", + Datadog::CI::Ext::Telemetry::TAG_IS_RETRY => "true" + } + end + + it { event_finished } + end end describe ".test_session_started" do