From f0d244582dceb50475af8b3edce048e7cffc86b8 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 29 Aug 2024 14:47:18 +0200 Subject: [PATCH 01/16] parse early_flake_detection_enabled? from remote library settings --- lib/datadog/ci/ext/telemetry.rb | 1 + lib/datadog/ci/ext/transport.rb | 1 + lib/datadog/ci/remote/library_settings.rb | 35 ++++++++++--- .../ci/remote/library_settings_client.rb | 3 +- lib/datadog/ci/test_retries/component.rb | 6 ++- sig/datadog/ci/ext/telemetry.rbs | 2 + sig/datadog/ci/ext/transport.rbs | 2 + sig/datadog/ci/remote/library_settings.rbs | 5 +- sig/datadog/ci/test_retries/component.rbs | 2 + .../ci/remote/library_settings_client_spec.rb | 41 +++++++++------ .../datadog/ci/test_retries/component_spec.rb | 52 +++++++++++++------ spec/support/contexts/ci_mode.rb | 7 ++- 12 files changed, 111 insertions(+), 46 deletions(-) diff --git a/lib/datadog/ci/ext/telemetry.rb b/lib/datadog/ci/ext/telemetry.rb index ed6c338c..959980e7 100644 --- a/lib/datadog/ci/ext/telemetry.rb +++ b/lib/datadog/ci/ext/telemetry.rb @@ -73,6 +73,7 @@ module Telemetry TAG_COMMAND = "command" TAG_COVERAGE_ENABLED = "coverage_enabled" TAG_ITR_SKIP_ENABLED = "itrskip_enabled" + TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled" TAG_PROVIDER = "provider" TAG_AUTO_INJECTED = "auto_injected" diff --git a/lib/datadog/ci/ext/transport.rb b/lib/datadog/ci/ext/transport.rb index 57fa7e0a..25adb1eb 100644 --- a/lib/datadog/ci/ext/transport.rb +++ b/lib/datadog/ci/ext/transport.rb @@ -37,6 +37,7 @@ module Transport DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY = "tests_skipping" DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git" DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled" + DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY = "early_flake_detection" DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits" diff --git a/lib/datadog/ci/remote/library_settings.rb b/lib/datadog/ci/remote/library_settings.rb index 63984e20..a1a85386 100644 --- a/lib/datadog/ci/remote/library_settings.rb +++ b/lib/datadog/ci/remote/library_settings.rb @@ -49,39 +49,58 @@ def payload def require_git? return @require_git if defined?(@require_git) - @require_git = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY) + @require_git = Utils::Parsing.convert_to_bool( + payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY, false) + ) end def itr_enabled? return @itr_enabled if defined?(@itr_enabled) - @itr_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY) + @itr_enabled = Utils::Parsing.convert_to_bool( + payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false) + ) end def code_coverage_enabled? return @code_coverage_enabled if defined?(@code_coverage_enabled) - @code_coverage_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY) + @code_coverage_enabled = Utils::Parsing.convert_to_bool( + payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY, false) + ) end def tests_skipping_enabled? return @tests_skipping_enabled if defined?(@tests_skipping_enabled) - @tests_skipping_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY) + @tests_skipping_enabled = Utils::Parsing.convert_to_bool( + payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY, false) + ) end def flaky_test_retries_enabled? return @flaky_test_retries_enabled if defined?(@flaky_test_retries_enabled) - @flaky_test_retries_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY) + @flaky_test_retries_enabled = Utils::Parsing.convert_to_bool( + payload.fetch( + Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY, false + ) + ) end - private + def early_flake_detection_enabled? + return @early_flake_detection_enabled if defined?(@early_flake_detection_enabled) - def bool(key) - Utils::Parsing.convert_to_bool(payload.fetch(key, false)) + @early_flake_detection_enabled = Utils::Parsing.convert_to_bool( + payload.fetch( + Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY, + {} + ).fetch("enabled", false) + ) end + private + def default_payload Ext::Transport::DD_API_SETTINGS_RESPONSE_DEFAULT end diff --git a/lib/datadog/ci/remote/library_settings_client.rb b/lib/datadog/ci/remote/library_settings_client.rb index 43c70131..9b50c0e6 100644 --- a/lib/datadog/ci/remote/library_settings_client.rb +++ b/lib/datadog/ci/remote/library_settings_client.rb @@ -58,7 +58,8 @@ def fetch(test_session) 1, { Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s, - Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s + Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s, + Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s } ) diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index b80b6b0b..3efd9521 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -11,7 +11,8 @@ module TestRetries # - retrying new tests - detect flaky tests as early as possible to prevent them from being merged class Component attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, - :retry_failed_tests_total_limit, :retry_failed_tests_count + :retry_failed_tests_total_limit, :retry_failed_tests_count, + :retry_new_tests_enabled def initialize( retry_failed_tests_enabled:, @@ -24,11 +25,14 @@ def initialize( # counter that store the current number of failed tests retried @retry_failed_tests_count = 0 + @retry_new_tests_enabled = true + @mutex = Mutex.new end def configure(library_settings) @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled? + @retry_new_tests_enabled &&= library_settings.early_flake_detection_enabled? end def with_retries(&block) diff --git a/sig/datadog/ci/ext/telemetry.rbs b/sig/datadog/ci/ext/telemetry.rbs index 34c4141b..33e89226 100644 --- a/sig/datadog/ci/ext/telemetry.rbs +++ b/sig/datadog/ci/ext/telemetry.rbs @@ -116,6 +116,8 @@ module Datadog TAG_ITR_SKIP_ENABLED: "itrskip_enabled" + TAG_EARLY_FLAKE_DETECTION_ENABLED: "early_flake_detection_enabled" + TAG_PROVIDER: "provider" TAG_AUTO_INJECTED: "auto_injected" diff --git a/sig/datadog/ci/ext/transport.rbs b/sig/datadog/ci/ext/transport.rbs index e701817e..88625092 100644 --- a/sig/datadog/ci/ext/transport.rbs +++ b/sig/datadog/ci/ext/transport.rbs @@ -50,6 +50,8 @@ module Datadog DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY: "flaky_test_retries_enabled" + DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY: "early_flake_detection" + DD_API_SETTINGS_RESPONSE_DEFAULT: Hash[String, untyped] DD_API_GIT_SEARCH_COMMITS_PATH: "/api/v2/git/repository/search_commits" diff --git a/sig/datadog/ci/remote/library_settings.rbs b/sig/datadog/ci/remote/library_settings.rbs index 68791562..e4b2b555 100644 --- a/sig/datadog/ci/remote/library_settings.rbs +++ b/sig/datadog/ci/remote/library_settings.rbs @@ -10,6 +10,7 @@ module Datadog @code_coverage_enabled: bool @tests_skipping_enabled: bool @flaky_test_retries_enabled: bool + @early_flake_detection_enabled: bool def initialize: (Datadog::CI::Transport::Adapters::Net::Response? http_response) -> void @@ -27,9 +28,9 @@ module Datadog def flaky_test_retries_enabled?: () -> bool - private + def early_flake_detection_enabled?: () -> bool - def bool: (String key) -> bool + private def default_payload: () -> Hash[String, untyped] end diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 1144687b..604f89ac 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -10,6 +10,8 @@ module Datadog attr_reader retry_failed_tests_count: Integer + attr_reader retry_new_tests_enabled: bool + @mutex: Thread::Mutex def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void diff --git a/spec/datadog/ci/remote/library_settings_client_spec.rb b/spec/datadog/ci/remote/library_settings_client_spec.rb index 31c01a58..1fc4ba6c 100644 --- a/spec/datadog/ci/remote/library_settings_client_spec.rb +++ b/spec/datadog/ci/remote/library_settings_client_spec.rb @@ -70,6 +70,26 @@ end context "when response is OK" do + let(:attributes) do + { + "code_coverage" => "1", + "tests_skipping" => "false", + "itr_enabled" => "True", + "require_git" => require_git, + "flaky_test_retries_enabled" => "true", + "early_flake_detection" => { + "enabled" => "true", + "slow_test_retries" => { + "5s" => 10, + "10s" => 5, + "30s" => 3, + "5m" => 2 + }, + "faulty_session_threshold" => 30 + } + } + end + let(:http_response) do double( "http_response", @@ -78,13 +98,7 @@ "data" => { "id" => "123", "type" => Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE, - "attributes" => { - "code_coverage" => "1", - "tests_skipping" => "false", - "itr_enabled" => "True", - "require_git" => require_git, - "flaky_test_retries_enabled" => "true" - } + "attributes" => attributes } }.to_json, request_compressed: false, @@ -95,21 +109,18 @@ it "parses the response" do expect(response.ok?).to be true - expect(response.payload).to eq({ - "code_coverage" => "1", - "tests_skipping" => "false", - "itr_enabled" => "True", - "require_git" => require_git, - "flaky_test_retries_enabled" => "true" - }) + expect(response.payload).to eq(attributes) expect(response.require_git?).to be false expect(response.itr_enabled?).to be true expect(response.code_coverage_enabled?).to be true expect(response.tests_skipping_enabled?).to be false expect(response.flaky_test_retries_enabled?).to be true + expect(response.early_flake_detection_enabled?).to be true metric = telemetry_metric(:inc, "git_requests.settings_response") - expect(metric.tags).to eq("coverage_enabled" => "true", "itrskip_enabled" => "false") + expect(metric.tags).to eq( + "coverage_enabled" => "true", "itrskip_enabled" => "false", "early_flake_detection_enabled" => "true" + ) end it_behaves_like "emits telemetry metric", :inc, "git_requests.settings", 1 diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index 27e6407d..a5a5da14 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -1,12 +1,21 @@ require_relative "../../../../lib/datadog/ci/test_retries/component" RSpec.describe Datadog::CI::TestRetries::Component do - let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings) } + let(:library_settings) do + instance_double( + Datadog::CI::Remote::LibrarySettings, + flaky_test_retries_enabled?: remote_flaky_test_retries_enabled, + early_flake_detection_enabled?: remote_early_flake_detection_enabled + ) + end let(:retry_failed_tests_enabled) { true } let(:retry_failed_tests_max_attempts) { 1 } let(:retry_failed_tests_total_limit) { 12 } + let(:remote_flaky_test_retries_enabled) { false } + let(:remote_early_flake_detection_enabled) { false } + subject(:component) do described_class.new( retry_failed_tests_enabled: retry_failed_tests_enabled, @@ -19,9 +28,7 @@ subject { component.configure(library_settings) } context "when flaky test retries are enabled" do - before do - allow(library_settings).to receive(:flaky_test_retries_enabled?).and_return(true) - end + let(:remote_flaky_test_retries_enabled) { true } it "enables retrying failed tests" do subject @@ -31,9 +38,7 @@ end context "when flaky test retries are disabled" do - before do - allow(library_settings).to receive(:flaky_test_retries_enabled?).and_return(false) - end + let(:remote_flaky_test_retries_enabled) { false } it "disables retrying failed tests" do subject @@ -44,10 +49,7 @@ context "when flaky test retries are disabled in local settings" do let(:retry_failed_tests_enabled) { false } - - before do - allow(library_settings).to receive(:flaky_test_retries_enabled?).and_return(true) - end + let(:remote_flaky_test_retries_enabled) { true } it "disables retrying failed tests even if it's enabled remotely" do subject @@ -55,6 +57,26 @@ expect(component.retry_failed_tests_enabled).to be false end end + + context "when early flake detecion is enabled" do + let(:remote_early_flake_detection_enabled) { true } + + it "enables retrying failed tests" do + subject + + expect(component.retry_new_tests_enabled).to be true + end + end + + context "when early flake detecion is disabled" do + let(:remote_early_flake_detection_enabled) { false } + + it "disables retrying failed tests" do + subject + + expect(component.retry_new_tests_enabled).to be false + end + end end describe "#retry_failed_tests_max_attempts" do @@ -80,7 +102,7 @@ end context "when retry failed tests is enabled" do - let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: true) } + let(:remote_flaky_test_retries_enabled) { true } context "when test span is failed" do let(:test_failed) { true } @@ -130,8 +152,6 @@ end context "when retry failed tests is disabled" do - let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: false) } - it { is_expected.to be_a(Datadog::CI::TestRetries::Strategy::NoRetry) } end end @@ -171,13 +191,11 @@ end context "when no retries strategy is used" do - let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: false) } - it { is_expected.to eq(1) } end context "when retried failed tests strategy is used" do - let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: true) } + let(:remote_flaky_test_retries_enabled) { true } context "when test span is failed" do let(:test_failed) { true } diff --git a/spec/support/contexts/ci_mode.rb b/spec/support/contexts/ci_mode.rb index 2354c53c..33ec12e3 100644 --- a/spec/support/contexts/ci_mode.rb +++ b/spec/support/contexts/ci_mode.rb @@ -27,6 +27,7 @@ let(:bundle_path) { nil } let(:use_single_threaded_coverage) { false } let(:flaky_test_retries_enabled) { false } + let(:early_flake_detection_enabled) { false } let(:retry_failed_tests_max_attempts) { 5 } let(:retry_failed_tests_total_limit) { 100 } @@ -66,7 +67,8 @@ itr_enabled?: itr_enabled, code_coverage_enabled?: code_coverage_enabled, tests_skipping_enabled?: tests_skipping_enabled, - flaky_test_retries_enabled?: flaky_test_retries_enabled + flaky_test_retries_enabled?: flaky_test_retries_enabled, + early_flake_detection_enabled?: early_flake_detection_enabled ), # This is for the second call to fetch_library_settings instance_double( @@ -80,7 +82,8 @@ itr_enabled?: itr_enabled, code_coverage_enabled?: !code_coverage_enabled, tests_skipping_enabled?: !tests_skipping_enabled, - flaky_test_retries_enabled?: flaky_test_retries_enabled + flaky_test_retries_enabled?: flaky_test_retries_enabled, + early_flake_detection_enabled?: early_flake_detection_enabled ) ) allow_any_instance_of(Datadog::CI::TestOptimisation::Skippable).to receive(:fetch_skippable_tests).and_return(skippable_tests_response) From 57a0d01394f3a7db3dca9b847800ee3967f2b3d9 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 29 Aug 2024 15:13:28 +0200 Subject: [PATCH 02/16] add DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED killswitch to settings --- lib/datadog/ci/configuration/settings.rb | 6 +++ lib/datadog/ci/ext/settings.rb | 1 + sig/datadog/ci/ext/settings.rbs | 1 + .../datadog/ci/configuration/settings_spec.rb | 41 +++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index b47c759f..3d7f8a80 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -106,6 +106,12 @@ def self.add_settings!(base) o.default 1000 end + option :retry_new_tests_enabled do |o| + o.type :bool + o.env CI::Ext::Settings::ENV_RETRY_NEW_TESTS_ENABLED + o.default true + end + define_method(:instrument) do |integration_name, options = {}, &block| return unless enabled diff --git a/lib/datadog/ci/ext/settings.rb b/lib/datadog/ci/ext/settings.rb index f422955f..a318ee26 100644 --- a/lib/datadog/ci/ext/settings.rb +++ b/lib/datadog/ci/ext/settings.rb @@ -18,6 +18,7 @@ module Settings ENV_RETRY_FAILED_TESTS_ENABLED = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED" ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT" ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT" + ENV_RETRY_NEW_TESTS_ENABLED = "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED" # Source: https://docs.datadoghq.com/getting_started/site/ DD_SITE_ALLOWLIST = %w[ diff --git a/sig/datadog/ci/ext/settings.rbs b/sig/datadog/ci/ext/settings.rbs index d60d0200..d96adf24 100644 --- a/sig/datadog/ci/ext/settings.rbs +++ b/sig/datadog/ci/ext/settings.rbs @@ -15,6 +15,7 @@ module Datadog ENV_RETRY_FAILED_TESTS_ENABLED: String ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS: String ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT: String + ENV_RETRY_NEW_TESTS_ENABLED: String DD_SITE_ALLOWLIST: Array[String] end diff --git a/spec/datadog/ci/configuration/settings_spec.rb b/spec/datadog/ci/configuration/settings_spec.rb index e5b7be6b..b70eae46 100644 --- a/spec/datadog/ci/configuration/settings_spec.rb +++ b/spec/datadog/ci/configuration/settings_spec.rb @@ -501,6 +501,47 @@ def patcher end end + describe "#retry_new_tests_enabled" do + subject(:retry_new_tests_enabled) { settings.ci.retry_new_tests_enabled } + + it { is_expected.to be true } + + context "when #{Datadog::CI::Ext::Settings::ENV_RETRY_NEW_TESTS_ENABLED}" do + around do |example| + ClimateControl.modify(Datadog::CI::Ext::Settings::ENV_RETRY_NEW_TESTS_ENABLED => enable) do + example.run + end + end + + context "is not defined" do + let(:enable) { nil } + + it { is_expected.to be true } + end + + context "is set to true" do + let(:enable) { "true" } + + it { is_expected.to be true } + end + + context "is set to false" do + let(:enable) { "false" } + + it { is_expected.to be false } + end + end + end + + describe "#retry_new_tests_enabled=" do + it "updates the #retry_failed_tests_enabled setting" do + expect { settings.ci.retry_new_tests_enabled = false } + .to change { settings.ci.retry_new_tests_enabled } + .from(true) + .to(false) + end + end + describe "#instrument" do let(:integration_name) { :fake } From 1c915430f1f683d6f1dd50051588da2bc5a513c3 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 29 Aug 2024 16:15:10 +0200 Subject: [PATCH 03/16] use DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED killswitch in TestRetries::Component --- lib/datadog/ci/configuration/components.rb | 3 ++- lib/datadog/ci/test_retries/component.rb | 5 +++-- sig/datadog/ci/test_retries/component.rbs | 2 +- .../datadog/ci/test_retries/component_spec.rb | 19 ++++++++++++++++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 55231eb2..d8fcdedd 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -116,7 +116,8 @@ def activate_ci!(settings) @test_retries = TestRetries::Component.new( retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled, retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts, - retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit + retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit, + retry_new_tests_enabled: settings.ci.retry_new_tests_enabled ) # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component @test_optimisation = build_test_optimisation(settings, test_visibility_api) diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 3efd9521..dcb5bf8c 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -17,7 +17,8 @@ class Component def initialize( retry_failed_tests_enabled:, retry_failed_tests_max_attempts:, - retry_failed_tests_total_limit: + retry_failed_tests_total_limit:, + retry_new_tests_enabled: ) @retry_failed_tests_enabled = retry_failed_tests_enabled @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts @@ -25,7 +26,7 @@ def initialize( # counter that store the current number of failed tests retried @retry_failed_tests_count = 0 - @retry_new_tests_enabled = true + @retry_new_tests_enabled = retry_new_tests_enabled @mutex = Mutex.new end diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 604f89ac..3b38dec9 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -14,7 +14,7 @@ module Datadog @mutex: Thread::Mutex - def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void + def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer, retry_new_tests_enabled: bool) -> void def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index a5a5da14..c0ebdeea 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -12,6 +12,7 @@ let(:retry_failed_tests_enabled) { true } let(:retry_failed_tests_max_attempts) { 1 } let(:retry_failed_tests_total_limit) { 12 } + let(:retry_new_tests_enabled) { true } let(:remote_flaky_test_retries_enabled) { false } let(:remote_early_flake_detection_enabled) { false } @@ -20,7 +21,8 @@ described_class.new( retry_failed_tests_enabled: retry_failed_tests_enabled, retry_failed_tests_max_attempts: retry_failed_tests_max_attempts, - retry_failed_tests_total_limit: retry_failed_tests_total_limit + retry_failed_tests_total_limit: retry_failed_tests_total_limit, + retry_new_tests_enabled: retry_new_tests_enabled ) end @@ -58,7 +60,7 @@ end end - context "when early flake detecion is enabled" do + context "when early flake detection is enabled" do let(:remote_early_flake_detection_enabled) { true } it "enables retrying failed tests" do @@ -68,7 +70,7 @@ end end - context "when early flake detecion is disabled" do + context "when early flake detection is disabled" do let(:remote_early_flake_detection_enabled) { false } it "disables retrying failed tests" do @@ -77,6 +79,17 @@ expect(component.retry_new_tests_enabled).to be false end end + + context "when early flake detection is disabled in local settings" do + let(:retry_new_tests_enabled) { false } + let(:remote_early_flake_detection_enabled) { true } + + it "disables retrying failed tests even if it's enabled remotely" do + subject + + expect(component.retry_new_tests_enabled).to be false + end + end end describe "#retry_failed_tests_max_attempts" do From 0798794fe3563e93fe124eff9f3150f6ae69225a Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 10:51:32 +0200 Subject: [PATCH 04/16] parse slow test retries payload from library settings --- lib/datadog/ci/remote/library_settings.rb | 12 +++ lib/datadog/ci/remote/slow_test_retries.rb | 53 ++++++++++++++ lib/datadog/ci/test_retries/component.rb | 2 +- sig/datadog/ci/remote/library_settings.rbs | 3 + sig/datadog/ci/remote/slow_test_retries.rbs | 27 +++++++ .../ci/remote/library_settings_client_spec.rb | 8 ++ .../ci/remote/slow_test_retries_spec.rb | 73 +++++++++++++++++++ 7 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 lib/datadog/ci/remote/slow_test_retries.rb create mode 100644 sig/datadog/ci/remote/slow_test_retries.rbs create mode 100644 spec/datadog/ci/remote/slow_test_retries_spec.rb diff --git a/lib/datadog/ci/remote/library_settings.rb b/lib/datadog/ci/remote/library_settings.rb index a1a85386..472f1f67 100644 --- a/lib/datadog/ci/remote/library_settings.rb +++ b/lib/datadog/ci/remote/library_settings.rb @@ -7,6 +7,8 @@ require_relative "../transport/telemetry" require_relative "../utils/parsing" +require_relative "slow_test_retries" + module Datadog module CI module Remote @@ -99,6 +101,16 @@ def early_flake_detection_enabled? ) end + def slow_test_retries + return @slow_test_retries if defined?(@slow_test_retries) + + @slow_test_retries = SlowTestRetries.new( + payload + .fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY, {}) + .fetch("slow_test_retries", {}) + ) + end + private def default_payload diff --git a/lib/datadog/ci/remote/slow_test_retries.rb b/lib/datadog/ci/remote/slow_test_retries.rb new file mode 100644 index 00000000..8ccfe236 --- /dev/null +++ b/lib/datadog/ci/remote/slow_test_retries.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Remote + # Parses "slow_test_retries" payload for early flake detection settings + # + # Example payload: + # { + # "5s" => 10, + # "10s" => 5, + # "30s" => 3, + # "5m" => 2 + # } + # + # The payload above means that for tests that run less than 5 seconds, we should retry them 10 times, + # for tests that run less than 10 seconds, we should retry them 5 times, and so on. + class SlowTestRetries + attr_reader :entries + + Entry = Struct.new(:duration, :max_attempts) + + DURATION_MEASURES = { + "s" => 1, + "m" => 60 + }.freeze + + def initialize(payload) + @entries = parse(payload) + end + + def max_attempts_for_duration(duration) + @entries.each do |entry| + return entry.max_attempts if duration < entry.duration + end + + 1 + end + + private + + def parse(payload) + (payload || {}).keys.filter_map do |key| + duration, measure = key.match(/(\d+)(\w+)/)&.captures + next if duration.nil? || measure.nil? || !DURATION_MEASURES.key?(measure) + + Entry.new(duration.to_f * DURATION_MEASURES.fetch(measure, 1), payload[key].to_i) + end.sort_by(&:duration) + end + end + end + end +end diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index dcb5bf8c..26c109a7 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -23,7 +23,7 @@ def initialize( @retry_failed_tests_enabled = retry_failed_tests_enabled @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts @retry_failed_tests_total_limit = retry_failed_tests_total_limit - # counter that store the current number of failed tests retried + # counter that stores the current number of failed tests retried @retry_failed_tests_count = 0 @retry_new_tests_enabled = retry_new_tests_enabled diff --git a/sig/datadog/ci/remote/library_settings.rbs b/sig/datadog/ci/remote/library_settings.rbs index e4b2b555..7a5b72c2 100644 --- a/sig/datadog/ci/remote/library_settings.rbs +++ b/sig/datadog/ci/remote/library_settings.rbs @@ -11,6 +11,7 @@ module Datadog @tests_skipping_enabled: bool @flaky_test_retries_enabled: bool @early_flake_detection_enabled: bool + @slow_test_retries: Datadog::CI::Remote::SlowTestRetries def initialize: (Datadog::CI::Transport::Adapters::Net::Response? http_response) -> void @@ -30,6 +31,8 @@ module Datadog def early_flake_detection_enabled?: () -> bool + def slow_test_retries: () -> Datadog::CI::Remote::SlowTestRetries + private def default_payload: () -> Hash[String, untyped] diff --git a/sig/datadog/ci/remote/slow_test_retries.rbs b/sig/datadog/ci/remote/slow_test_retries.rbs new file mode 100644 index 00000000..28228de6 --- /dev/null +++ b/sig/datadog/ci/remote/slow_test_retries.rbs @@ -0,0 +1,27 @@ +module Datadog + module CI + module Remote + class SlowTestRetries + attr_reader entries: Enumerable[Entry] + + class Entry + attr_accessor duration: Float + + attr_accessor max_attempts: Integer + + def initialize: (Float duration, Integer max_attempts) -> void + end + + DURATION_MEASURES: Hash[String, Integer] + + def initialize: (Hash[String, String | Integer] payload) -> void + + def max_attempts_for_duration: (Float duration) -> Integer + + private + + def parse: (Hash[String, String | Integer] payload) -> Enumerable[Entry] + end + end + end +end diff --git a/spec/datadog/ci/remote/library_settings_client_spec.rb b/spec/datadog/ci/remote/library_settings_client_spec.rb index 1fc4ba6c..b4126892 100644 --- a/spec/datadog/ci/remote/library_settings_client_spec.rb +++ b/spec/datadog/ci/remote/library_settings_client_spec.rb @@ -116,6 +116,14 @@ expect(response.tests_skipping_enabled?).to be false expect(response.flaky_test_retries_enabled?).to be true expect(response.early_flake_detection_enabled?).to be true + expect(response.slow_test_retries.entries).to eq( + [ + Datadog::CI::Remote::SlowTestRetries::Entry.new(5.0, 10), + Datadog::CI::Remote::SlowTestRetries::Entry.new(10.0, 5), + Datadog::CI::Remote::SlowTestRetries::Entry.new(30.0, 3), + Datadog::CI::Remote::SlowTestRetries::Entry.new(300.0, 2) + ] + ) metric = telemetry_metric(:inc, "git_requests.settings_response") expect(metric.tags).to eq( diff --git a/spec/datadog/ci/remote/slow_test_retries_spec.rb b/spec/datadog/ci/remote/slow_test_retries_spec.rb new file mode 100644 index 00000000..33d75de5 --- /dev/null +++ b/spec/datadog/ci/remote/slow_test_retries_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require_relative "../../../../lib/datadog/ci/remote/slow_test_retries" + +RSpec.describe Datadog::CI::Remote::SlowTestRetries do + let(:payload) do + { + "5m" => 2, + "5s" => 10, + "30s" => 3, + "10s" => 5, + "ffdsfdsfds" => nil, + "10m" => nil, + "10h" => 12 + } + end + + subject(:slow_test_retries) { described_class.new(payload) } + + describe "#initialize" do + subject { slow_test_retries.entries } + + it do + is_expected.to eq([ + described_class::Entry.new(5.0, 10), + described_class::Entry.new(10.0, 5), + described_class::Entry.new(30.0, 3), + described_class::Entry.new(300.0, 2), + described_class::Entry.new(600.0, 0) + ]) + end + end + + describe "#max_attempts_for_duration" do + subject { slow_test_retries.max_attempts_for_duration(duration) } + + context "when the duration is less than 5 seconds" do + let(:duration) { 4.9 } + + it { is_expected.to eq(10) } + end + + context "when the duration is less than 10 seconds" do + let(:duration) { 9.9 } + + it { is_expected.to eq(5) } + end + + context "when the duration is less than 30 seconds" do + let(:duration) { 29.9 } + + it { is_expected.to eq(3) } + end + + context "when the duration is less than 5 minutes" do + let(:duration) { 299.9 } + + it { is_expected.to eq(2) } + end + + context "when the duration is less than 10 minutes" do + let(:duration) { 599.9 } + + it { is_expected.to eq(0) } + end + + context "when the duration is more than 10 minutes" do + let(:duration) { 600.1 } + + it { is_expected.to eq(1) } + end + end +end From a8ed9a975c924d2b19cfa8b660698b2eb7da6a46 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 13:52:30 +0200 Subject: [PATCH 05/16] use slow_test_retries in TestRetries::Component --- lib/datadog/ci/test_retries/component.rb | 8 +++++++- sig/datadog/ci/test_retries/component.rbs | 2 ++ spec/datadog/ci/test_retries/component_spec.rb | 17 +++++++++++++---- spec/support/contexts/ci_mode.rb | 16 ++++++++++++++-- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 26c109a7..8fda573e 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -12,7 +12,7 @@ module TestRetries class Component attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit, :retry_failed_tests_count, - :retry_new_tests_enabled + :retry_new_tests_enabled, :retry_new_tests_duration_thresholds def initialize( retry_failed_tests_enabled:, @@ -27,6 +27,7 @@ def initialize( @retry_failed_tests_count = 0 @retry_new_tests_enabled = retry_new_tests_enabled + @retry_new_tests_duration_thresholds = nil @mutex = Mutex.new end @@ -34,6 +35,11 @@ def initialize( def configure(library_settings) @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled? @retry_new_tests_enabled &&= library_settings.early_flake_detection_enabled? + + return unless @retry_new_tests_enabled + + # setup retrying new tests + @retry_new_tests_duration_thresholds = library_settings.slow_test_retries end def with_retries(&block) diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 3b38dec9..9a9e2988 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -12,6 +12,8 @@ module Datadog attr_reader retry_new_tests_enabled: bool + attr_reader retry_new_tests_duration_thresholds: Datadog::CI::Remote::SlowTestRetries? + @mutex: Thread::Mutex def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer, retry_new_tests_enabled: bool) -> void diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index c0ebdeea..567861ea 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -5,7 +5,8 @@ instance_double( Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: remote_flaky_test_retries_enabled, - early_flake_detection_enabled?: remote_early_flake_detection_enabled + early_flake_detection_enabled?: remote_early_flake_detection_enabled, + slow_test_retries: slow_test_retries ) end @@ -17,6 +18,13 @@ let(:remote_flaky_test_retries_enabled) { false } let(:remote_early_flake_detection_enabled) { false } + let(:slow_test_retries) do + instance_double( + Datadog::CI::Remote::SlowTestRetries, + max_attempts_for_duration: 10 + ) + end + subject(:component) do described_class.new( retry_failed_tests_enabled: retry_failed_tests_enabled, @@ -63,17 +71,18 @@ context "when early flake detection is enabled" do let(:remote_early_flake_detection_enabled) { true } - it "enables retrying failed tests" do + it "enables retrying new tests" do subject expect(component.retry_new_tests_enabled).to be true + expect(component.retry_new_tests_duration_thresholds.max_attempts_for_duration(1.2)).to eq(10) end end context "when early flake detection is disabled" do let(:remote_early_flake_detection_enabled) { false } - it "disables retrying failed tests" do + it "disables retrying new tests" do subject expect(component.retry_new_tests_enabled).to be false @@ -84,7 +93,7 @@ let(:retry_new_tests_enabled) { false } let(:remote_early_flake_detection_enabled) { true } - it "disables retrying failed tests even if it's enabled remotely" do + it "disables retrying new tests even if it's enabled remotely" do subject expect(component.retry_new_tests_enabled).to be false diff --git a/spec/support/contexts/ci_mode.rb b/spec/support/contexts/ci_mode.rb index 33ec12e3..35ac66a1 100644 --- a/spec/support/contexts/ci_mode.rb +++ b/spec/support/contexts/ci_mode.rb @@ -32,6 +32,16 @@ let(:retry_failed_tests_max_attempts) { 5 } let(:retry_failed_tests_total_limit) { 100 } + let(:slow_test_retries_payload) do + { + "5s" => 10, + "10s" => 5, + "30s" => 3, + "10m" => 2 + } + end + let(:slow_test_retries) { Datadog::CI::Remote::SlowTestRetries.new(slow_test_retries_payload) } + let(:itr_correlation_id) { "itr_correlation_id" } let(:itr_skippable_tests) { [] } @@ -68,7 +78,8 @@ code_coverage_enabled?: code_coverage_enabled, tests_skipping_enabled?: tests_skipping_enabled, flaky_test_retries_enabled?: flaky_test_retries_enabled, - early_flake_detection_enabled?: early_flake_detection_enabled + early_flake_detection_enabled?: early_flake_detection_enabled, + slow_test_retries: slow_test_retries ), # This is for the second call to fetch_library_settings instance_double( @@ -83,7 +94,8 @@ code_coverage_enabled?: !code_coverage_enabled, tests_skipping_enabled?: !tests_skipping_enabled, flaky_test_retries_enabled?: flaky_test_retries_enabled, - early_flake_detection_enabled?: early_flake_detection_enabled + early_flake_detection_enabled?: early_flake_detection_enabled, + slow_test_retries: slow_test_retries ) ) allow_any_instance_of(Datadog::CI::TestOptimisation::Skippable).to receive(:fetch_skippable_tests).and_return(skippable_tests_response) From 063dea73d5b685033c16baf5caed21a42f717fd8 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 14:06:04 +0200 Subject: [PATCH 06/16] move enabled and slow_test_retries keys to constants --- lib/datadog/ci/ext/transport.rb | 3 +++ lib/datadog/ci/remote/library_settings.rb | 16 +++++++++------- sig/datadog/ci/ext/transport.rbs | 6 ++++++ sig/datadog/ci/remote/library_settings.rbs | 2 ++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/datadog/ci/ext/transport.rb b/lib/datadog/ci/ext/transport.rb index 25adb1eb..61c1392b 100644 --- a/lib/datadog/ci/ext/transport.rb +++ b/lib/datadog/ci/ext/transport.rb @@ -38,6 +38,9 @@ module Transport DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git" DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled" DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY = "early_flake_detection" + DD_API_SETTINGS_RESPONSE_ENABLED_KEY = "enabled" + DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY = "early_flake_detection" + DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY = "faulty_session_threshold" DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits" diff --git a/lib/datadog/ci/remote/library_settings.rb b/lib/datadog/ci/remote/library_settings.rb index 472f1f67..31384c9c 100644 --- a/lib/datadog/ci/remote/library_settings.rb +++ b/lib/datadog/ci/remote/library_settings.rb @@ -94,10 +94,7 @@ def early_flake_detection_enabled? return @early_flake_detection_enabled if defined?(@early_flake_detection_enabled) @early_flake_detection_enabled = Utils::Parsing.convert_to_bool( - payload.fetch( - Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY, - {} - ).fetch("enabled", false) + early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ENABLED_KEY, false) ) end @@ -105,14 +102,19 @@ def slow_test_retries return @slow_test_retries if defined?(@slow_test_retries) @slow_test_retries = SlowTestRetries.new( - payload - .fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY, {}) - .fetch("slow_test_retries", {}) + early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY, {}) ) end private + def early_flake_detection_payload + payload.fetch( + Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY, + {} + ) + end + def default_payload Ext::Transport::DD_API_SETTINGS_RESPONSE_DEFAULT end diff --git a/sig/datadog/ci/ext/transport.rbs b/sig/datadog/ci/ext/transport.rbs index 88625092..3dc50095 100644 --- a/sig/datadog/ci/ext/transport.rbs +++ b/sig/datadog/ci/ext/transport.rbs @@ -52,6 +52,12 @@ module Datadog DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY: "early_flake_detection" + DD_API_SETTINGS_RESPONSE_ENABLED_KEY: "enabled" + + DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY: "slow_test_retries" + + DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY: "faulty_session_threshold" + DD_API_SETTINGS_RESPONSE_DEFAULT: Hash[String, untyped] DD_API_GIT_SEARCH_COMMITS_PATH: "/api/v2/git/repository/search_commits" diff --git a/sig/datadog/ci/remote/library_settings.rbs b/sig/datadog/ci/remote/library_settings.rbs index 7a5b72c2..3479f356 100644 --- a/sig/datadog/ci/remote/library_settings.rbs +++ b/sig/datadog/ci/remote/library_settings.rbs @@ -35,6 +35,8 @@ module Datadog private + def early_flake_detection_payload: () -> Hash[String, untyped] + def default_payload: () -> Hash[String, untyped] end end From 5a163ce600152afb2a0f3b25fce08ca9211e0add Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 14:09:48 +0200 Subject: [PATCH 07/16] fix DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY --- lib/datadog/ci/ext/transport.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datadog/ci/ext/transport.rb b/lib/datadog/ci/ext/transport.rb index 61c1392b..ec37373a 100644 --- a/lib/datadog/ci/ext/transport.rb +++ b/lib/datadog/ci/ext/transport.rb @@ -39,7 +39,7 @@ module Transport DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled" DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY = "early_flake_detection" DD_API_SETTINGS_RESPONSE_ENABLED_KEY = "enabled" - DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY = "early_flake_detection" + DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY = "slow_test_retries" DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY = "faulty_session_threshold" DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze From b9aadf365817820c4944777f5ac6184df3c4373a Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 14:27:00 +0200 Subject: [PATCH 08/16] parse faulty_session_threshold from remote settings --- lib/datadog/ci/remote/library_settings.rb | 8 ++++++++ sig/datadog/ci/remote/library_settings.rbs | 3 +++ spec/datadog/ci/remote/library_settings_client_spec.rb | 1 + 3 files changed, 12 insertions(+) diff --git a/lib/datadog/ci/remote/library_settings.rb b/lib/datadog/ci/remote/library_settings.rb index 31384c9c..61e61a5c 100644 --- a/lib/datadog/ci/remote/library_settings.rb +++ b/lib/datadog/ci/remote/library_settings.rb @@ -106,6 +106,14 @@ def slow_test_retries ) end + def faulty_session_threshold + return @faulty_session_threshold if defined?(@faulty_session_threshold) + + @faulty_session_threshold = early_flake_detection_payload.fetch( + Ext::Transport::DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY, 0 + ) + end + private def early_flake_detection_payload diff --git a/sig/datadog/ci/remote/library_settings.rbs b/sig/datadog/ci/remote/library_settings.rbs index 3479f356..9c2c676a 100644 --- a/sig/datadog/ci/remote/library_settings.rbs +++ b/sig/datadog/ci/remote/library_settings.rbs @@ -12,6 +12,7 @@ module Datadog @flaky_test_retries_enabled: bool @early_flake_detection_enabled: bool @slow_test_retries: Datadog::CI::Remote::SlowTestRetries + @faulty_session_threshold: Integer def initialize: (Datadog::CI::Transport::Adapters::Net::Response? http_response) -> void @@ -33,6 +34,8 @@ module Datadog def slow_test_retries: () -> Datadog::CI::Remote::SlowTestRetries + def faulty_session_threshold: () -> Integer + private def early_flake_detection_payload: () -> Hash[String, untyped] diff --git a/spec/datadog/ci/remote/library_settings_client_spec.rb b/spec/datadog/ci/remote/library_settings_client_spec.rb index b4126892..4e83c6a8 100644 --- a/spec/datadog/ci/remote/library_settings_client_spec.rb +++ b/spec/datadog/ci/remote/library_settings_client_spec.rb @@ -124,6 +124,7 @@ Datadog::CI::Remote::SlowTestRetries::Entry.new(300.0, 2) ] ) + expect(response.faulty_session_threshold).to eq(30) metric = telemetry_metric(:inc, "git_requests.settings_response") expect(metric.tags).to eq( From 080014c4f5ed82ffc380d5cfe89d6f1dac9b2986 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 14:50:07 +0200 Subject: [PATCH 09/16] set retry_new_tests_percentage_limit in TestRetries::Component --- lib/datadog/ci/test_retries/component.rb | 6 ++++-- sig/datadog/ci/test_retries/component.rbs | 2 ++ spec/datadog/ci/test_retries/component_spec.rb | 5 ++++- spec/support/contexts/ci_mode.rb | 7 +++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 8fda573e..007728de 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -12,7 +12,7 @@ module TestRetries class Component attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit, :retry_failed_tests_count, - :retry_new_tests_enabled, :retry_new_tests_duration_thresholds + :retry_new_tests_enabled, :retry_new_tests_duration_thresholds, :retry_new_tests_percentage_limit def initialize( retry_failed_tests_enabled:, @@ -28,6 +28,7 @@ def initialize( @retry_new_tests_enabled = retry_new_tests_enabled @retry_new_tests_duration_thresholds = nil + @retry_new_tests_percentage_limit = 0 @mutex = Mutex.new end @@ -38,8 +39,9 @@ def configure(library_settings) return unless @retry_new_tests_enabled - # setup retrying new tests + # configure retrying new tests @retry_new_tests_duration_thresholds = library_settings.slow_test_retries + @retry_new_tests_percentage_limit = library_settings.faulty_session_threshold end def with_retries(&block) diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 9a9e2988..66449ae3 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -14,6 +14,8 @@ module Datadog attr_reader retry_new_tests_duration_thresholds: Datadog::CI::Remote::SlowTestRetries? + attr_reader retry_new_tests_percentage_limit: Integer + @mutex: Thread::Mutex def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer, retry_new_tests_enabled: bool) -> void diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index 567861ea..43586d96 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -6,7 +6,8 @@ Datadog::CI::Remote::LibrarySettings, flaky_test_retries_enabled?: remote_flaky_test_retries_enabled, early_flake_detection_enabled?: remote_early_flake_detection_enabled, - slow_test_retries: slow_test_retries + slow_test_retries: slow_test_retries, + faulty_session_threshold: retry_new_tests_percentage_limit ) end @@ -14,6 +15,7 @@ let(:retry_failed_tests_max_attempts) { 1 } let(:retry_failed_tests_total_limit) { 12 } let(:retry_new_tests_enabled) { true } + let(:retry_new_tests_percentage_limit) { 30 } let(:remote_flaky_test_retries_enabled) { false } let(:remote_early_flake_detection_enabled) { false } @@ -76,6 +78,7 @@ expect(component.retry_new_tests_enabled).to be true expect(component.retry_new_tests_duration_thresholds.max_attempts_for_duration(1.2)).to eq(10) + expect(component.retry_new_tests_percentage_limit).to eq(retry_new_tests_percentage_limit) end end diff --git a/spec/support/contexts/ci_mode.rb b/spec/support/contexts/ci_mode.rb index 35ac66a1..58dd4e2c 100644 --- a/spec/support/contexts/ci_mode.rb +++ b/spec/support/contexts/ci_mode.rb @@ -28,6 +28,7 @@ let(:use_single_threaded_coverage) { false } let(:flaky_test_retries_enabled) { false } let(:early_flake_detection_enabled) { false } + let(:faulty_session_threshold) { 30 } let(:retry_failed_tests_max_attempts) { 5 } let(:retry_failed_tests_total_limit) { 100 } @@ -79,7 +80,8 @@ tests_skipping_enabled?: tests_skipping_enabled, flaky_test_retries_enabled?: flaky_test_retries_enabled, early_flake_detection_enabled?: early_flake_detection_enabled, - slow_test_retries: slow_test_retries + slow_test_retries: slow_test_retries, + faulty_session_threshold: faulty_session_threshold ), # This is for the second call to fetch_library_settings instance_double( @@ -95,7 +97,8 @@ tests_skipping_enabled?: !tests_skipping_enabled, flaky_test_retries_enabled?: flaky_test_retries_enabled, early_flake_detection_enabled?: early_flake_detection_enabled, - slow_test_retries: slow_test_retries + slow_test_retries: slow_test_retries, + faulty_session_threshold: faulty_session_threshold ) ) allow_any_instance_of(Datadog::CI::TestOptimisation::Skippable).to receive(:fetch_skippable_tests).and_return(skippable_tests_response) From a99a46a6594879d7a294347bdde26e2a69f15cf7 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 15:48:54 +0200 Subject: [PATCH 10/16] additional constants for new test retries, rename skippable_test_id to datadog_test_id --- lib/datadog/ci/ext/telemetry.rb | 6 ++++++ lib/datadog/ci/ext/transport.rb | 3 +++ lib/datadog/ci/remote/slow_test_retries.rb | 2 +- lib/datadog/ci/test.rb | 2 +- lib/datadog/ci/test_optimisation/component.rb | 8 ++++---- lib/datadog/ci/test_optimisation/skippable.rb | 2 +- lib/datadog/ci/utils/test_run.rb | 2 +- sig/datadog/ci/ext/telemetry.rbs | 10 ++++++++++ sig/datadog/ci/ext/transport.rbs | 4 ++++ sig/datadog/ci/utils/test_run.rbs | 2 +- spec/datadog/ci/remote/slow_test_retries_spec.rb | 2 +- spec/datadog/ci/utils/test_run_spec.rb | 4 ++-- 12 files changed, 35 insertions(+), 12 deletions(-) diff --git a/lib/datadog/ci/ext/telemetry.rb b/lib/datadog/ci/ext/telemetry.rb index 959980e7..d0feb603 100644 --- a/lib/datadog/ci/ext/telemetry.rb +++ b/lib/datadog/ci/ext/telemetry.rb @@ -54,6 +54,12 @@ module Telemetry METRIC_CODE_COVERAGE_IS_EMPTY = "code_coverage.is_empty" METRIC_CODE_COVERAGE_FILES = "code_coverage.files" + METRIC_EFD_UNIQUE_TESTS_REQUEST = "early_flake_detection.request" + METRIC_EFD_UNIQUE_TESTS_REQUEST_MS = "early_flake_detection.request_ms" + METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS = "early_flake_detection.request_errors" + METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES = "early_flake_detection.response_bytes" + METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS = "early_flake_detection.response_tests" + METRIC_TEST_SESSION = "test_session" TAG_TEST_FRAMEWORK = "test_framework" diff --git a/lib/datadog/ci/ext/transport.rb b/lib/datadog/ci/ext/transport.rb index ec37373a..2e64e62f 100644 --- a/lib/datadog/ci/ext/transport.rb +++ b/lib/datadog/ci/ext/transport.rb @@ -50,6 +50,9 @@ module Transport DD_API_SKIPPABLE_TESTS_PATH = "/api/v2/ci/tests/skippable" DD_API_SKIPPABLE_TESTS_TYPE = "test_params" + DD_API_UNIQUE_TESTS_PATH = "/api/v2/ci/libraries/tests" + DD_API_UNIQUE_TESTS_TYPE = "ci_app_libraries_tests_request" + CONTENT_TYPE_MESSAGEPACK = "application/msgpack" CONTENT_TYPE_JSON = "application/json" CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data" diff --git a/lib/datadog/ci/remote/slow_test_retries.rb b/lib/datadog/ci/remote/slow_test_retries.rb index 8ccfe236..f768427a 100644 --- a/lib/datadog/ci/remote/slow_test_retries.rb +++ b/lib/datadog/ci/remote/slow_test_retries.rb @@ -34,7 +34,7 @@ def max_attempts_for_duration(duration) return entry.max_attempts if duration < entry.duration end - 1 + 0 end private diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb index 97744c59..71ced4da 100644 --- a/lib/datadog/ci/test.rb +++ b/lib/datadog/ci/test.rb @@ -148,7 +148,7 @@ def parameters private def record_test_result(datadog_status) - test_id = Utils::TestRun.skippable_test_id(name, test_suite_name, parameters) + test_id = Utils::TestRun.datadog_test_id(name, test_suite_name, parameters) # if this test was already executed in this test suite, mark it as retried if test_suite&.test_executed?(test_id) diff --git a/lib/datadog/ci/test_optimisation/component.rb b/lib/datadog/ci/test_optimisation/component.rb index 729febc2..5f722d0d 100644 --- a/lib/datadog/ci/test_optimisation/component.rb +++ b/lib/datadog/ci/test_optimisation/component.rb @@ -143,8 +143,8 @@ def stop_coverage(test) def mark_if_skippable(test) return if !enabled? || !skipping_tests? - skippable_test_id = Utils::TestRun.skippable_test_id(test.name, test.test_suite_name, test.parameters) - if @skippable_tests.include?(skippable_test_id) + datadog_test_id = Utils::TestRun.datadog_test_id(test.name, test.test_suite_name, test.parameters) + if @skippable_tests.include?(datadog_test_id) if forked? Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" } return @@ -152,9 +152,9 @@ def mark_if_skippable(test) test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true") - Datadog.logger.debug { "Marked test as skippable: #{skippable_test_id}" } + Datadog.logger.debug { "Marked test as skippable: #{datadog_test_id}" } else - Datadog.logger.debug { "Test is not skippable: #{skippable_test_id}" } + Datadog.logger.debug { "Test is not skippable: #{datadog_test_id}" } end end diff --git a/lib/datadog/ci/test_optimisation/skippable.rb b/lib/datadog/ci/test_optimisation/skippable.rb index 8a63e71d..d65ae4f5 100644 --- a/lib/datadog/ci/test_optimisation/skippable.rb +++ b/lib/datadog/ci/test_optimisation/skippable.rb @@ -36,7 +36,7 @@ def tests next unless test_data["type"] == Ext::Test::ITR_TEST_SKIPPING_MODE attrs = test_data["attributes"] || {} - res << Utils::TestRun.skippable_test_id(attrs["name"], attrs["suite"], attrs["parameters"]) + res << Utils::TestRun.datadog_test_id(attrs["name"], attrs["suite"], attrs["parameters"]) end res diff --git a/lib/datadog/ci/utils/test_run.rb b/lib/datadog/ci/utils/test_run.rb index 53da3ab9..12ae1a5a 100644 --- a/lib/datadog/ci/utils/test_run.rb +++ b/lib/datadog/ci/utils/test_run.rb @@ -10,7 +10,7 @@ def self.command @command = "#{$0} #{ARGV.join(" ")}" end - def self.skippable_test_id(test_name, suite, parameters = nil) + def self.datadog_test_id(test_name, suite, parameters = nil) "#{suite}.#{test_name}.#{parameters}" end diff --git a/sig/datadog/ci/ext/telemetry.rbs b/sig/datadog/ci/ext/telemetry.rbs index 33e89226..89908ab7 100644 --- a/sig/datadog/ci/ext/telemetry.rbs +++ b/sig/datadog/ci/ext/telemetry.rbs @@ -70,6 +70,16 @@ module Datadog METRIC_ITR_SKIPPABLE_TESTS_RESPONSE_TESTS: "itr_skippable_tests.response_tests" + METRIC_EFD_UNIQUE_TESTS_REQUEST: "early_flake_detection.request" + + METRIC_EFD_UNIQUE_TESTS_REQUEST_MS: "early_flake_detection.request_ms" + + METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS: "early_flake_detection.request_errors" + + METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES: "early_flake_detection.response_bytes" + + METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS: "early_flake_detection.response_tests" + METRIC_ITR_SKIPPED: "itr_skipped" METRIC_ITR_UNSKIPPABLE: "itr_unskippable" diff --git a/sig/datadog/ci/ext/transport.rbs b/sig/datadog/ci/ext/transport.rbs index 3dc50095..b0ca5de2 100644 --- a/sig/datadog/ci/ext/transport.rbs +++ b/sig/datadog/ci/ext/transport.rbs @@ -68,6 +68,10 @@ module Datadog DD_API_SKIPPABLE_TESTS_TYPE: "test_params" + DD_API_UNIQUE_TESTS_PATH: "/api/v2/ci/libraries/tests" + + DD_API_UNIQUE_TESTS_TYPE: "ci_app_libraries_tests_request" + CONTENT_TYPE_MESSAGEPACK: "application/msgpack" CONTENT_TYPE_JSON: "application/json" diff --git a/sig/datadog/ci/utils/test_run.rbs b/sig/datadog/ci/utils/test_run.rbs index 9b78c27b..04240393 100644 --- a/sig/datadog/ci/utils/test_run.rbs +++ b/sig/datadog/ci/utils/test_run.rbs @@ -6,7 +6,7 @@ module Datadog def self.command: () -> String - def self.skippable_test_id: (String test_name, String? test_suite, ?String? parameters) -> String + def self.datadog_test_id: (String test_name, String? test_suite, ?String? parameters) -> String def self.test_parameters: (?arguments: Hash[untyped, untyped], ?metadata: Hash[untyped, untyped]) -> String diff --git a/spec/datadog/ci/remote/slow_test_retries_spec.rb b/spec/datadog/ci/remote/slow_test_retries_spec.rb index 33d75de5..6597609e 100644 --- a/spec/datadog/ci/remote/slow_test_retries_spec.rb +++ b/spec/datadog/ci/remote/slow_test_retries_spec.rb @@ -67,7 +67,7 @@ context "when the duration is more than 10 minutes" do let(:duration) { 600.1 } - it { is_expected.to eq(1) } + it { is_expected.to eq(0) } end end end diff --git a/spec/datadog/ci/utils/test_run_spec.rb b/spec/datadog/ci/utils/test_run_spec.rb index a6ead39e..d5b341dd 100644 --- a/spec/datadog/ci/utils/test_run_spec.rb +++ b/spec/datadog/ci/utils/test_run_spec.rb @@ -7,8 +7,8 @@ it { is_expected.to eq("#{$0} #{ARGV.join(" ")}") } end - describe ".skippable_test_id" do - subject { described_class.skippable_test_id(test_name, suite, parameters) } + describe ".datadog_test_id" do + subject { described_class.datadog_test_id(test_name, suite, parameters) } let(:test_name) { "test_name" } let(:suite) { "suite" } From db34827f2d76e1303edd89d96591659eccf36efb Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 16:22:14 +0200 Subject: [PATCH 11/16] add UniqueTestsClient to fetch a set of unique tests from backend --- .../ci/test_retries/unique_tests_client.rb | 132 +++++++++++ .../ci/test_retries/unique_tests_client.rbs | 36 +++ .../test_retries/unique_tests_client_spec.rb | 216 ++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 lib/datadog/ci/test_retries/unique_tests_client.rb create mode 100644 sig/datadog/ci/test_retries/unique_tests_client.rbs create mode 100644 spec/datadog/ci/test_retries/unique_tests_client_spec.rb diff --git a/lib/datadog/ci/test_retries/unique_tests_client.rb b/lib/datadog/ci/test_retries/unique_tests_client.rb new file mode 100644 index 00000000..eadfba98 --- /dev/null +++ b/lib/datadog/ci/test_retries/unique_tests_client.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require "json" + +require_relative "../ext/telemetry" +require_relative "../ext/transport" +require_relative "../transport/telemetry" +require_relative "../utils/telemetry" +require_relative "../utils/test_run" + +module Datadog + module CI + module TestRetries + # fetch a list of unique known tests from the backend + class UniqueTestsClient + class Response + def initialize(http_response) + @http_response = http_response + @json = nil + end + + def ok? + resp = @http_response + !resp.nil? && resp.ok? + end + + def tests + res = Set.new + + payload + .fetch("data", {}) + .fetch("attributes", {}) + .fetch("tests", {}) + .each do |_test_module, suites_hash| + suites_hash.each do |test_suite, tests| + tests.each do |test_name| + res << Utils::TestRun.datadog_test_id(test_name, test_suite) + end + end + end + + res + end + + private + + def payload + cached = @json + return cached unless cached.nil? + + resp = @http_response + return @json = {} if resp.nil? || !ok? + + begin + @json = JSON.parse(resp.payload) + rescue JSON::ParserError => e + Datadog.logger.error("Failed to parse unique known tests response payload: #{e}. Payload was: #{resp.payload}") + @json = {} + end + end + end + + def initialize(dd_env:, api: nil, config_tags: {}) + @api = api + @dd_env = dd_env + @config_tags = config_tags + end + + def fetch_unique_tests(test_session) + api = @api + return Response.new(nil) unless api + + request_payload = payload(test_session) + Datadog.logger.debug("Fetching unique known tests with request: #{request_payload}") + + http_response = api.api_request( + path: Ext::Transport::DD_API_UNIQUE_TESTS_PATH, + payload: request_payload + ) + + Transport::Telemetry.api_requests( + Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST, + 1, + compressed: http_response.request_compressed + ) + Utils::Telemetry.distribution(Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_MS, http_response.duration_ms) + Utils::Telemetry.distribution( + Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES, + http_response.response_size.to_f, + {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s} + ) + + unless http_response.ok? + Transport::Telemetry.api_requests_errors( + Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS, + 1, + error_type: http_response.telemetry_error_type, + status_code: http_response.code + ) + end + + Response.new(http_response) + end + + private + + def payload(test_session) + { + "data" => { + "id" => Datadog::Core::Environment::Identity.id, + "type" => Ext::Transport::DD_API_UNIQUE_TESTS_TYPE, + "attributes" => { + "repository_url" => test_session.git_repository_url, + "service" => test_session.service, + "env" => @dd_env, + "sha" => test_session.git_commit_sha, + "configurations" => { + Ext::Test::TAG_OS_PLATFORM => test_session.os_platform, + Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture, + Ext::Test::TAG_OS_VERSION => test_session.os_version, + Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name, + Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version, + "custom" => @config_tags + } + } + } + }.to_json + end + end + end + end +end diff --git a/sig/datadog/ci/test_retries/unique_tests_client.rbs b/sig/datadog/ci/test_retries/unique_tests_client.rbs new file mode 100644 index 00000000..14c0c084 --- /dev/null +++ b/sig/datadog/ci/test_retries/unique_tests_client.rbs @@ -0,0 +1,36 @@ +module Datadog + module CI + module TestRetries + class UniqueTestsClient + @api: Datadog::CI::Transport::Api::Base? + @dd_env: String? + @config_tags: Hash[String, String] + + class Response + @http_response: Datadog::CI::Transport::Adapters::Net::Response? + @json: Hash[String, untyped]? + + def initialize: (Datadog::CI::Transport::Adapters::Net::Response? http_response) -> void + + def ok?: () -> bool + + def correlation_id: () -> String? + + def tests: () -> Set[String] + + private + + def payload: () -> Hash[String, untyped] + end + + def initialize: (?api: Datadog::CI::Transport::Api::Base?, dd_env: String?, ?config_tags: Hash[String, String]) -> void + + def fetch_unique_tests: (Datadog::CI::TestSession test_session) -> Response + + private + + def payload: (Datadog::CI::TestSession test_session) -> String + end + end + end +end diff --git a/spec/datadog/ci/test_retries/unique_tests_client_spec.rb b/spec/datadog/ci/test_retries/unique_tests_client_spec.rb new file mode 100644 index 00000000..8b0d3cec --- /dev/null +++ b/spec/datadog/ci/test_retries/unique_tests_client_spec.rb @@ -0,0 +1,216 @@ +# frozen_string_literal: true + +require_relative "../../../../lib/datadog/ci/test_retries/unique_tests_client" + +RSpec.describe Datadog::CI::TestRetries::UniqueTestsClient do + include_context "Telemetry spy" + + let(:api) { spy("api") } + let(:dd_env) { "ci" } + let(:config_tags) { {} } + + subject(:client) { described_class.new(api: api, dd_env: dd_env, config_tags: config_tags) } + + describe "#fetch_unique_tests" do + subject { client.fetch_unique_tests(test_session) } + + let(:service) { "service" } + let(:tracer_span) do + Datadog::Tracing::SpanOperation.new("session", service: service).tap do |span| + span.set_tags({ + "git.repository_url" => "repository_url", + "git.branch" => "branch", + "git.commit.sha" => "commit_sha", + "os.platform" => "platform", + "os.architecture" => "arch", + "os.version" => "version", + "runtime.name" => "runtime_name", + "runtime.version" => "runtime_version" + }) + end + end + let(:test_session) { Datadog::CI::TestSession.new(tracer_span) } + + let(:path) { Datadog::CI::Ext::Transport::DD_API_UNIQUE_TESTS_PATH } + + it "requests the unique tests" do + subject + + expect(api).to have_received(:api_request) do |args| + expect(args[:path]).to eq(path) + + data = JSON.parse(args[:payload])["data"] + + expect(data["type"]).to eq(Datadog::CI::Ext::Transport::DD_API_UNIQUE_TESTS_TYPE) + + attributes = data["attributes"] + expect(attributes["service"]).to eq(service) + expect(attributes["env"]).to eq(dd_env) + expect(attributes["repository_url"]).to eq("repository_url") + expect(attributes["sha"]).to eq("commit_sha") + + configurations = attributes["configurations"] + expect(configurations["os.platform"]).to eq("platform") + expect(configurations["os.architecture"]).to eq("arch") + expect(configurations["os.version"]).to eq("version") + expect(configurations["runtime.name"]).to eq("runtime_name") + expect(configurations["runtime.version"]).to eq("runtime_version") + end + end + + context "parsing response" do + subject(:response) { client.fetch_unique_tests(test_session) } + + context "when api is present" do + before do + allow(api).to receive(:api_request).and_return(http_response) + end + + context "when response is OK" do + let(:http_response) do + double( + "http_response", + ok?: true, + payload: { + data: { + id: "wTGavjGXpUg", + type: "ci_app_libraries_tests", + attributes: { + tests: { + "rspec" => { + "AdminControllerTest" => [ + "test_new", + "test_index", + "test_create" + ] + } + } + } + } + }.to_json, + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100 + ) + end + + it "parses the response" do + expect(response.ok?).to be true + expect(response.tests).to eq(Set.new(["AdminControllerTest.test_new.", "AdminControllerTest.test_index.", "AdminControllerTest.test_create."])) + end + + it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request", 1 + it_behaves_like "emits telemetry metric", :distribution, "early_flake_detection.request_ms" + it_behaves_like "emits telemetry metric", :distribution, "early_flake_detection.response_bytes" + end + + context "when response is not OK" do + let(:http_response) do + double( + "http_response", + ok?: false, + payload: "", + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100, + telemetry_error_type: nil, + code: 422 + ) + end + + it "parses the response" do + expect(response.ok?).to be false + expect(response.tests).to be_empty + end + + it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request_errors", 1 + end + + context "when response is OK but JSON is malformed" do + let(:http_response) do + double( + "http_response", + ok?: true, + payload: "not json", + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100 + ) + end + + before do + expect(Datadog.logger).to receive(:error).with(/Failed to parse unique known tests response payload/) + end + + it "parses the response" do + expect(response.ok?).to be true + expect(response.tests).to be_empty + end + end + + context "when response is OK but JSON has different format" do + let(:http_response) do + double( + "http_response", + ok?: true, + payload: { + "attributes" => { + "suite" => "test_suite_name", + "name" => "test_name", + "parameters" => "string", + "configurations" => { + "os.platform" => "linux", + "os.version" => "bionic", + "os.architecture" => "amd64", + "runtime.vendor" => "string", + "runtime.architecture" => "amd64" + } + } + }.to_json, + request_compressed: false, + duration_ms: 1.2, + gzipped_content?: false, + response_size: 100 + ) + end + + it "parses the response" do + expect(response.ok?).to be true + expect(response.tests).to be_empty + end + end + end + + context "when there is no api" do + let(:api) { nil } + + it "returns an empty response" do + expect(response.ok?).to be false + expect(response.tests).to be_empty + end + end + end + + context "when there are custom configurations" do + let(:config_tags) do + { + "tag1" => "value1" + } + end + + it "requests the unique tests with custom configurations" do + subject + + expect(api).to have_received(:api_request) do |args| + data = JSON.parse(args[:payload])["data"] + configurations = data["attributes"]["configurations"] + + expect(configurations["custom"]).to eq("tag1" => "value1") + end + end + end + end +end From d17805b84342f3129a8f797eb5ccfd5e8a307915 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 30 Aug 2024 16:46:24 +0200 Subject: [PATCH 12/16] build and inject UniqueTestsClient in TestRetries::Component on library configuration stage --- lib/datadog/ci/configuration/components.rb | 12 +++++++++++- lib/datadog/ci/test_retries/component.rb | 5 ++++- sig/datadog/ci/configuration/components.rbs | 2 ++ sig/datadog/ci/test_retries/component.rbs | 4 +++- spec/datadog/ci/test_retries/component_spec.rb | 8 +++++++- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index d8fcdedd..4db8d121 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -11,6 +11,7 @@ require_relative "../test_optimisation/coverage/writer" require_relative "../test_retries/component" require_relative "../test_retries/null_component" +require_relative "../test_retries/unique_tests_client" require_relative "../test_visibility/component" require_relative "../test_visibility/flush" require_relative "../test_visibility/null_component" @@ -117,7 +118,8 @@ def activate_ci!(settings) retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled, retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts, retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit, - retry_new_tests_enabled: settings.ci.retry_new_tests_enabled + retry_new_tests_enabled: settings.ci.retry_new_tests_enabled, + unique_tests_client: build_unique_tests_client(settings, test_visibility_api) ) # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component @test_optimisation = build_test_optimisation(settings, test_visibility_api) @@ -237,6 +239,14 @@ def build_library_settings_client(settings, api) ) end + def build_unique_tests_client(settings, api) + TestRetries::UniqueTestsClient.new( + api: api, + dd_env: settings.env, + config_tags: custom_configuration(settings) + ) + end + # fetch custom tags provided by the user in DD_TAGS env var # with prefix test.configuration. def custom_configuration(settings) diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 007728de..02ad7ec3 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -18,7 +18,8 @@ def initialize( retry_failed_tests_enabled:, retry_failed_tests_max_attempts:, retry_failed_tests_total_limit:, - retry_new_tests_enabled: + retry_new_tests_enabled:, + unique_tests_client: ) @retry_failed_tests_enabled = retry_failed_tests_enabled @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts @@ -30,6 +31,8 @@ def initialize( @retry_new_tests_duration_thresholds = nil @retry_new_tests_percentage_limit = 0 + @unique_tests_client = unique_tests_client + @mutex = Mutex.new end diff --git a/sig/datadog/ci/configuration/components.rbs b/sig/datadog/ci/configuration/components.rbs index 42a76623..059e4d7e 100644 --- a/sig/datadog/ci/configuration/components.rbs +++ b/sig/datadog/ci/configuration/components.rbs @@ -34,6 +34,8 @@ module Datadog def build_library_settings_client: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::Remote::LibrarySettingsClient + def build_unique_tests_client: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::TestRetries::UniqueTestsClient + def custom_configuration: (untyped settings) -> Hash[String, String] def configure_telemetry: (untyped settings) -> void diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 66449ae3..5721f174 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -18,7 +18,9 @@ module Datadog @mutex: Thread::Mutex - def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer, retry_new_tests_enabled: bool) -> void + @unique_tests_client: Datadog::CI::TestRetries::UniqueTestsClient + + def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer, retry_new_tests_enabled: bool, unique_tests_client: TestRetries::UniqueTestsClient) -> void def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index 43586d96..b930bbed 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -1,4 +1,7 @@ +# frozen_string_literal: true + require_relative "../../../../lib/datadog/ci/test_retries/component" +require_relative "../../../../lib/datadog/ci/test_retries/unique_tests_client" RSpec.describe Datadog::CI::TestRetries::Component do let(:library_settings) do @@ -20,6 +23,8 @@ let(:remote_flaky_test_retries_enabled) { false } let(:remote_early_flake_detection_enabled) { false } + let(:unique_tests_client) { instance_double(Datadog::CI::TestRetries::UniqueTestsClient) } + let(:slow_test_retries) do instance_double( Datadog::CI::Remote::SlowTestRetries, @@ -32,7 +37,8 @@ retry_failed_tests_enabled: retry_failed_tests_enabled, retry_failed_tests_max_attempts: retry_failed_tests_max_attempts, retry_failed_tests_total_limit: retry_failed_tests_total_limit, - retry_new_tests_enabled: retry_new_tests_enabled + retry_new_tests_enabled: retry_new_tests_enabled, + unique_tests_client: unique_tests_client ) end From ddc2443c9e3bf79760d14785114af07a1ff74929 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 2 Sep 2024 12:46:28 +0200 Subject: [PATCH 13/16] fetch and store unique tests set when configuring TestRetries::Component --- lib/datadog/ci/remote/component.rb | 2 +- lib/datadog/ci/test_retries/component.rb | 14 ++++++- .../ci/test_retries/unique_tests_client.rb | 4 +- sig/datadog/ci/test_retries/component.rbs | 6 ++- .../ci/test_retries/unique_tests_client.rbs | 2 +- spec/datadog/ci/remote/component_spec.rb | 4 +- .../datadog/ci/test_retries/component_spec.rb | 40 ++++++++++++++----- .../test_retries/unique_tests_client_spec.rb | 15 +++---- spec/support/contexts/ci_mode.rb | 5 ++- 9 files changed, 63 insertions(+), 29 deletions(-) diff --git a/lib/datadog/ci/remote/component.rb b/lib/datadog/ci/remote/component.rb index f610c088..91a39c7f 100644 --- a/lib/datadog/ci/remote/component.rb +++ b/lib/datadog/ci/remote/component.rb @@ -28,7 +28,7 @@ def configure(test_session) end test_optimisation.configure(library_configuration, test_session) - test_retries.configure(library_configuration) + test_retries.configure(library_configuration, test_session) end private diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 02ad7ec3..67527040 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -12,7 +12,8 @@ module TestRetries class Component attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit, :retry_failed_tests_count, - :retry_new_tests_enabled, :retry_new_tests_duration_thresholds, :retry_new_tests_percentage_limit + :retry_new_tests_enabled, :retry_new_tests_duration_thresholds, :retry_new_tests_percentage_limit, + :retry_new_tests_unique_tests_set, :retry_new_tests_fault_reason def initialize( retry_failed_tests_enabled:, @@ -30,13 +31,16 @@ def initialize( @retry_new_tests_enabled = retry_new_tests_enabled @retry_new_tests_duration_thresholds = nil @retry_new_tests_percentage_limit = 0 + @retry_new_tests_unique_tests_set = Set.new + # indicates that retrying new tests failed and was disabled + @retry_new_tests_fault_reason = nil @unique_tests_client = unique_tests_client @mutex = Mutex.new end - def configure(library_settings) + def configure(library_settings, test_session) @retry_failed_tests_enabled &&= library_settings.flaky_test_retries_enabled? @retry_new_tests_enabled &&= library_settings.early_flake_detection_enabled? @@ -45,6 +49,12 @@ def configure(library_settings) # configure retrying new tests @retry_new_tests_duration_thresholds = library_settings.slow_test_retries @retry_new_tests_percentage_limit = library_settings.faulty_session_threshold + @retry_new_tests_unique_tests_set = @unique_tests_client.fetch_unique_tests(test_session) + + if @retry_new_tests_unique_tests_set.empty? + @retry_new_tests_enabled = false + @retry_new_tests_fault_reason = "unique tests set is empty" + end end def with_retries(&block) diff --git a/lib/datadog/ci/test_retries/unique_tests_client.rb b/lib/datadog/ci/test_retries/unique_tests_client.rb index eadfba98..1aa76748 100644 --- a/lib/datadog/ci/test_retries/unique_tests_client.rb +++ b/lib/datadog/ci/test_retries/unique_tests_client.rb @@ -68,7 +68,7 @@ def initialize(dd_env:, api: nil, config_tags: {}) def fetch_unique_tests(test_session) api = @api - return Response.new(nil) unless api + return Set.new unless api request_payload = payload(test_session) Datadog.logger.debug("Fetching unique known tests with request: #{request_payload}") @@ -99,7 +99,7 @@ def fetch_unique_tests(test_session) ) end - Response.new(http_response) + Response.new(http_response).tests end private diff --git a/sig/datadog/ci/test_retries/component.rbs b/sig/datadog/ci/test_retries/component.rbs index 5721f174..0ae6d246 100644 --- a/sig/datadog/ci/test_retries/component.rbs +++ b/sig/datadog/ci/test_retries/component.rbs @@ -16,13 +16,17 @@ module Datadog attr_reader retry_new_tests_percentage_limit: Integer + attr_reader retry_new_tests_unique_tests_set: Set[String] + + attr_reader retry_new_tests_fault_reason: String? + @mutex: Thread::Mutex @unique_tests_client: Datadog::CI::TestRetries::UniqueTestsClient def initialize: (retry_failed_tests_enabled: bool, retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer, retry_new_tests_enabled: bool, unique_tests_client: TestRetries::UniqueTestsClient) -> void - def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void + def configure: (Datadog::CI::Remote::LibrarySettings library_settings, Datadog::CI::TestSession test_session) -> void def with_retries: () { () -> void } -> void diff --git a/sig/datadog/ci/test_retries/unique_tests_client.rbs b/sig/datadog/ci/test_retries/unique_tests_client.rbs index 14c0c084..0ca6019f 100644 --- a/sig/datadog/ci/test_retries/unique_tests_client.rbs +++ b/sig/datadog/ci/test_retries/unique_tests_client.rbs @@ -25,7 +25,7 @@ module Datadog def initialize: (?api: Datadog::CI::Transport::Api::Base?, dd_env: String?, ?config_tags: Hash[String, String]) -> void - def fetch_unique_tests: (Datadog::CI::TestSession test_session) -> Response + def fetch_unique_tests: (Datadog::CI::TestSession test_session) -> Set[String] private diff --git a/spec/datadog/ci/remote/component_spec.rb b/spec/datadog/ci/remote/component_spec.rb index 29ef41ad..c045282f 100644 --- a/spec/datadog/ci/remote/component_spec.rb +++ b/spec/datadog/ci/remote/component_spec.rb @@ -34,7 +34,7 @@ before do expect(test_optimisation).to receive(:configure).with(library_configuration, test_session) - expect(test_retries).to receive(:configure).with(library_configuration) + expect(test_retries).to receive(:configure).with(library_configuration, test_session) end it { subject } @@ -49,7 +49,7 @@ .with(test_session).and_return(library_configuration) expect(test_optimisation).to receive(:configure).with(library_configuration, test_session) - expect(test_retries).to receive(:configure).with(library_configuration) + expect(test_retries).to receive(:configure).with(library_configuration, test_session) end it { subject } diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index b930bbed..e37ecd6e 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -23,7 +23,13 @@ let(:remote_flaky_test_retries_enabled) { false } let(:remote_early_flake_detection_enabled) { false } - let(:unique_tests_client) { instance_double(Datadog::CI::TestRetries::UniqueTestsClient) } + let(:unique_tests_set) { Set.new(["test1", "test2"]) } + let(:unique_tests_client) do + instance_double( + Datadog::CI::TestRetries::UniqueTestsClient, + fetch_unique_tests: unique_tests_set + ) + end let(:slow_test_retries) do instance_double( @@ -32,6 +38,9 @@ ) end + let(:tracer_span) { Datadog::Tracing::SpanOperation.new("session") } + let(:test_session) { Datadog::CI::TestSession.new(tracer_span) } + subject(:component) do described_class.new( retry_failed_tests_enabled: retry_failed_tests_enabled, @@ -43,7 +52,7 @@ end describe "#configure" do - subject { component.configure(library_settings) } + subject { component.configure(library_settings, test_session) } context "when flaky test retries are enabled" do let(:remote_flaky_test_retries_enabled) { true } @@ -79,12 +88,25 @@ context "when early flake detection is enabled" do let(:remote_early_flake_detection_enabled) { true } - it "enables retrying new tests" do - subject + context "when unique tests set is empty" do + let(:unique_tests_set) { Set.new } + + it "disables retrying new tests and adds fault reason" do + subject - expect(component.retry_new_tests_enabled).to be true - expect(component.retry_new_tests_duration_thresholds.max_attempts_for_duration(1.2)).to eq(10) - expect(component.retry_new_tests_percentage_limit).to eq(retry_new_tests_percentage_limit) + expect(component.retry_new_tests_enabled).to be false + expect(component.retry_new_tests_fault_reason).to eq("unique tests set is empty") + end + end + + context "when unique tests set is not empty" do + it "enables retrying new tests" do + subject + + expect(component.retry_new_tests_enabled).to be true + expect(component.retry_new_tests_duration_thresholds.max_attempts_for_duration(1.2)).to eq(10) + expect(component.retry_new_tests_percentage_limit).to eq(retry_new_tests_percentage_limit) + end end end @@ -129,7 +151,7 @@ let(:test_span) { instance_double(Datadog::CI::Test, failed?: test_failed) } before do - component.configure(library_settings) + component.configure(library_settings, test_session) end context "when retry failed tests is enabled" do @@ -218,7 +240,7 @@ end before do - component.configure(library_settings) + component.configure(library_settings, test_session) end context "when no retries strategy is used" do diff --git a/spec/datadog/ci/test_retries/unique_tests_client_spec.rb b/spec/datadog/ci/test_retries/unique_tests_client_spec.rb index 8b0d3cec..6031d5a2 100644 --- a/spec/datadog/ci/test_retries/unique_tests_client_spec.rb +++ b/spec/datadog/ci/test_retries/unique_tests_client_spec.rb @@ -96,8 +96,7 @@ end it "parses the response" do - expect(response.ok?).to be true - expect(response.tests).to eq(Set.new(["AdminControllerTest.test_new.", "AdminControllerTest.test_index.", "AdminControllerTest.test_create."])) + expect(response).to eq(Set.new(["AdminControllerTest.test_new.", "AdminControllerTest.test_index.", "AdminControllerTest.test_create."])) end it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request", 1 @@ -121,8 +120,7 @@ end it "parses the response" do - expect(response.ok?).to be false - expect(response.tests).to be_empty + expect(response).to be_empty end it_behaves_like "emits telemetry metric", :inc, "early_flake_detection.request_errors", 1 @@ -146,8 +144,7 @@ end it "parses the response" do - expect(response.ok?).to be true - expect(response.tests).to be_empty + expect(response).to be_empty end end @@ -178,8 +175,7 @@ end it "parses the response" do - expect(response.ok?).to be true - expect(response.tests).to be_empty + expect(response).to be_empty end end end @@ -188,8 +184,7 @@ let(:api) { nil } it "returns an empty response" do - expect(response.ok?).to be false - expect(response.tests).to be_empty + expect(response).to be_empty end end end diff --git a/spec/support/contexts/ci_mode.rb b/spec/support/contexts/ci_mode.rb index 58dd4e2c..2245f24e 100644 --- a/spec/support/contexts/ci_mode.rb +++ b/spec/support/contexts/ci_mode.rb @@ -45,7 +45,6 @@ let(:itr_correlation_id) { "itr_correlation_id" } let(:itr_skippable_tests) { [] } - let(:skippable_tests_response) do instance_double( Datadog::CI::TestOptimisation::Skippable::Response, @@ -55,6 +54,8 @@ ) end + let(:unique_tests_set) { Set.new } + let(:test_visibility) { Datadog.send(:components).test_visibility } before do @@ -104,6 +105,8 @@ allow_any_instance_of(Datadog::CI::TestOptimisation::Skippable).to receive(:fetch_skippable_tests).and_return(skippable_tests_response) allow_any_instance_of(Datadog::CI::TestOptimisation::Coverage::Transport).to receive(:send_events).and_return([]) + allow_any_instance_of(Datadog::CI::TestRetries::UniqueTestsClient).to receive(:fetch_unique_tests).and_return(unique_tests_set) + Datadog.configure do |c| # library switch c.ci.enabled = ci_enabled From 2296b7cd0e96b550b8814cd8c64fa49d50df6e3d Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 2 Sep 2024 13:04:00 +0200 Subject: [PATCH 14/16] emit early_flake_detection.response_tests metric after unique tests are fetched --- lib/datadog/ci/test_retries/component.rb | 8 ++++++++ spec/datadog/ci/test_retries/component_spec.rb | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 67527040..1da4bbd5 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -3,6 +3,9 @@ require_relative "strategy/no_retry" require_relative "strategy/retry_failed" +require_relative "../ext/telemetry" +require_relative "../utils/telemetry" + module Datadog module CI module TestRetries @@ -54,6 +57,11 @@ def configure(library_settings, test_session) if @retry_new_tests_unique_tests_set.empty? @retry_new_tests_enabled = false @retry_new_tests_fault_reason = "unique tests set is empty" + else + Utils::Telemetry.distribution( + Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS, + @retry_new_tests_unique_tests_set.size.to_f + ) end end diff --git a/spec/datadog/ci/test_retries/component_spec.rb b/spec/datadog/ci/test_retries/component_spec.rb index e37ecd6e..3a346eba 100644 --- a/spec/datadog/ci/test_retries/component_spec.rb +++ b/spec/datadog/ci/test_retries/component_spec.rb @@ -4,6 +4,8 @@ require_relative "../../../../lib/datadog/ci/test_retries/unique_tests_client" RSpec.describe Datadog::CI::TestRetries::Component do + include_context "Telemetry spy" + let(:library_settings) do instance_double( Datadog::CI::Remote::LibrarySettings, @@ -107,6 +109,8 @@ expect(component.retry_new_tests_duration_thresholds.max_attempts_for_duration(1.2)).to eq(10) expect(component.retry_new_tests_percentage_limit).to eq(retry_new_tests_percentage_limit) end + + it_behaves_like "emits telemetry metric", :distribution, "early_flake_detection.response_tests", 2 end end From b58b477f2a5f45f1fa93c77e94a6a0a99bbc8019 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 2 Sep 2024 15:38:09 +0200 Subject: [PATCH 15/16] configure test retries and test optimisation in parallel using threaded Workers --- lib/datadog/ci/remote/component.rb | 15 +++++++++++++-- lib/datadog/ci/test_retries/component.rb | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/datadog/ci/remote/component.rb b/lib/datadog/ci/remote/component.rb index 91a39c7f..35dd562e 100644 --- a/lib/datadog/ci/remote/component.rb +++ b/lib/datadog/ci/remote/component.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative "../worker" + module Datadog module CI module Remote @@ -27,8 +29,17 @@ def configure(test_session) end end - test_optimisation.configure(library_configuration, test_session) - test_retries.configure(library_configuration, test_session) + # configure different components in parallel because they might + configuration_workers = [ + Worker.new { test_optimisation.configure(library_configuration, test_session) }, + Worker.new { test_retries.configure(library_configuration, test_session) } + ] + + # launch configuration workers + configuration_workers.each(&:perform) + + # block until all workers are done (or 60 seconds has passed) + configuration_workers.each(&:wait_until_done) end private diff --git a/lib/datadog/ci/test_retries/component.rb b/lib/datadog/ci/test_retries/component.rb index 1da4bbd5..0406ace6 100644 --- a/lib/datadog/ci/test_retries/component.rb +++ b/lib/datadog/ci/test_retries/component.rb @@ -57,6 +57,8 @@ def configure(library_settings, test_session) if @retry_new_tests_unique_tests_set.empty? @retry_new_tests_enabled = false @retry_new_tests_fault_reason = "unique tests set is empty" + + Datadog.logger.debug("Unique tests set is empty, retrying new tests disabled") else Utils::Telemetry.distribution( Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS, From 46630f44cadee382b3fb60b8e57cc3a4d30cbe22 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 2 Sep 2024 15:43:52 +0200 Subject: [PATCH 16/16] finish comment --- lib/datadog/ci/remote/component.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datadog/ci/remote/component.rb b/lib/datadog/ci/remote/component.rb index 35dd562e..3edb65d6 100644 --- a/lib/datadog/ci/remote/component.rb +++ b/lib/datadog/ci/remote/component.rb @@ -29,7 +29,7 @@ def configure(test_session) end end - # configure different components in parallel because they might + # configure different components in parallel because they might block on HTTP requests configuration_workers = [ Worker.new { test_optimisation.configure(library_configuration, test_session) }, Worker.new { test_retries.configure(library_configuration, test_session) }