From ced48cee5fbb923a91e4757d4f15375345cc8805 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Nov 2024 14:23:45 +0100 Subject: [PATCH 01/15] extract patch_integration method in Contrib::Instrumentation --- lib/datadog/ci/contrib/instrumentation.rb | 53 ++++++++++++---------- sig/datadog/ci/contrib/instrumentation.rbs | 5 +- sig/datadog/ci/contrib/integration.rbs | 2 + 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/lib/datadog/ci/contrib/instrumentation.rb b/lib/datadog/ci/contrib/instrumentation.rb index a283fe6c..9430980a 100644 --- a/lib/datadog/ci/contrib/instrumentation.rb +++ b/lib/datadog/ci/contrib/instrumentation.rb @@ -21,29 +21,12 @@ def self.register_integration(integration_class) # This method is called when user has `c.ci.instrument :integration_name` in their code. def self.instrument(integration_name, options = {}, &block) integration = fetch_integration(integration_name) + # when manually instrumented, it might be configured via code integration.configure(options, &block) return unless integration.enabled - patch_results = integration.patch - if patch_results[:ok] - # try to patch dependant integrations (for example knapsack that depends on rspec) - dependants = integration.dependants - .map { |name| fetch_integration(name) } - .filter { |integration| integration.patchable? } - - Datadog.logger.debug("Found dependent integrations for #{integration_name}: #{dependants}") - - dependants.each do |dependent_integration| - dependent_integration.patch - end - else - error_message = <<-ERROR - Available?: #{patch_results[:available]}, Loaded?: #{patch_results[:loaded]}, - Compatible?: #{patch_results[:compatible]}, Patchable?: #{patch_results[:patchable]}" - ERROR - Datadog.logger.warn("Unable to patch #{integration_name} (#{error_message})") - end + patch_integration(integration, with_dependencies: true) end # This method instruments all additional test libraries (ex: selenium-webdriver) that need to be instrumented @@ -58,15 +41,11 @@ def self.instrument_on_session_start @registry.each do |name, integration| next unless integration.late_instrument? + next unless integration.enabled Datadog.logger.debug "#{name} is allowed to be late instrumented" - patch_results = integration.patch - if patch_results[:ok] - Datadog.logger.debug("#{name} is patched") - else - Datadog.logger.debug("#{name} is not patched (#{patch_results})") - end + patch_integration(integration) end end @@ -82,6 +61,30 @@ def self.integration_name(subclass) raise "Integration name could not be derived for #{subclass}" if result.nil? result end + + def self.patch_integration(integration, with_dependencies: false) + patch_results = integration.patch + + if patch_results[:ok] + Datadog.logger.debug("#{integration.class} is patched") + + return unless with_dependencies + + # try to patch dependant integrations (for example knapsack that depends on rspec) + dependants = integration.dependants + .map { |name| fetch_integration(name) } + .filter { |integration| integration.patchable? } + + Datadog.logger.debug("Found dependent integrations for #{integration.class}: #{dependants}") + + dependants.each do |dependent_integration| + patch_integration(dependent_integration, with_dependencies: true) + end + + else + Datadog.logger.debug("Attention: #{integration.class} is not patched (#{patch_results})") + end + end end end end diff --git a/sig/datadog/ci/contrib/instrumentation.rbs b/sig/datadog/ci/contrib/instrumentation.rbs index a10222fd..1f511280 100644 --- a/sig/datadog/ci/contrib/instrumentation.rbs +++ b/sig/datadog/ci/contrib/instrumentation.rbs @@ -11,13 +11,16 @@ module Datadog def self.instrument: (Symbol integration_name, ?::Hash[untyped, untyped] options) { (?) -> untyped } -> void + def self.instrument_on_session_start: () -> void + def self.fetch_integration: (Symbol name) -> untyped def self.integration_name: (Class) -> Symbol def self.register_integration: (Class integration) -> void - def self.instrument_on_session_start: () -> void + def self.patch_integration: (Contrib::Integration integration, ?with_dependencies: bool) -> void + end end end diff --git a/sig/datadog/ci/contrib/integration.rbs b/sig/datadog/ci/contrib/integration.rbs index 96be033b..a5f6fbac 100644 --- a/sig/datadog/ci/contrib/integration.rbs +++ b/sig/datadog/ci/contrib/integration.rbs @@ -24,6 +24,8 @@ module Datadog def patch: () -> Hash[Symbol, bool] + def dependants: () -> Array[Symbol] + def late_instrument?: () -> bool def new_configuration: () -> Datadog::CI::Contrib::Settings From 81739206046c73a6dc377c59e29aaa4405102caf Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Nov 2024 14:43:17 +0100 Subject: [PATCH 02/15] minimal auto instrumentation code --- lib/datadog/ci/contrib/instrumentation.rb | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/lib/datadog/ci/contrib/instrumentation.rb b/lib/datadog/ci/contrib/instrumentation.rb index 9430980a..53f6fdea 100644 --- a/lib/datadog/ci/contrib/instrumentation.rb +++ b/lib/datadog/ci/contrib/instrumentation.rb @@ -16,6 +16,46 @@ def self.register_integration(integration_class) @registry[integration_name(integration_class)] = integration_class.new end + # Auto instrumentation of all integrations. + # + # Registers a :script_compiled tracepoint to watch for new Ruby files being loaded. + # On every file load it checks if any of the integrations are patchable now. + # Only the integrations that are available in the environment are checked. + def self.auto_instrument + Datadog.logger.debug("Auto instrumenting all integrations...") + + auto_instrumented_integrations = [] + @registry.each do |name, integration| + # ignore integrations that are not in the Gemfile or have incompatible versions + next unless integration.compatible? + + # late instrumented integrations will be patched when the test session starts + next if integration.late_instrument? + + Datadog.logger.debug("#{name} should be auto instrumented") + auto_instrumented_integrations << integration + end + + if auto_instrumented_integrations.empty? + Datadog.logger.warn( + "Auto instrumentation was requested, but no available integrations were found. " \ + "Tests will be run without Datadog instrumentation." + ) + nil + end + + script_compiled_tracepoint = TracePoint.new(:script_compiled) do + auto_instrumented_integrations.each do |integration| + next unless integration.loaded? + + Datadog.logger.debug("#{integration.class} is loaded") + + patch_integration(integration) + end + end + script_compiled_tracepoint.enable + end + # Manual instrumentation of a specific integration. # # This method is called when user has `c.ci.instrument :integration_name` in their code. From 88a18372979eacebf857c2e2dee75c45c8d56b64 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Nov 2024 16:39:23 +0100 Subject: [PATCH 03/15] add datadog/ci/auto_instrument entrypoint --- lib/datadog/ci/auto_instrument.rb | 3 +++ lib/datadog/ci/contrib/instrumentation.rb | 2 +- sig/datadog/ci/contrib/instrumentation.rbs | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 lib/datadog/ci/auto_instrument.rb diff --git a/lib/datadog/ci/auto_instrument.rb b/lib/datadog/ci/auto_instrument.rb new file mode 100644 index 00000000..9a241843 --- /dev/null +++ b/lib/datadog/ci/auto_instrument.rb @@ -0,0 +1,3 @@ +require "datadog/ci" + +Datadog::CI::Contrib::Instrumentation.auto_instrument diff --git a/lib/datadog/ci/contrib/instrumentation.rb b/lib/datadog/ci/contrib/instrumentation.rb index 53f6fdea..b4564913 100644 --- a/lib/datadog/ci/contrib/instrumentation.rb +++ b/lib/datadog/ci/contrib/instrumentation.rb @@ -41,7 +41,7 @@ def self.auto_instrument "Auto instrumentation was requested, but no available integrations were found. " \ "Tests will be run without Datadog instrumentation." ) - nil + return end script_compiled_tracepoint = TracePoint.new(:script_compiled) do diff --git a/sig/datadog/ci/contrib/instrumentation.rbs b/sig/datadog/ci/contrib/instrumentation.rbs index 1f511280..8d168107 100644 --- a/sig/datadog/ci/contrib/instrumentation.rbs +++ b/sig/datadog/ci/contrib/instrumentation.rbs @@ -9,6 +9,8 @@ module Datadog def self.registry: () -> Hash[Symbol, untyped] + def self.auto_instrument: () -> void + def self.instrument: (Symbol integration_name, ?::Hash[untyped, untyped] options) { (?) -> untyped } -> void def self.instrument_on_session_start: () -> void From fb26faee3d84e4f519a0fd2cd0d69fad86eacd58 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Nov 2024 16:46:49 +0100 Subject: [PATCH 04/15] fix loaded? definitions for integrations; do not patch already patched integrations --- lib/datadog/ci/contrib/cucumber/integration.rb | 2 +- lib/datadog/ci/contrib/instrumentation.rb | 1 + lib/datadog/ci/contrib/integration.rb | 10 ++++++---- lib/datadog/ci/contrib/minitest/integration.rb | 3 ++- lib/datadog/ci/contrib/rspec/integration.rb | 4 +++- sig/datadog/ci/contrib/integration.rbs | 4 +++- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/datadog/ci/contrib/cucumber/integration.rb b/lib/datadog/ci/contrib/cucumber/integration.rb index 02d9b182..73384623 100644 --- a/lib/datadog/ci/contrib/cucumber/integration.rb +++ b/lib/datadog/ci/contrib/cucumber/integration.rb @@ -17,7 +17,7 @@ def version end def loaded? - !defined?(::Cucumber).nil? && !defined?(::Cucumber::Runtime).nil? + !defined?(::Cucumber).nil? && !defined?(::Cucumber::Runtime).nil? && !defined?(::Cucumber::Configuration).nil? end def compatible? diff --git a/lib/datadog/ci/contrib/instrumentation.rb b/lib/datadog/ci/contrib/instrumentation.rb index b4564913..9811ac8e 100644 --- a/lib/datadog/ci/contrib/instrumentation.rb +++ b/lib/datadog/ci/contrib/instrumentation.rb @@ -46,6 +46,7 @@ def self.auto_instrument script_compiled_tracepoint = TracePoint.new(:script_compiled) do auto_instrumented_integrations.each do |integration| + next if integration.patched? next unless integration.loaded? Datadog.logger.debug("#{integration.class} is loaded") diff --git a/lib/datadog/ci/contrib/integration.rb b/lib/datadog/ci/contrib/integration.rb index 8bb09dd5..494284ba 100644 --- a/lib/datadog/ci/contrib/integration.rb +++ b/lib/datadog/ci/contrib/integration.rb @@ -93,9 +93,7 @@ def patcher # @!visibility private def patch - # @type var patcher_klass: untyped - patcher_klass = patcher - if !patchable? || patcher_klass.nil? + if !patchable? || patcher.nil? return { ok: false, available: available?, @@ -105,10 +103,14 @@ def patch } end - patcher_klass.patch + patcher.patch {ok: true} end + def patched? + patcher&.patched? + end + # Can the patch for this integration be applied automatically? # @return [Boolean] can the tracer activate this instrumentation without explicit user input? def late_instrument? diff --git a/lib/datadog/ci/contrib/minitest/integration.rb b/lib/datadog/ci/contrib/minitest/integration.rb index 9a9f84ed..c766a42b 100644 --- a/lib/datadog/ci/contrib/minitest/integration.rb +++ b/lib/datadog/ci/contrib/minitest/integration.rb @@ -17,7 +17,8 @@ def version end def loaded? - !defined?(::Minitest).nil? + !defined?(::Minitest).nil? && !defined?(::Minitest::Runnable).nil? && !defined?(::Minitest::Test).nil? && + !defined?(::Minitest::CompositeReporter).nil? end def compatible? diff --git a/lib/datadog/ci/contrib/rspec/integration.rb b/lib/datadog/ci/contrib/rspec/integration.rb index 6483c73d..9b3c03ab 100644 --- a/lib/datadog/ci/contrib/rspec/integration.rb +++ b/lib/datadog/ci/contrib/rspec/integration.rb @@ -22,7 +22,9 @@ def version def loaded? !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? && - !defined?(::RSpec::Core::Example).nil? + !defined?(::RSpec::Core::Example).nil? && + !defined?(::RSpec::Core::Runner).nil? && + !defined?(::RSpec::Core::ExampleGroup).nil? end def compatible? diff --git a/sig/datadog/ci/contrib/integration.rbs b/sig/datadog/ci/contrib/integration.rbs index a5f6fbac..4c5d9edb 100644 --- a/sig/datadog/ci/contrib/integration.rbs +++ b/sig/datadog/ci/contrib/integration.rbs @@ -20,7 +20,9 @@ module Datadog def enabled: () -> bool - def patcher: () -> Datadog::Tracing::Contrib::Patcher? + def patcher: () -> untyped + + def patched?: () -> bool? def patch: () -> Hash[Symbol, bool] From 49258a8d941a59a75f437c6cddd901b0bee35763 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 21 Nov 2024 13:41:01 +0100 Subject: [PATCH 05/15] remove telemetry logs from patcher - patching might happen before Datadog is configured --- lib/datadog/ci/contrib/patcher.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/datadog/ci/contrib/patcher.rb b/lib/datadog/ci/contrib/patcher.rb index 74e0fce0..bf2b69ff 100644 --- a/lib/datadog/ci/contrib/patcher.rb +++ b/lib/datadog/ci/contrib/patcher.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "datadog/core/utils/only_once" -require "datadog/core/telemetry/logger" module Datadog module CI @@ -42,7 +41,6 @@ def patch # @param e [Exception] def on_patch_error(e) Datadog.logger.error("Failed to apply #{patch_name} patch. Cause: #{e} Location: #{Array(e.backtrace).first}") - Datadog::Core::Telemetry::Logger.report(e, description: "Failed to apply #{patch_name} patch") @patch_error_result = { type: e.class.name, From a9ce5845019c7d221571193acd588aa3d153b8a6 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 21 Nov 2024 14:42:20 +0100 Subject: [PATCH 06/15] working auto-instrumentation --- lib/datadog/ci/contrib/instrumentation.rb | 57 ++++++++++++++++------ sig/datadog/ci/contrib/instrumentation.rbs | 7 +++ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/lib/datadog/ci/contrib/instrumentation.rb b/lib/datadog/ci/contrib/instrumentation.rb index 9811ac8e..e212aca7 100644 --- a/lib/datadog/ci/contrib/instrumentation.rb +++ b/lib/datadog/ci/contrib/instrumentation.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "datadog/core/utils/only_once" + module Datadog module CI module Contrib @@ -24,18 +26,7 @@ def self.register_integration(integration_class) def self.auto_instrument Datadog.logger.debug("Auto instrumenting all integrations...") - auto_instrumented_integrations = [] - @registry.each do |name, integration| - # ignore integrations that are not in the Gemfile or have incompatible versions - next unless integration.compatible? - - # late instrumented integrations will be patched when the test session starts - next if integration.late_instrument? - - Datadog.logger.debug("#{name} should be auto instrumented") - auto_instrumented_integrations << integration - end - + auto_instrumented_integrations = fetch_auto_instrumented_integrations if auto_instrumented_integrations.empty? Datadog.logger.warn( "Auto instrumentation was requested, but no available integrations were found. " \ @@ -44,15 +35,26 @@ def self.auto_instrument return end - script_compiled_tracepoint = TracePoint.new(:script_compiled) do + script_compiled_tracepoint = TracePoint.new(:script_compiled) do |tp| + all_patched = true + auto_instrumented_integrations.each do |integration| next if integration.patched? + + all_patched = false next unless integration.loaded? - Datadog.logger.debug("#{integration.class} is loaded") + auto_configure_datadog + Datadog.logger.debug("#{integration.class} is loaded") patch_integration(integration) end + + if all_patched + Datadog.logger.debug("All expected integrations are patched, disabling the script_compiled tracepoint") + + tp.disable + end end script_compiled_tracepoint.enable end @@ -126,6 +128,33 @@ def self.patch_integration(integration, with_dependencies: false) Datadog.logger.debug("Attention: #{integration.class} is not patched (#{patch_results})") end end + + def self.fetch_auto_instrumented_integrations + @registry.filter_map do |name, integration| + # ignore integrations that are not in the Gemfile or have incompatible versions + next unless integration.compatible? + + # late instrumented integrations will be patched when the test session starts + next if integration.late_instrument? + + Datadog.logger.debug("#{name} should be auto instrumented") + integration + end + end + + def self.auto_configure_datadog + configure_once.run do + Datadog.logger.debug("Applying Datadog configuration in CI mode...") + Datadog.configure do |c| + c.ci.enabled = true + c.tracing.enabled = true + end + end + end + + def self.configure_once + @configure_once ||= Datadog::Core::Utils::OnlyOnce.new + end end end end diff --git a/sig/datadog/ci/contrib/instrumentation.rbs b/sig/datadog/ci/contrib/instrumentation.rbs index 8d168107..8d820d3e 100644 --- a/sig/datadog/ci/contrib/instrumentation.rbs +++ b/sig/datadog/ci/contrib/instrumentation.rbs @@ -7,6 +7,8 @@ module Datadog self.@registry: Hash[Symbol, untyped] + self.@configure_once: Datadog::Core::Utils::OnlyOnce + def self.registry: () -> Hash[Symbol, untyped] def self.auto_instrument: () -> void @@ -23,6 +25,11 @@ module Datadog def self.patch_integration: (Contrib::Integration integration, ?with_dependencies: bool) -> void + def self.fetch_auto_instrumented_integrations: () -> Array[Contrib::Integration] + + def self.auto_configure_datadog: () -> void + + def self.configure_once: () -> Datadog::Core::Utils::OnlyOnce end end end From 7e725e07e50dfbe5227a5e5c26195948a7387a0b Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 22 Nov 2024 12:05:13 +0100 Subject: [PATCH 07/15] fix README wording and documentation links --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a6a3c479..8d54e677 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -# Datadog Test Visibility for Ruby +# Datadog Test Optimization for Ruby [![Gem Version](https://badge.fury.io/rb/datadog-ci.svg)](https://badge.fury.io/rb/datadog-ci) [![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)](https://datadoghq.dev/datadog-ci-rb/) -[![codecov](https://codecov.io/gh/DataDog/datadog-ci-rb/branch/main/graph/badge.svg)](https://app.codecov.io/gh/DataDog/datadog-ci-rb/branch/main) -[![CircleCI](https://dl.circleci.com/status-badge/img/gh/DataDog/datadog-ci-rb/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DataDog/datadog-ci-rb/tree/main) Datadog's Ruby Library for instrumenting your tests. Learn more on our [official website](https://docs.datadoghq.com/tests/) and check out our [documentation for this library](https://docs.datadoghq.com/tests/setup/ruby/?tab=cloudciprovideragentless). @@ -11,12 +9,12 @@ Learn more on our [official website](https://docs.datadoghq.com/tests/) and chec ## Features - [Test Visibility](https://docs.datadoghq.com/tests/) - collect metrics and results for your tests -- [Intelligent test runner](https://docs.datadoghq.com/intelligent_test_runner/) - save time by selectively running only tests affected by code changes -- [Auto test retries](https://docs.datadoghq.com/tests/auto_test_retries/?tab=ruby) - retrying failing tests up to N times to avoid failing your build due to flaky tests -- [Early flake detection](https://docs.datadoghq.com/tests/early_flake_detection?tab=ruby) - Datadog’s test flakiness solution that identifies flakes early by running newly added tests multiple times +- [Test impact analysis](https://docs.datadoghq.com/tests/test_impact_analysis/) - save time by selectively running only tests affected by code changes +- [Flaky test management](https://docs.datadoghq.com/tests/flaky_test_management/) - track, alert, search your flaky tests in Datadog UI +- [Auto test retries](https://docs.datadoghq.com/tests/flaky_test_management/auto_test_retries/?tab=ruby) - retrying failing tests up to N times to avoid failing your build due to flaky tests +- [Early flake detection](https://docs.datadoghq.com/tests/flaky_test_management/early_flake_detection/?tab=ruby) - Datadog’s test flakiness solution that identifies flakes early by running newly added tests multiple times - [Search and manage CI tests](https://docs.datadoghq.com/tests/search/) - [Enhance developer workflows](https://docs.datadoghq.com/tests/developer_workflows) -- [Flaky test management](https://docs.datadoghq.com/tests/guides/flaky_test_management/) - [Add custom measures to your tests](https://docs.datadoghq.com/tests/guides/add_custom_measures/?tab=ruby) - [Browser tests integration with Datadog RUM](https://docs.datadoghq.com/tests/browser_tests) @@ -37,7 +35,7 @@ If you used [test visibility for Ruby](https://docs.datadoghq.com/tests/setup/ru ## Setup - [Test visibility setup](https://docs.datadoghq.com/tests/setup/ruby/?tab=cloudciprovideragentless) -- [Intelligent test runner setup](https://docs.datadoghq.com/intelligent_test_runner/setup/ruby) (test visibility setup is required before setting up intelligent test runner) +- [Test impact analysis setup](https://docs.datadoghq.com/tests/test_impact_analysis/setup/ruby/?tab=cloudciprovideragentless) (test visibility setup is required before setting up test impact analysis) ## Contributing From fd23ba8573769cb3af126f05687b2f5d3b680f81 Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 22 Nov 2024 12:05:22 +0100 Subject: [PATCH 08/15] minor fix for logs formatting --- lib/datadog/ci/test_retries/strategy/retry_new.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datadog/ci/test_retries/strategy/retry_new.rb b/lib/datadog/ci/test_retries/strategy/retry_new.rb index 0890feb7..277a60a1 100644 --- a/lib/datadog/ci/test_retries/strategy/retry_new.rb +++ b/lib/datadog/ci/test_retries/strategy/retry_new.rb @@ -101,7 +101,7 @@ def calculate_total_retries_limit(library_settings, test_session) end @total_limit = (tests_count * percentage_limit / 100.0).ceil Datadog.logger.debug do - "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}%) of #{tests_count}" + "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}% of #{tests_count})" end end From 7c4e42298706f7e54065aabf63f1ad46964d47ef Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 22 Nov 2024 13:00:23 +0100 Subject: [PATCH 09/15] synchronize a critical section for script_compiled tracepoint --- lib/datadog/ci/contrib/instrumentation.rb | 28 ++++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/datadog/ci/contrib/instrumentation.rb b/lib/datadog/ci/contrib/instrumentation.rb index e212aca7..b921f881 100644 --- a/lib/datadog/ci/contrib/instrumentation.rb +++ b/lib/datadog/ci/contrib/instrumentation.rb @@ -35,25 +35,30 @@ def self.auto_instrument return end + # note that `Kernel.require` might be called from a different thread, so + # there is a possibility of concurrent execution of this tracepoint + mutex = Mutex.new script_compiled_tracepoint = TracePoint.new(:script_compiled) do |tp| all_patched = true - auto_instrumented_integrations.each do |integration| - next if integration.patched? + mutex.synchronize do + auto_instrumented_integrations.each do |integration| + next if integration.patched? - all_patched = false - next unless integration.loaded? + all_patched = false + next unless integration.loaded? - auto_configure_datadog + auto_configure_datadog - Datadog.logger.debug("#{integration.class} is loaded") - patch_integration(integration) - end + Datadog.logger.debug("#{integration.class} is loaded") + patch_integration(integration) + end - if all_patched - Datadog.logger.debug("All expected integrations are patched, disabling the script_compiled tracepoint") + if all_patched + Datadog.logger.debug("All expected integrations are patched, disabling the script_compiled tracepoint") - tp.disable + tp.disable + end end end script_compiled_tracepoint.enable @@ -152,6 +157,7 @@ def self.auto_configure_datadog end end + # This is not thread safe, it is synchronized by the caller in the tracepoint def self.configure_once @configure_once ||= Datadog::Core::Utils::OnlyOnce.new end From c184f564811e8fd62ce684386d36feef5e7106b5 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Nov 2024 11:40:05 +0100 Subject: [PATCH 10/15] test auto instrumentation for minitest --- Rakefile | 4 ++ .../instrumentation_spec.rb | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 spec/datadog/ci/contrib/minitest_auto_instrument/instrumentation_spec.rb diff --git a/Rakefile b/Rakefile index b2106288..94216aab 100644 --- a/Rakefile +++ b/Rakefile @@ -58,6 +58,9 @@ TEST_METADATA = { "minitest" => { "minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby" }, + "minitest_auto_instrument" => { + "minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby" + }, "activesupport" => { "activesupport-4" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby", "activesupport-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby", @@ -149,6 +152,7 @@ namespace :spec do rspec minitest minitest_shoulda_context + minitest_auto_instrument activesupport ci_queue_minitest ci_queue_rspec diff --git a/spec/datadog/ci/contrib/minitest_auto_instrument/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest_auto_instrument/instrumentation_spec.rb new file mode 100644 index 00000000..2e2cfd8e --- /dev/null +++ b/spec/datadog/ci/contrib/minitest_auto_instrument/instrumentation_spec.rb @@ -0,0 +1,38 @@ +RSpec.describe "Minitest auto instrumentation" do + include_context "CI mode activated" do + let(:integration_name) { :no_instrument } + end + + before do + require_relative "../../../../../lib/datadog/ci/auto_instrument" + + require "minitest" + + class SomeTest < Minitest::Test + def test_pass + assert true + end + + def test_pass_other + assert true + end + end + + Minitest.run([]) + end + + it "instruments test session" do + expect(test_session_span).not_to be_nil + expect(test_module_span).not_to be_nil + + expect(first_test_suite_span).not_to be_nil + expect(first_test_suite_span.name).to eq( + "SomeTest at spec/datadog/ci/contrib/minitest_auto_instrument/instrumentation_spec.rb" + ) + + expect(test_spans).to have(2).items + expect(test_spans).to have_unique_tag_values_count(:test_session_id, 1) + expect(test_spans).to have_unique_tag_values_count(:test_module_id, 1) + expect(test_spans).to have_unique_tag_values_count(:test_suite_id, 1) + end +end From a8d7c5c9b175415cfab74b82f7ae28b3558c2b21 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Nov 2024 12:16:05 +0100 Subject: [PATCH 11/15] auto instrumentation tests for knapsack --- Rakefile | 4 + .../instrumentation_spec.rb | 80 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 spec/datadog/ci/contrib/knapsack_auto_instrument/instrumentation_spec.rb diff --git a/Rakefile b/Rakefile index 94216aab..53db03c3 100644 --- a/Rakefile +++ b/Rakefile @@ -79,6 +79,9 @@ TEST_METADATA = { "knapsack_rspec" => { "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby" }, + "knapsack_auto_instrument" => { + "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby" + }, "selenium" => { "selenium-4-capybara-3" => "❌ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ❌ 3.4 / ✅ jruby" }, @@ -157,6 +160,7 @@ namespace :spec do ci_queue_minitest ci_queue_rspec knapsack_rspec + knapsack_auto_instrument selenium timecop ].each do |contrib| desc "" # "Explicitly hiding from `rake -T`" diff --git a/spec/datadog/ci/contrib/knapsack_auto_instrument/instrumentation_spec.rb b/spec/datadog/ci/contrib/knapsack_auto_instrument/instrumentation_spec.rb new file mode 100644 index 00000000..e709b97f --- /dev/null +++ b/spec/datadog/ci/contrib/knapsack_auto_instrument/instrumentation_spec.rb @@ -0,0 +1,80 @@ +RSpec.describe "Knapsack Pro runner when Datadog::CI is auto instrumented" do + let(:integration) { Datadog::CI::Contrib::Instrumentation.fetch_integration(:rspec) } + + include_context "CI mode activated" do + let(:integration_name) { :no_instrument } + end + + before do + require_relative "../../../../../lib/datadog/ci/auto_instrument" + + require "fileutils" + require "knapsack_pro" + + expect(Datadog::CI).to receive(:start_test_session).never + expect(Datadog::CI).to receive(:start_test_module).never + expect(Datadog::CI).to receive(:start_test_suite).never + expect(Datadog::CI).to receive(:start_test).never + + allow_any_instance_of(Datadog::Core::Remote::Negotiation).to( + receive(:endpoint?).with("/evp_proxy/v4/").and_return(true) + ) + + allow(Datadog::CI::Utils::TestRun).to receive(:command).and_return("knapsack:queue:rspec") + + allow_any_instance_of(KnapsackPro::Runners::Queue::RSpecRunner).to receive(:test_file_paths).and_return( + ["./spec/datadog/ci/contrib/knapsack_rspec/suite_under_test/some_test_rspec.rb"], + [] + ) + + # raise to prevent Knapsack from running Kernel.exit(0) + allow(KnapsackPro::Report).to receive(:save_node_queue_to_api).and_raise(ArgumentError) + end + + it "instruments this rspec session" do + with_new_rspec_environment do + ClimateControl.modify( + "KNAPSACK_PRO_CI_NODE_BUILD_ID" => "144", + "KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC" => "example_token", + "KNAPSACK_PRO_FIXED_QUEUE_SPLIT" => "true", + "KNAPSACK_PRO_QUEUE_ID" => nil + ) do + KnapsackPro::Adapters::RSpecAdapter.bind + KnapsackPro::Runners::Queue::RSpecRunner.run("", devnull, devnull) + rescue ArgumentError + # suppress invalid API key error + end + end + + # test session and module traced + expect(test_session_span).not_to be_nil + expect(test_session_span).to have_test_tag(:framework, "rspec") + expect(test_session_span).to have_test_tag(:framework_version, integration.version.to_s) + + expect(test_module_span).not_to be_nil + + # test session and module are failed + expect([test_session_span, test_module_span]).to all have_fail_status + + # single test suite span + expect(test_suite_spans).to have(1).item + expect(test_suite_spans.first).to have_test_tag(:status, Datadog::CI::Ext::Test::Status::FAIL) + expect(test_suite_spans.first).to have_test_tag( + :suite, + "SomeTest at ./spec/datadog/ci/contrib/knapsack_rspec/suite_under_test/some_test_rspec.rb" + ) + + # there is test span for every test case + expect(test_spans).to have(2).items + # test spans belong to a single test suite + expect(test_spans).to have_unique_tag_values_count(:test_suite_id, 1) + expect(test_spans).to have_tag_values_no_order( + :status, + [Datadog::CI::Ext::Test::Status::FAIL, Datadog::CI::Ext::Test::Status::PASS] + ) + + # every test span is connected to test module and test session + expect(test_spans).to all have_test_tag(:test_module_id) + expect(test_spans).to all have_test_tag(:test_session_id) + end +end From 9089fbbbcd34a38a8d9c6d0cd701219193ed8b31 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Nov 2024 13:20:06 +0100 Subject: [PATCH 12/15] add empty file to please CI --- sig/datadog/ci/auto_instrument.rbs | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 sig/datadog/ci/auto_instrument.rbs diff --git a/sig/datadog/ci/auto_instrument.rbs b/sig/datadog/ci/auto_instrument.rbs new file mode 100644 index 00000000..e69de29b From 9077daadcf583cc4fb200740acf267b8edb253f9 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Nov 2024 13:27:59 +0100 Subject: [PATCH 13/15] auto instrumentation is not available for JRuby --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 53db03c3..c5edfd4c 100644 --- a/Rakefile +++ b/Rakefile @@ -59,7 +59,7 @@ TEST_METADATA = { "minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby" }, "minitest_auto_instrument" => { - "minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby" + "minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby" }, "activesupport" => { "activesupport-4" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby", From 1fb8c8a49efb277a687182a09b3a3c5e772c7ffb Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Nov 2024 13:29:06 +0100 Subject: [PATCH 14/15] ignore linter error for test file --- .standard_todo.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.standard_todo.yml b/.standard_todo.yml index 2e4a50dd..93369ceb 100644 --- a/.standard_todo.yml +++ b/.standard_todo.yml @@ -10,5 +10,7 @@ ignore: - Style/Alias - spec/datadog/ci/contrib/minitest/instrumentation_spec.rb: - Lint/ConstantDefinitionInBlock + - spec/datadog/ci/contrib/minitest_auto_instrument/instrumentation_spec.rb: + - Lint/ConstantDefinitionInBlock - spec/datadog/ci/contrib/timecop/instrumentation_spec.rb: - Lint/ConstantDefinitionInBlock From bdb658969cac122ea9539dff1c4fd5a011cfe46a Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Nov 2024 14:00:56 +0100 Subject: [PATCH 15/15] add exec CLI command for auto instrumentation --- exe/ddcirb | 4 +++- lib/datadog/ci/cli/cli.rb | 6 +++++- lib/datadog/ci/cli/command/exec.rb | 29 +++++++++++++++++++++++++++++ sig/datadog/ci/cli/cli.rbs | 2 +- sig/datadog/ci/cli/command/exec.rbs | 15 +++++++++++++++ spec/datadog/ci/cli/cli_spec.rb | 1 + 6 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 lib/datadog/ci/cli/command/exec.rb create mode 100644 sig/datadog/ci/cli/command/exec.rbs diff --git a/exe/ddcirb b/exe/ddcirb index 2575638e..f91a0bc2 100755 --- a/exe/ddcirb +++ b/exe/ddcirb @@ -2,4 +2,6 @@ require "datadog/ci/cli/cli" -Datadog::CI::CLI.exec(ARGV.first) +command = ARGV.shift + +Datadog::CI::CLI.exec(command, ARGV) diff --git a/lib/datadog/ci/cli/cli.rb b/lib/datadog/ci/cli/cli.rb index 99df9df9..cb061a83 100644 --- a/lib/datadog/ci/cli/cli.rb +++ b/lib/datadog/ci/cli/cli.rb @@ -1,14 +1,17 @@ require "datadog" require "datadog/ci" +require_relative "command/exec" require_relative "command/skippable_tests_percentage" require_relative "command/skippable_tests_percentage_estimate" module Datadog module CI module CLI - def self.exec(action) + def self.exec(action, args = []) case action + when "exec" + Command::Exec.new(args).exec when "skipped-tests", "skippable-tests" Command::SkippableTestsPercentage.new.exec when "skipped-tests-estimate", "skippable-tests-estimate" @@ -17,6 +20,7 @@ def self.exec(action) puts("Usage: bundle exec ddcirb [command] [options]. Available commands:") puts(" skippable-tests - calculates the exact percentage of skipped tests and prints it to stdout or file") puts(" skippable-tests-estimate - estimates the percentage of skipped tests and prints it to stdout or file") + puts(" exec YOUR_TEST_COMMAND - automatically instruments your test command with Datadog and executes it") end end end diff --git a/lib/datadog/ci/cli/command/exec.rb b/lib/datadog/ci/cli/command/exec.rb new file mode 100644 index 00000000..a8b2ae29 --- /dev/null +++ b/lib/datadog/ci/cli/command/exec.rb @@ -0,0 +1,29 @@ +require_relative "base" +require_relative "../../test_optimisation/skippable_percentage/estimator" + +module Datadog + module CI + module CLI + module Command + class Exec < Base + def initialize(args) + super() + + @args = args + end + + def exec + rubyopts = [ + "-rdatadog/ci/auto_instrument" + ] + + existing_rubyopt = ENV["RUBYOPT"] + ENV["RUBYOPT"] = existing_rubyopt ? "#{existing_rubyopt} #{rubyopts.join(" ")}" : rubyopts.join(" ") + + Kernel.exec(*@args) + end + end + end + end + end +end diff --git a/sig/datadog/ci/cli/cli.rbs b/sig/datadog/ci/cli/cli.rbs index 45e48a8f..12c5231e 100644 --- a/sig/datadog/ci/cli/cli.rbs +++ b/sig/datadog/ci/cli/cli.rbs @@ -1,7 +1,7 @@ module Datadog module CI module CLI - def self.exec: (String action) -> void + def self.exec: (String action, ?Array[String] args) -> void end end end diff --git a/sig/datadog/ci/cli/command/exec.rbs b/sig/datadog/ci/cli/command/exec.rbs new file mode 100644 index 00000000..36b0d03a --- /dev/null +++ b/sig/datadog/ci/cli/command/exec.rbs @@ -0,0 +1,15 @@ +module Datadog + module CI + module CLI + module Command + class Exec < Base + @args: Array[String] + + def initialize: (Array[String] args) -> void + + def exec: () -> void + end + end + end + end +end diff --git a/spec/datadog/ci/cli/cli_spec.rb b/spec/datadog/ci/cli/cli_spec.rb index 3d6c8011..4485ad19 100644 --- a/spec/datadog/ci/cli/cli_spec.rb +++ b/spec/datadog/ci/cli/cli_spec.rb @@ -36,6 +36,7 @@ Usage: bundle exec ddcirb [command] [options]. Available commands: skippable-tests - calculates the exact percentage of skipped tests and prints it to stdout or file skippable-tests-estimate - estimates the percentage of skipped tests and prints it to stdout or file + exec YOUR_TEST_COMMAND - automatically instruments your test command with Datadog and executes it USAGE end end