rake tasks to estimate percentage of skippable tests

Steepfile

@@ -31,4 +31,5 @@ target :lib do
   library "capybara"
   library "timecop"
   library "webmock"
+  library "rake"
 end

lib/datadog/ci/configuration/components.rb

@@ -11,7 +11,7 @@
 require_relative "../test_visibility/serializers/factories/test_level"
 require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
-require_relative "../test_visibility/stats_collector"
+require_relative "../test_visibility/null_transport"
 require_relative "../transport/api/builder"
 require_relative "../transport/remote_settings_api"
 require_relative "../utils/test_run"
@@ -22,15 +22,15 @@ module CI
     module Configuration
       # Adds CI behavior to Datadog trace components
       module Components
-        attr_reader :ci_recorder, :itr, :stats_collector
+        attr_reader :ci_recorder, :itr
 
         def initialize(settings)
+          @itr = nil
+          @ci_recorder = TestVisibility::NullRecorder.new
+
           # Activate CI mode if enabled
           if settings.ci.enabled
             activate_ci!(settings)
-          else
-            @itr = nil
-            @ci_recorder = TestVisibility::NullRecorder.new
           end
 
           super
@@ -54,7 +54,12 @@ def activate_ci!(settings)
             return
           end
 
-          # Configure ddtrace library for test visibility mode
+          # Builds test visibility API layer in agentless or EvP proxy mode
+          test_visibility_api = build_test_visibility_api(settings)
+          # bail out early if api is misconfigured
+          return unless settings.ci.enabled
+
+          # Configure datadog gem for test visibility mode
 
           # Deactivate telemetry
           settings.telemetry.enabled = false
@@ -76,6 +81,8 @@ def activate_ci!(settings)
             end
           end
 
+          # Configure Datadog::Tracing module
+
           # No need not use 128-bit trace ids for test visibility,
           # they are used for OTEL compatibility in Datadog tracer
           settings.tracing.trace_id_128_bit_generation_enabled = false
@@ -85,82 +92,32 @@ def activate_ci!(settings)
           settings.tracing.test_mode.async = true
           settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Partial.new
 
-          # Builds test visibility API layer in agentless or EvP proxy mode
-          test_visibility_api = build_test_visibility_api(settings)
-
-          # transport creation
           trace_writer_options = settings.ci.writer_options
           trace_writer_options[:shutdown_timeout] = 60
           trace_writer_options[:buffer_size] = 10_000
-
-          coverage_writer = nil
-
-          # StatsCollector is used to collect test stats when used with dry run mode.
-          # Note that stats are only collected if test visibility is configured to collect stats.
-          # In this case no traces are sent to Datadog.
-          @stats_collector = TestVisibility::StatsCollector.new
-
-          if true
-            # configure tracer to collect stats for test optimisation
-            trace_writer_options[:transport] = @stats_collector
-          elsif test_visibility_api
-            # setup writer for code coverage payloads
-            coverage_writer = ITR::Coverage::Writer.new(
-              transport: ITR::Coverage::Transport.new(api: test_visibility_api)
-            )
-
-            # configure tracing writer to send traces to test visibility backend
-            trace_writer_options[:transport] = TestVisibility::Transport.new(
-              api: test_visibility_api,
-              serializers_factory: serializers_factory(settings),
-              dd_env: settings.env
-            )
-          else
-            # only legacy APM protocol is supported, so no test suite level visibility
-            settings.ci.force_test_level_visibility = true
-
-            # ITR is not supported with APM protocol
-            settings.ci.itr_enabled = false
-          end
+          tracing_transport = build_tracing_transport(settings, test_visibility_api)
+          trace_writer_options[:transport] = tracing_transport if tracing_transport
 
           settings.tracing.test_mode.writer_options = trace_writer_options
 
-          custom_configuration_tags = Utils::TestRun.custom_configuration(settings.tags)
-
-          remote_settings_api = Transport::RemoteSettingsApi.new(
-            api: test_visibility_api,
-            dd_env: settings.env,
-            config_tags: custom_configuration_tags
-          )
-
-          itr = ITR::Runner.new(
+          # @type ivar @itr: Datadog::CI::ITR::Runner
+          @itr = ITR::Runner.new(
             api: test_visibility_api,
             dd_env: settings.env,
-            config_tags: custom_configuration_tags,
-            coverage_writer: coverage_writer,
+            config_tags: custom_configuration(settings),
+            coverage_writer: build_coverage_writer(settings, test_visibility_api),
             enabled: settings.ci.enabled && settings.ci.itr_enabled,
             bundle_location: settings.ci.itr_code_coverage_excluded_bundle_path,
             use_single_threaded_coverage: settings.ci.itr_code_coverage_use_single_threaded_mode
           )
 
-          git_tree_uploader = Git::TreeUploader.new(api: test_visibility_api)
-          git_tree_upload_worker = if settings.ci.git_metadata_upload_enabled
-            Worker.new do |repository_url|
-              git_tree_uploader.call(repository_url)
-            end
-          else
-            DummyWorker.new
-          end
-
           # CI visibility recorder global instance
           @ci_recorder = TestVisibility::Recorder.new(
             test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility,
-            itr: itr,
-            remote_settings_api: remote_settings_api,
-            git_tree_upload_worker: git_tree_upload_worker
+            itr: @itr,
+            remote_settings_api: build_remote_settings_client(settings, test_visibility_api),
+            git_tree_upload_worker: build_git_upload_worker(settings, test_visibility_api)
           )
-
-          @itr = itr
         end
 
         def build_test_visibility_api(settings)
@@ -189,12 +146,62 @@ def build_test_visibility_api(settings)
               Datadog.logger.debug(
                 "Old agent version detected, no evp_proxy support. Forcing test level visibility mode"
               )
+
+              # only legacy APM protocol is supported, so no test suite level visibility
+              settings.ci.force_test_level_visibility = true
+
+              # ITR is not supported with APM protocol
+              settings.ci.itr_enabled = false
             end
           end
 
           api
         end
 
+        def build_tracing_transport(settings, api)
+          return TestVisibility::NullTransport.new if settings.ci.discard_traces
+          return nil if api.nil?
+
+          TestVisibility::Transport.new(
+            api: api,
+            serializers_factory: serializers_factory(settings),
+            dd_env: settings.env
+          )
+        end
+
+        def build_coverage_writer(settings, api)
+          return nil if api.nil?
+
+          ITR::Coverage::Writer.new(
+            transport: ITR::Coverage::Transport.new(api: api)
+          )
+        end
+
+        def build_git_upload_worker(settings, api)
+          if settings.ci.git_metadata_upload_enabled
+            git_tree_uploader = Git::TreeUploader.new(api: api)
+            Worker.new do |repository_url|
+              git_tree_uploader.call(repository_url)
+            end
+          else
+            DummyWorker.new
+          end
+        end
+
+        def build_remote_settings_client(settings, api)
+          Transport::RemoteSettingsApi.new(
+            api: api,
+            dd_env: settings.env,
+            config_tags: custom_configuration(settings)
+          )
+        end
+
+        # fetch custom tags provided by the user in DD_TAGS env var
+        # with prefix test.configuration.
+        def custom_configuration(settings)
+          @custom_configuration ||= Utils::TestRun.custom_configuration(settings.tags)
+        end
+
         def serializers_factory(settings)
           if settings.ci.force_test_level_visibility
             TestVisibility::Serializers::Factories::TestLevel

lib/datadog/ci/configuration/settings.rb

@@ -82,6 +82,12 @@ def self.add_settings!(base)
                 o.default false
               end
 
+              # internal only
+              option :discard_traces do |o|
+                o.type :bool
+                o.default false
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

lib/datadog/ci/test_visibility/null_recorder.rb

@@ -45,6 +45,10 @@ def active_test_suite(test_suite_name)
         def shutdown!
         end
 
+        def itr_enabled?
+          false
+        end
+
         private
 
         def skip_tracing(block = nil)

lib/datadog/ci/test_visibility/stats_collector.rb → lib/datadog/ci/test_visibility/null_transport.rb

@@ -1,15 +1,9 @@
 # frozen_string_literal: true
 
-require "set"
-
-require_relative "../ext/app_types"
-require_relative "../ext/test"
-require_relative "../utils/test_run"
-
 module Datadog
   module CI
     module TestVisibility
-      class StatsCollector
+      class NullTransport
         def initialize
         end
 

lib/tasks/rspec.rake

@@ -7,9 +7,8 @@ namespace :datadog do
 
         Datadog.configure do |c|
           c.ci.enabled = true
-          # disabling ITR makes no sense for this task
           c.ci.itr_enabled = true
-
+          c.ci.discard_traces = true
           c.ci.instrument :rspec, dry_run_enabled: true
         end
 
@@ -18,14 +17,41 @@ namespace :datadog do
           spec
         ]
         options = ::RSpec::Core::ConfigurationOptions.new(rspec_cli_options)
-        exit_code = ::RSpec::Core::Runner.new(options).run($stderr, $stdout)
+        devnull = File.new("/dev/null", "w")
+        exit_code = ::RSpec::Core::Runner.new(options).run(devnull, devnull)
 
         if exit_code != 0
           Datadog.logger.error("RSpec dry-run failed with exit code #{exit_code}")
         end
 
         itr = Datadog.send(:components).itr
-        puts "Skippable percentage: #{itr.skipped_tests_count.to_f / itr.total_tests_count}"
+        print((itr.skipped_tests_count.to_f / itr.total_tests_count).floor(2))
+      end
+
+      task :skippable_percentage_estimate do
+        require "datadog/ci"
+
+        if ENV["DD_SERVICE"].nil?
+          Datadog.logger.error("DD_SERVICE is not set. You must provide it to estimate the skippable percentage.")
+          exit 1
+        end
+
+        Datadog.configure do |c|
+          c.ci.enabled = true
+          c.ci.itr_enabled = true
+          c.ci.discard_traces = true
+        end
+
+        test_session = Datadog::CI.start_test_session
+        test_session&.finish
+
+        itr = Datadog.send(:components).itr
+
+        skippable_tests_count = itr.skippable_tests.count
+        estimated_tests_count = Dir["spec/**/*_spec.rb"].map { |path| File.read(path) }.join.scan(/(  it)|(  scenario)/).count
+
+        result = [(skippable_tests_count.to_f / estimated_tests_count).floor(2), 0.99].min
+        print(result)
       end
     end
   end

sig/datadog/ci/configuration/components.rbs

@@ -4,6 +4,7 @@ module Datadog
       module Components : Datadog::Core::Configuration::Components
         @ci_recorder: Datadog::CI::TestVisibility::Recorder | Datadog::CI::TestVisibility::NullRecorder
         @itr: Datadog::CI::ITR::Runner?
+        @custom_configuration: Hash[String, String]
 
         attr_reader ci_recorder: Datadog::CI::TestVisibility::Recorder | Datadog::CI::TestVisibility::NullRecorder
         attr_reader itr: Datadog::CI::ITR::Runner?
@@ -18,6 +19,16 @@ module Datadog
 
         def check_dd_site: (untyped settings) -> void
 
+        def build_tracing_transport: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> (Datadog::CI::TestVisibility::Transport? | Datadog::CI::TestVisibility::NullTransport)
+
+        def build_coverage_writer: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::ITR::Coverage::Writer?
+
+        def build_git_upload_worker: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::Worker
+
+        def build_remote_settings_client: (untyped settings, Datadog::CI::Transport::Api::Base? api) -> Datadog::CI::Transport::RemoteSettingsApi
+
+        def custom_configuration: (untyped settings) -> Hash[String, String]
+
         def timecop?: () -> bool
       end
     end

sig/datadog/ci/tasks.rbs

@@ -0,0 +1,9 @@
+module Datadog
+  module CI
+    module Tasks
+      extend ::Rake::DSL
+
+      def self.load!: () -> void
+    end
+  end
+end

sig/datadog/ci/test_visibility/null_transport.rbs

@@ -0,0 +1,11 @@
+module Datadog
+  module CI
+    module TestVisibility
+      class NullTransport
+        def initialize: () -> void
+
+        def send_traces: (untyped traces) -> []
+      end
+    end
+  end
+end

vendor/rbs/rake/0/rake.rbs

@@ -0,0 +1,6 @@
+module Rake
+end
+
+module Rake::DSL
+  def import: (String file) -> void
+end