add skippable-tests-estimate cli command, add verbose mode

lib/datadog/ci/cli/cli.rb

@@ -7,9 +7,12 @@ def self.exec(action)
         case action
         when "skipped-tests", "skippable-tests"
           exec_skippable_tests_percentage
+        when "skipped-tests-estimate", "skippable-tests-estimate"
+          exec_skippable_tests_percentage_estimate
         else
-          puts("Available commands:")
+          puts("Usage: bundle exec ddcirb [command] [options]. Available commands:")
           puts("  skippable-tests - calculates the exact percentage of skipped tests and prints it to stdout or file")
+          puts("  skippable-tests-estimate - estimates the percentage of skipped tests and prints it to stdout or file")
         end
       end
 
@@ -22,12 +25,36 @@ def self.exec_skippable_tests_percentage
 
           opts.on("-f", "--file FILENAME", "Output result to file FILENAME")
           opts.on("--rspec-opts=[OPTIONS]", "Command line options to pass to RSpec")
+          opts.on("--verbose", "verbose output to stdout")
         end.parse!(into: ddcirb_options)
 
         additional_rspec_opts = (ddcirb_options[:"rspec-opts"] || "").split
 
         percentage_skipped = ::Datadog::CI::TestOptimisation::SkippablePercentage.new(
-          rspec_cli_options: additional_rspec_opts
+          rspec_cli_options: additional_rspec_opts,
+          verbose: !ddcirb_options[:verbose].nil?
+        ).calculate
+
+        if ddcirb_options[:file]
+          File.write(ddcirb_options[:file], percentage_skipped)
+        else
+          print(percentage_skipped)
+        end
+      end
+
+      def self.exec_skippable_tests_percentage_estimate
+        require "datadog/ci/test_optimisation/estimate_skippable_percentage"
+
+        ddcirb_options = {}
+        OptionParser.new do |opts|
+          opts.banner = "Usage: bundle exec ddcirb skippable-tests-estimate [options]"
+
+          opts.on("-f", "--file FILENAME", "Output result to file FILENAME")
+          opts.on("--verbose", "verbose output to stdout")
+        end.parse!(into: ddcirb_options)
+
+        percentage_skipped = ::Datadog::CI::TestOptimisation::EstimateSkippablePercentage.new(
+          verbose: !ddcirb_options[:verbose].nil?
         ).calculate
 
         if ddcirb_options[:file]

lib/datadog/ci/test_optimisation/estimate_skippable_percentage.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestOptimisation
+      # This class estimates the percentage of tests that are going to be skipped in the next run
+      # without actually running the tests. This estimate is very rough:
+      #
+      # - it counts the number of lines that start with "it" or "scenario" in the spec files, which could be inaccurate
+      #   if you defiuse shared examples
+      # - it only counts the number of tests that could be skipped, this does not mean that they will be actually skipped:
+      #   if in this commit you replaced all the tests in your test suite with new ones, all the tests would be run (but
+      #   this is highly unlikely)
+      #
+      # It is useful to determine the number of parallel jobs that are required for the CI pipeline.
+      #
+      # NOTE: Only RSpec is supported at the moment.
+      class EstimateSkippablePercentage
+        def initialize(verbose: false)
+          @verbose = verbose
+        end
+
+        def calculate
+          require "datadog/ci"
+
+          Datadog.configure do |c|
+            c.ci.enabled = true
+            c.ci.itr_enabled = true
+            c.ci.discard_traces = true
+            c.tracing.enabled = true
+          end
+
+          spec_files = Dir["spec/**/*_spec.rb"]
+          estimated_tests_count = spec_files.sum do |file|
+            content = File.read(file)
+            content.scan(/(^\s*it\s+)|(^\s*scenario\s+)/).size
+          end
+          puts "Estimated tests count: #{estimated_tests_count}" if @verbose
+
+          # starting and finishing a test session is required to get the skippable tests response
+          Datadog::CI.start_test_session(total_tests_count: estimated_tests_count)&.finish
+
+          test_optimisation = Datadog.send(:components).test_optimisation
+
+          skippable_tests_count = test_optimisation.skippable_tests.count
+
+          puts "Skippable tests count: #{skippable_tests_count}" if @verbose
+
+          [(skippable_tests_count.to_f / estimated_tests_count).floor(2), 0.99].min
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/test_optimisation/skippable_percentage.rb

@@ -3,15 +3,16 @@
 module Datadog
   module CI
     module TestOptimisation
-      # This class claculates the percentage of tests that are going to be skipped in the next run
+      # This class calculates the percentage of tests that are going to be skipped in the next run
       # without actually running the tests.
       #
       # It is useful to determine the number of parallel jobs that are required for the CI pipeline.
       #
       # NOTE: Only RSpec is supported at the moment.
       class SkippablePercentage
-        def initialize(rspec_cli_options: [])
+        def initialize(rspec_cli_options: [], verbose: false)
           @rspec_cli_options = rspec_cli_options || []
+          @verbose = verbose
         end
 
         def calculate
@@ -39,13 +40,17 @@ def calculate
 
           rspec_config_options = ::RSpec::Core::ConfigurationOptions.new(cli_options_array)
           devnull = File.new("/dev/null", "w")
-          exit_code = ::RSpec::Core::Runner.new(rspec_config_options).run(devnull, devnull)
+          out = @verbose ? $stdout : devnull
+          err = @verbose ? $stderr : devnull
+          exit_code = ::RSpec::Core::Runner.new(rspec_config_options).run(out, err)
 
           if exit_code != 0
             Datadog.logger.error("RSpec dry-run failed with exit code #{exit_code}")
           end
 
           test_optimisation = Datadog.send(:components).test_optimisation
+          puts "Total tests count: #{test_optimisation.total_tests_count}" if @verbose
+          puts "Skipped tests count: #{test_optimisation.skipped_tests_count}" if @verbose
 
           (test_optimisation.skipped_tests_count.to_f / test_optimisation.total_tests_count).floor(2)
         end

lib/tasks/rspec.rake

@@ -23,13 +23,14 @@ namespace :datadog do
           c.tracing.enabled = true
         end
 
-        test_session = Datadog::CI.start_test_session
+        estimated_tests_count = Dir["spec/**/*_spec.rb"].map { |path| File.read(path) }.join.scan(/(  it)|(  scenario)/).count
+
+        test_session = Datadog::CI.start_test_session(total_tests_count: estimated_tests_count)
         test_session&.finish
 
         test_optimisation = Datadog.send(:components).test_optimisation
 
         skippable_tests_count = test_optimisation.skippable_tests.count
-        estimated_tests_count = Dir["spec/**/*_spec.rb"].map { |path| File.read(path) }.join.scan(/(  it)|(  scenario)/).count
 
         result = [(skippable_tests_count.to_f / estimated_tests_count).floor(2), 0.99].min
         print(result)

sig/datadog/ci/cli/cli.rbs

@@ -4,6 +4,8 @@ module Datadog
       def self.exec: (String action) -> void
 
       def self.exec_skippable_tests_percentage: () -> void
+
+      def self.exec_skippable_tests_percentage_estimate: () -> void
     end
   end
 end

sig/datadog/ci/test_optimisation/estimate_skippable_percentage.rbs

@@ -0,0 +1,13 @@
+module Datadog
+  module CI
+    module TestOptimisation
+      class EstimateSkippablePercentage
+        @verbose: bool
+
+        def initialize: (?verbose: bool) -> void
+
+        def calculate: () -> Float
+      end
+    end
+  end
+end

sig/datadog/ci/test_optimisation/skippable_percentage.rbs

@@ -3,10 +3,11 @@ module Datadog
     module TestOptimisation
       class SkippablePercentage
         @rspec_cli_options: Array[String]
+        @verbose: bool
 
-        def initialize: (?rspec_cli_options: Array[String]) -> void
+        def initialize: (?rspec_cli_options: Array[String], ?verbose: bool) -> void
 
-        def calculate: () -> untyped
+        def calculate: () -> Float
       end
     end
   end