Merge pull request #159 from DataDog/anmarchenko/itr_runner_fetch_ski…

…ppable_tests

[CIVIS-2934] Request skippable tests when configuring ITR

Steepfile

@@ -15,6 +15,7 @@ target :lib do
   library "securerandom"
   library "tmpdir"
   library "fileutils"
+  library "socket"
 
   repo_path "vendor/rbs"
   library "ddtrace"

lib/datadog/ci/configuration/components.rb

@@ -96,6 +96,8 @@ def activate_ci!(settings)
           )
 
           itr = ITR::Runner.new(
+            api: test_visibility_api,
+            dd_env: settings.env,
             coverage_writer: coverage_writer,
             enabled: settings.ci.enabled && settings.ci.itr_enabled
           )

lib/datadog/ci/ext/test.rb

@@ -43,6 +43,7 @@ module Test
         # Environment runtime tags
         TAG_OS_ARCHITECTURE = "os.architecture"
         TAG_OS_PLATFORM = "os.platform"
+        TAG_OS_VERSION = "os.version"
         TAG_RUNTIME_NAME = "runtime.name"
         TAG_RUNTIME_VERSION = "runtime.version"
 

lib/datadog/ci/itr/runner.rb

@@ -10,6 +10,7 @@
 require_relative "../utils/parsing"
 
 require_relative "coverage/event"
+require_relative "skippable"
 
 module Datadog
   module CI
@@ -18,20 +19,30 @@ module ITR
       # Integrates with backend to provide test impact analysis data and
       # skip tests that are not impacted by the changes
       class Runner
+        attr_reader :correlation_id, :skippable_tests
+
         def initialize(
+          dd_env:,
+          api: nil,
           coverage_writer: nil,
           enabled: false
         )
           @enabled = enabled
+          @api = api
+          @dd_env = dd_env
+
           @test_skipping_enabled = false
           @code_coverage_enabled = false
 
           @coverage_writer = coverage_writer
 
+          @correlation_id = nil
+          @skippable_tests = []
+
           Datadog.logger.debug("ITR Runner initialized with enabled: #{@enabled}")
         end
 
-        def configure(remote_configuration, test_session)
+        def configure(remote_configuration, test_session:, git_tree_upload_worker:)
           Datadog.logger.debug("Configuring ITR Runner with remote configuration: #{remote_configuration}")
 
           @enabled = Utils::Parsing.convert_to_bool(
@@ -55,6 +66,8 @@ def configure(remote_configuration, test_session)
           load_datadog_cov! if @code_coverage_enabled
 
           Datadog.logger.debug("Configured ITR Runner with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
+
+          fetch_skippable_tests(test_session: test_session, git_tree_upload_worker: git_tree_upload_worker)
         end
 
         def enabled?
@@ -129,6 +142,20 @@ def ensure_test_source_covered(test_source_file, coverage)
 
           coverage[absolute_test_source_file_path] = true
         end
+
+        def fetch_skippable_tests(test_session:, git_tree_upload_worker:)
+          return unless skipping_tests?
+
+          # we can only request skippable tests if git metadata is already uploaded
+          git_tree_upload_worker.wait_until_done
+
+          skippable_response = Skippable.new(api: @api, dd_env: @dd_env).fetch_skippable_tests(test_session)
+          @correlation_id = skippable_response.correlation_id
+          @skippable_tests = skippable_response.tests
+
+          Datadog.logger.debug { "Fetched skippable tests: \n #{@skippable_tests}" }
+          Datadog.logger.debug { "ITR correlation ID: #{@correlation_id}" }
+        end
       end
     end
   end

lib/datadog/ci/itr/skippable.rb

@@ -4,24 +4,12 @@
 
 require_relative "../ext/transport"
 require_relative "../ext/test"
+require_relative "../utils/test_run"
 
 module Datadog
   module CI
     module ITR
       class Skippable
-        class Test
-          attr_reader :name, :suite
-
-          def initialize(name:, suite:)
-            @name = name
-            @suite = suite
-          end
-
-          def ==(other)
-            name == other.name && suite == other.suite
-          end
-        end
-
         class Response
           def initialize(http_response)
             @http_response = http_response
@@ -38,13 +26,17 @@ def correlation_id
           end
 
           def tests
+            res = Set.new
+
             payload.fetch("data", [])
-              .filter_map do |test_data|
+              .each do |test_data|
                 next unless test_data["type"] == Ext::Test::ITR_TEST_SKIPPING_MODE
 
                 attrs = test_data["attributes"] || {}
-                Test.new(name: attrs["name"], suite: attrs["suite"])
+                res << Utils::TestRun.test_full_name(attrs["name"], attrs["suite"])
               end
+
+            res
           end
 
           private
@@ -65,7 +57,7 @@ def payload
           end
         end
 
-        def initialize(api: nil, dd_env: nil)
+        def initialize(dd_env:, api: nil)
           @api = api
           @dd_env = dd_env
         end
@@ -98,10 +90,11 @@ def payload(test_session)
                 "repository_url" => test_session.git_repository_url,
                 "sha" => test_session.git_commit_sha,
                 "configurations" => {
-                  "os.platform" => test_session.os_platform,
-                  "os.architecture" => test_session.os_architecture,
-                  "runtime.name" => test_session.runtime_name,
-                  "runtime.version" => test_session.runtime_version
+                  Ext::Test::TAG_OS_PLATFORM => test_session.os_platform,
+                  Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture,
+                  Ext::Test::TAG_OS_VERSION => test_session.os_version,
+                  Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name,
+                  Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version
                 }
               }
             }

lib/datadog/ci/span.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "datadog/core/environment/platform"
+
 require_relative "ext/test"
 require_relative "utils/test_run"
 
@@ -151,6 +153,12 @@ def os_platform
         tracer_span.get_tag(Ext::Test::TAG_OS_PLATFORM)
       end
 
+      # Returns the OS version extracted from the environment.
+      # @return [String] OS version.
+      def os_version
+        tracer_span.get_tag(Ext::Test::TAG_OS_VERSION)
+      end
+
       # Returns the runtime name extracted from the environment.
       # @return [String] runtime name.
       def runtime_name
@@ -166,6 +174,7 @@ def runtime_version
       def set_environment_runtime_tags
         tracer_span.set_tag(Ext::Test::TAG_OS_ARCHITECTURE, ::RbConfig::CONFIG["host_cpu"])
         tracer_span.set_tag(Ext::Test::TAG_OS_PLATFORM, ::RbConfig::CONFIG["host_os"])
+        tracer_span.set_tag(Ext::Test::TAG_OS_VERSION, Core::Environment::Platform.kernel_release)
         tracer_span.set_tag(Ext::Test::TAG_RUNTIME_NAME, Core::Environment::Ext::LANG_ENGINE)
         tracer_span.set_tag(Ext::Test::TAG_RUNTIME_VERSION, Core::Environment::Ext::ENGINE_VERSION)
         tracer_span.set_tag(Ext::Test::TAG_COMMAND, Utils::TestRun.command)

lib/datadog/ci/test_visibility/recorder.rb

@@ -229,7 +229,12 @@ def configure_library(test_session)
               Datadog.logger.debug { "git metadata upload did not complete in time when configuring library" }
             end
           end
-          @itr.configure(remote_configuration.payload, test_session)
+
+          @itr.configure(
+            remote_configuration.payload,
+            test_session: test_session,
+            git_tree_upload_worker: @git_tree_upload_worker
+          )
         end
 
         def skip_tracing(block = nil)

lib/datadog/ci/test_visibility/transport.rb

@@ -14,7 +14,7 @@ class Transport < Datadog::CI::Transport::EventPlatformTransport
 
         def initialize(
           api:,
-          dd_env: nil,
+          dd_env:,
           serializers_factory: Datadog::CI::TestVisibility::Serializers::Factories::TestLevel,
           max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE
         )

lib/datadog/ci/transport/http.rb

@@ -4,6 +4,7 @@
 require "datadog/core/transport/http/adapters/net"
 require "datadog/core/transport/http/env"
 require "datadog/core/transport/request"
+require "socket"
 
 require_relative "gzip"
 require_relative "../ext/transport"
@@ -20,6 +21,8 @@ class HTTP
           :compress
 
         DEFAULT_TIMEOUT = 30
+        MAX_RETRIES = 3
+        INITIAL_BACKOFF = 1
 
         def initialize(host:, timeout: DEFAULT_TIMEOUT, port: nil, ssl: true, compress: false)
           @host = host
@@ -29,7 +32,7 @@ def initialize(host:, timeout: DEFAULT_TIMEOUT, port: nil, ssl: true, compress:
           @compress = compress.nil? ? false : compress
         end
 
-        def request(path:, payload:, headers:, verb: "post")
+        def request(path:, payload:, headers:, verb: "post", retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
           if compress
             headers[Ext::Transport::HEADER_CONTENT_ENCODING] = Ext::Transport::CONTENT_ENCODING_GZIP
             payload = Gzip.compress(payload)
@@ -41,9 +44,7 @@ def request(path:, payload:, headers:, verb: "post")
           end
 
           response = ResponseDecorator.new(
-            adapter.call(
-              build_env(path: path, payload: payload, headers: headers, verb: verb)
-            )
+            perform_http_call(path: path, payload: payload, headers: headers, verb: verb, retries: retries, backoff: backoff)
           )
 
           Datadog.logger.debug do
@@ -55,6 +56,25 @@ def request(path:, payload:, headers:, verb: "post")
 
         private
 
+        def perform_http_call(path:, payload:, headers:, verb:, retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
+          adapter.call(
+            build_env(path: path, payload: payload, headers: headers, verb: verb)
+          )
+        rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, SocketError, Net::HTTPBadResponse => e
+          Datadog.logger.debug("Failed to send request with #{e} (#{e.message})")
+
+          if retries.positive?
+            sleep(backoff)
+
+            perform_http_call(
+              path: path, payload: payload, headers: headers, verb: verb, retries: retries - 1, backoff: backoff * 2
+            )
+          else
+            Datadog.logger.error("Failed to send request after #{MAX_RETRIES} retries")
+            raise e
+          end
+        end
+
         def build_env(path:, payload:, headers:, verb:)
           env = Datadog::Core::Transport::HTTP::Env.new(
             Datadog::Core::Transport::Request.new

lib/datadog/ci/transport/remote_settings_api.rb

@@ -51,7 +51,7 @@ def default_payload
           end
         end
 
-        def initialize(api: nil, dd_env: nil)
+        def initialize(dd_env:, api: nil)
           @api = api
           @dd_env = dd_env
         end
@@ -86,10 +86,11 @@ def payload(test_session)
                 "sha" => test_session.git_commit_sha,
                 "test_level" => Ext::Test::ITR_TEST_SKIPPING_MODE,
                 "configurations" => {
-                  "os.platform" => test_session.os_platform,
-                  "os.arch" => test_session.os_architecture,
-                  "runtime.name" => test_session.runtime_name,
-                  "runtime.version" => test_session.runtime_version
+                  Ext::Test::TAG_OS_PLATFORM => test_session.os_platform,
+                  Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture,
+                  Ext::Test::TAG_OS_VERSION => test_session.os_version,
+                  Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name,
+                  Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version
                 }
               }
             }

lib/datadog/ci/utils/test_run.rb

@@ -9,6 +9,10 @@ def self.command
 
           @command = "#{$0} #{ARGV.join(" ")}"
         end
+
+        def self.test_full_name(test_name, suite)
+          "#{suite}.#{test_name}"
+        end
       end
     end
   end

sig/datadog/ci/ext/test.rbs

@@ -52,6 +52,8 @@ module Datadog
 
         TAG_OS_PLATFORM: "os.platform"
 
+        TAG_OS_VERSION: "os.version"
+
         TAG_RUNTIME_NAME: "runtime.name"
 
         TAG_RUNTIME_VERSION: "runtime.version"

sig/datadog/ci/itr/runner.rbs

@@ -5,12 +5,16 @@ module Datadog
         @enabled: bool
         @test_skipping_enabled: bool
         @code_coverage_enabled: bool
-        @api: Datadog::CI::Transport::Api::Base?
+        @correlation_id: String
+        @skippable_tests: Array[Datadog::CI::ITR::Skippable::Test]
         @coverage_writer: Datadog::CI::ITR::Coverage::Writer?
 
-        def initialize: (?enabled: bool, coverage_writer: Datadog::CI::ITR::Coverage::Writer?) -> void
+        @api: Datadog::CI::Transport::Api::Base?
+        @dd_env: String?
+
+        def initialize: (dd_env: String?, ?enabled: bool, coverage_writer: Datadog::CI::ITR::Coverage::Writer?, api: Datadog::CI::Transport::Api::Base?) -> void
 
-        def configure: (Hash[String, untyped] remote_configuration, Datadog::CI::TestSession test_session) -> void
+        def configure: (Hash[String, untyped] remote_configuration, test_session: Datadog::CI::TestSession, git_tree_upload_worker: Datadog::CI::Worker) -> void
 
         def enabled?: () -> bool
 
@@ -33,6 +37,8 @@ module Datadog
         def write: (Datadog::CI::ITR::Coverage::Event event) -> void
 
         def ensure_test_source_covered: (String test_source_file, Hash[String, untyped] coverage) -> void
+
+        def fetch_skippable_tests: (test_session: Datadog::CI::TestSession, git_tree_upload_worker: Datadog::CI::Worker) -> void
       end
     end
   end

sig/datadog/ci/itr/skippable.rbs

@@ -5,18 +5,6 @@ module Datadog
         @api: Datadog::CI::Transport::Api::Base?
         @dd_env: String?
 
-        class Test
-          @name: String?
-
-          @suite: String?
-
-          attr_reader name: String?
-
-          attr_reader suite: String?
-
-          def initialize: (name: String?, suite: String?) -> void
-        end
-
         class Response
           @http_response: Datadog::Core::Transport::HTTP::Adapters::Net::Response?
           @json: Hash[String, untyped]?
@@ -27,14 +15,14 @@ module Datadog
 
           def correlation_id: () -> String?
 
-          def tests: () -> Array[Test]
+          def tests: () -> Set[String]
 
           private
 
           def payload: () -> Hash[String, untyped]
         end
 
-        def initialize: (?api: Datadog::CI::Transport::Api::Base?, ?dd_env: String?) -> void
+        def initialize: (?api: Datadog::CI::Transport::Api::Base?, dd_env: String?) -> void
 
         def fetch_skippable_tests: (Datadog::CI::TestSession test_session) -> Response
 

sig/datadog/ci/span.rbs

@@ -55,6 +55,8 @@ module Datadog
 
       def os_platform: () -> String?
 
+      def os_version: () -> String?
+
       def runtime_name: () -> String?
 
       def runtime_version: () -> String?