Merge pull request #123 from DataDog/anmarchenko/evp_proxy_v4_support

[CIVIS-8782] gzip agent payloads support via evp_proxy/v4

lib/datadog/ci/configuration/components.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "datadog/core/configuration/agent_settings_resolver"
-require "datadog/core/remote/negotiation"
-
-require_relative "../ext/transport"
 require_relative "../ext/settings"
 require_relative "../test_visibility/flush"
 require_relative "../test_visibility/recorder"
@@ -32,18 +28,7 @@ def initialize(settings)
         end
 
         def activate_ci!(settings)
-          test_visibility_transport = nil
-          agent_settings = Datadog::Core::Configuration::AgentSettingsResolver.call(settings)
-
-          if settings.ci.agentless_mode_enabled
-            check_dd_site(settings)
-            test_visibility_transport = build_agentless_transport(settings)
-          elsif can_use_evp_proxy?(settings, agent_settings)
-            test_visibility_transport = build_evp_proxy_transport(settings, agent_settings)
-          else
-            settings.ci.force_test_level_visibility = true
-          end
-
+          # Configure ddtrace library for CI visibility mode
           # Deactivate telemetry
           settings.telemetry.enabled = false
 
@@ -60,9 +45,16 @@ def activate_ci!(settings)
           # Choose user defined TraceFlush or default to CI TraceFlush
           settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Partial.new
 
+          # transport creation
           writer_options = settings.ci.writer_options
-          if test_visibility_transport
-            writer_options[:transport] = test_visibility_transport
+          test_visibility_api = build_test_visibility_api(settings)
+
+          if test_visibility_api
+            writer_options[:transport] = Datadog::CI::TestVisibility::Transport.new(
+              api: test_visibility_api,
+              serializers_factory: serializers_factory(settings),
+              dd_env: settings.env
+            )
             writer_options[:shutdown_timeout] = 60
             writer_options[:buffer_size] = 10_000
 
@@ -71,51 +63,45 @@ def activate_ci!(settings)
 
           settings.tracing.test_mode.writer_options = writer_options
 
+          # CI visibility recorder global instance
           @ci_recorder = TestVisibility::Recorder.new(
             test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility
           )
         end
 
-        def can_use_evp_proxy?(settings, agent_settings)
-          Datadog::Core::Remote::Negotiation.new(settings, agent_settings).endpoint?(
-            Ext::Transport::EVP_PROXY_PATH_PREFIX
-          )
-        end
+        def build_test_visibility_api(settings)
+          if settings.ci.agentless_mode_enabled
+            check_dd_site(settings)
 
-        def build_agentless_transport(settings)
-          if settings.api_key.nil?
-            # agentless mode is requested but no API key is provided -
-            # we cannot continue and log an error
-            # Tests are running without CI visibility enabled
+            Datadog.logger.debug("CI visibility configured to use agentless transport")
 
-            Datadog.logger.error(
-              "DATADOG CONFIGURATION - CI VISIBILITY - ATTENTION - " \
-              "Agentless mode was enabled but DD_API_KEY is not set: CI visibility is disabled. " \
-              "Please make sure to set valid api key in DD_API_KEY environment variable"
-            )
+            api = Transport::Api::Builder.build_agentless_api(settings)
+            if api.nil?
+              Datadog.logger.error do
+                "DATADOG CONFIGURATION - CI VISIBILITY - ATTENTION - " \
+                "Agentless mode was enabled but DD_API_KEY is not set: CI visibility is disabled. " \
+                "Please make sure to set valid api key in DD_API_KEY environment variable"
+              end
 
-            settings.ci.enabled = false
+              # Tests are running without CI visibility enabled
+              settings.ci.enabled = false
+            end
 
-            nil
           else
-            Datadog.logger.debug("CI visibility configured to use agentless transport")
+            Datadog.logger.debug("CI visibility configured to use agent transport via EVP proxy")
 
-            Datadog::CI::TestVisibility::Transport.new(
-              api: Transport::Api::Builder.build_ci_test_cycle_api(settings),
-              serializers_factory: serializers_factory(settings),
-              dd_env: settings.env
-            )
-          end
-        end
+            api = Transport::Api::Builder.build_evp_proxy_api(settings)
+            if api.nil?
+              Datadog.logger.debug(
+                "Old agent version detected, no evp_proxy support. Forcing test level visibility mode"
+              )
 
-        def build_evp_proxy_transport(settings, agent_settings)
-          Datadog.logger.debug("CI visibility configured to use agent transport via EVP proxy")
+              # CI visibility is still enabled but in legacy test level visibility mode
+              settings.ci.force_test_level_visibility = true
+            end
+          end
 
-          Datadog::CI::TestVisibility::Transport.new(
-            api: Transport::Api::Builder.build_evp_proxy_api(agent_settings),
-            serializers_factory: serializers_factory(settings),
-            dd_env: settings.env
-          )
+          api
         end
 
         def serializers_factory(settings)
@@ -130,11 +116,11 @@ def check_dd_site(settings)
           return if settings.site.nil?
           return if Ext::Settings::DD_SITE_ALLOWLIST.include?(settings.site)
 
-          Datadog.logger.warn(
+          Datadog.logger.warn do
             "CI VISIBILITY CONFIGURATION " \
             "Agentless mode was enabled but DD_SITE is not set to one of the following: #{Ext::Settings::DD_SITE_ALLOWLIST.join(", ")}. " \
             "Please make sure to set valid site in DD_SITE environment variable"
-          )
+          end
         end
       end
     end

lib/datadog/ci/ext/transport.rb

@@ -12,7 +12,14 @@ module Transport
         HEADER_EVP_SUBDOMAIN = "X-Datadog-EVP-Subdomain"
         HEADER_CONTAINER_ID = "Datadog-Container-ID"
 
-        EVP_PROXY_PATH_PREFIX = "/evp_proxy/v2/"
+        EVP_PROXY_V2_PATH_PREFIX = "/evp_proxy/v2/"
+        EVP_PROXY_V4_PATH_PREFIX = "/evp_proxy/v4/"
+        EVP_PROXY_PATH_PREFIXES = [EVP_PROXY_V4_PATH_PREFIX, EVP_PROXY_V2_PATH_PREFIX].freeze
+        EVP_PROXY_COMPRESSION_SUPPORTED = {
+          EVP_PROXY_V4_PATH_PREFIX => true,
+          EVP_PROXY_V2_PATH_PREFIX => false
+        }
+
         TEST_VISIBILITY_INTAKE_HOST_PREFIX = "citestcycle-intake"
         TEST_VISIBILITY_INTAKE_PATH = "/api/v2/citestcycle"
 

lib/datadog/ci/transport/api/ci_test_cycle.rb → lib/datadog/ci/transport/api/agentless.rb

@@ -7,7 +7,7 @@ module Datadog
   module CI
     module Transport
       module Api
-        class CiTestCycle < Base
+        class Agentless < Base
           attr_reader :api_key
 
           def initialize(api_key:, http:)

lib/datadog/ci/transport/api/builder.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
-require_relative "ci_test_cycle"
+require "datadog/core/configuration/agent_settings_resolver"
+require "datadog/core/remote/negotiation"
+
+require_relative "agentless"
 require_relative "evp_proxy"
 require_relative "../http"
 require_relative "../../ext/transport"
@@ -10,7 +13,9 @@ module CI
     module Transport
       module Api
         module Builder
-          def self.build_ci_test_cycle_api(settings)
+          def self.build_agentless_api(settings)
+            return nil if settings.api_key.nil?
+
             dd_site = settings.site || Ext::Transport::DEFAULT_DD_SITE
             url = settings.ci.agentless_url ||
               "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
@@ -25,19 +30,31 @@ def self.build_ci_test_cycle_api(settings)
               compress: true
             )
 
-            CiTestCycle.new(api_key: settings.api_key, http: http)
+            Agentless.new(api_key: settings.api_key, http: http)
           end
 
-          def self.build_evp_proxy_api(agent_settings)
+          def self.build_evp_proxy_api(settings)
+            agent_settings = Datadog::Core::Configuration::AgentSettingsResolver.call(settings)
+            negotiation = Datadog::Core::Remote::Negotiation.new(settings, agent_settings)
+
+            # temporary, remove this when patch will be accepted in Core to make logging configurable
+            negotiation.instance_variable_set(:@logged, {no_config_endpoint: true})
+
+            evp_proxy_path_prefix = Ext::Transport::EVP_PROXY_PATH_PREFIXES.find do |path_prefix|
+              negotiation.endpoint?(path_prefix)
+            end
+
+            return nil if evp_proxy_path_prefix.nil?
+
             http = Datadog::CI::Transport::HTTP.new(
               host: agent_settings.hostname,
               port: agent_settings.port,
               ssl: agent_settings.ssl,
               timeout: agent_settings.timeout_seconds,
-              compress: false
+              compress: Ext::Transport::EVP_PROXY_COMPRESSION_SUPPORTED[evp_proxy_path_prefix]
             )
 
-            EvpProxy.new(http: http)
+            EvpProxy.new(http: http, path_prefix: evp_proxy_path_prefix)
           end
         end
       end

lib/datadog/ci/transport/api/evp_proxy.rb

@@ -10,8 +10,15 @@ module CI
     module Transport
       module Api
         class EvpProxy < Base
+          def initialize(http:, path_prefix: Ext::Transport::EVP_PROXY_V2_PATH_PREFIX)
+            super(http: http)
+
+            path_prefix = "#{path_prefix}/" unless path_prefix.end_with?("/")
+            @path_prefix = path_prefix
+          end
+
           def request(path:, payload:, verb: "post")
-            path = "#{Ext::Transport::EVP_PROXY_PATH_PREFIX}#{path.sub(/^\//, "")}"
+            path = "#{@path_prefix}#{path.sub(/^\//, "")}"
 
             super(
               path: path,

sig/datadog/ci/configuration/components.rbs

@@ -10,10 +10,10 @@ module Datadog
 
         def activate_ci!: (untyped settings) -> untyped
 
-        def build_agentless_transport: (untyped settings) -> Datadog::CI::TestVisibility::Transport?
-        def build_evp_proxy_transport: (untyped settings, untyped agent_settings) -> Datadog::CI::TestVisibility::Transport
-        def can_use_evp_proxy?: (untyped settings, untyped agent_settings) -> bool
+        def build_test_visibility_api: (untyped settings) -> Datadog::CI::Transport::Api::Base?
+
         def serializers_factory: (untyped settings) -> (singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel))
+
         def check_dd_site: (untyped settings) -> void
       end
     end

sig/datadog/ci/ext/transport.rbs

@@ -14,7 +14,13 @@ module Datadog
 
         HEADER_CONTAINER_ID: "Datadog-Container-ID"
 
-        EVP_PROXY_PATH_PREFIX: "/evp_proxy/v2/"
+        EVP_PROXY_V2_PATH_PREFIX: "/evp_proxy/v2/"
+
+        EVP_PROXY_V4_PATH_PREFIX: "/evp_proxy/v4/"
+
+        EVP_PROXY_PATH_PREFIXES: ::Array[String]
+
+        EVP_PROXY_COMPRESSION_SUPPORTED: ::Hash[String, bool]
 
         TEST_VISIBILITY_INTAKE_HOST_PREFIX: "citestcycle-intake"
 

sig/datadog/ci/transport/api/ci_test_cycle.rbs → sig/datadog/ci/transport/api/agentless.rbs

@@ -2,7 +2,7 @@ module Datadog
   module CI
     module Transport
       module Api
-        class CiTestCycle < Base
+        class Agentless < Base
           attr_reader api_key: String
 
           @api_key: String

sig/datadog/ci/transport/api/builder.rbs

@@ -3,8 +3,8 @@ module Datadog
     module Transport
       module Api
         module Builder
-          def self.build_ci_test_cycle_api: (untyped settings) -> Datadog::CI::Transport::Api::CiTestCycle
-          def self.build_evp_proxy_api: (untyped agent_settings) -> Datadog::CI::Transport::Api::EvpProxy
+          def self.build_agentless_api: (untyped settings) -> Datadog::CI::Transport::Api::Agentless?
+          def self.build_evp_proxy_api: (untyped agent_settings) -> Datadog::CI::Transport::Api::EvpProxy?
         end
       end
     end

sig/datadog/ci/transport/api/evp_proxy.rbs

@@ -4,6 +4,9 @@ module Datadog
       module Api
         class EvpProxy < Base
           @container_id: String?
+          @path_prefix: String
+
+          def initialize: (http: Datadog::CI::Transport::HTTP, ?path_prefix: String) -> void
 
           def request: (path: String, payload: String, ?verb: ::String) -> Datadog::CI::Transport::HTTP::ResponseDecorator
 

spec/datadog/ci/configuration/components_spec.rb

@@ -64,9 +64,13 @@
             .to receive(:new)
             .and_return(negotiation)
 
+          allow(negotiation)
+            .to receive(:endpoint?).with("/evp_proxy/v4/")
+            .and_return(evp_proxy_v4_supported)
+
           allow(negotiation)
             .to receive(:endpoint?).with("/evp_proxy/v2/")
-            .and_return(evp_proxy_supported)
+            .and_return(evp_proxy_v2_supported)
 
           # Spy on test mode behavior
           allow(settings.tracing.test_mode)
@@ -104,7 +108,8 @@
         let(:dd_site) { nil }
         let(:agentless_enabled) { false }
         let(:force_test_level_visibility) { false }
-        let(:evp_proxy_supported) { false }
+        let(:evp_proxy_v2_supported) { false }
+        let(:evp_proxy_v4_supported) { false }
 
         context "is enabled" do
           let(:enabled) { true }
@@ -135,8 +140,8 @@
             context "is disabled" do
               let(:agentless_enabled) { false }
 
-              context "and when agent supports EVP proxy" do
-                let(:evp_proxy_supported) { true }
+              context "and when agent supports EVP proxy v2" do
+                let(:evp_proxy_v2_supported) { true }
 
                 it "sets async for test mode and constructs transport with EVP proxy API" do
                   expect(settings.tracing.test_mode)
@@ -151,9 +156,23 @@
                 end
               end
 
-              context "and when agent does not support EVP proxy" do
-                let(:evp_proxy_supported) { false }
+              context "and when agent supports EVP proxy v4" do
+                let(:evp_proxy_v4_supported) { true }
+
+                it "sets async for test mode and constructs transport with EVP proxy API" do
+                  expect(settings.tracing.test_mode)
+                    .to have_received(:async=)
+                    .with(true)
+
+                  expect(settings.tracing.test_mode).to have_received(:writer_options=) do |options|
+                    expect(options[:transport]).to be_kind_of(Datadog::CI::TestVisibility::Transport)
+                    expect(options[:transport].api).to be_kind_of(Datadog::CI::Transport::Api::EvpProxy)
+                    expect(options[:shutdown_timeout]).to eq(60)
+                  end
+                end
+              end
 
+              context "and when agent does not support EVP proxy" do
                 it "falls back to default transport and disables test suite level visibility" do
                   expect(settings.tracing.test_mode)
                     .to have_received(:enabled=)
@@ -190,7 +209,7 @@
 
                   expect(settings.tracing.test_mode).to have_received(:writer_options=) do |options|
                     expect(options[:transport]).to be_kind_of(Datadog::CI::TestVisibility::Transport)
-                    expect(options[:transport].api).to be_kind_of(Datadog::CI::Transport::Api::CiTestCycle)
+                    expect(options[:transport].api).to be_kind_of(Datadog::CI::Transport::Api::Agentless)
                     expect(options[:shutdown_timeout]).to eq(60)
                   end
                 end
@@ -200,9 +219,11 @@
                 let(:dd_site) { "wrong" }
 
                 it "logs a warning" do
-                  expect(Datadog.logger).to have_received(:warn).with(
-                    /CI VISIBILITY CONFIGURATION Agentless mode was enabled but DD_SITE is not set to one of the following/
-                  )
+                  expect(Datadog.logger).to have_received(:warn) do |*_args, &block|
+                    expect(block.call).to match(
+                      /CI VISIBILITY CONFIGURATION Agentless mode was enabled but DD_SITE is not set to one of the following/
+                    )
+                  end
                 end
               end
 

spec/datadog/ci/transport/api/ci_test_cycle_spec.rb → spec/datadog/ci/transport/api/agentless_spec.rb

@@ -1,6 +1,6 @@
-require_relative "../../../../../lib/datadog/ci/transport/api/ci_test_cycle"
+require_relative "../../../../../lib/datadog/ci/transport/api/agentless"
 
-RSpec.describe Datadog::CI::Transport::Api::CiTestCycle do
+RSpec.describe Datadog::CI::Transport::Api::Agentless do
   subject do
     described_class.new(
       api_key: api_key,