Merge pull request #270 from DataDog/anmarchenko/add_more_telemetry

[SDTEST-1293] Internal: improve libraries telemetry

lib/datadog/ci/ext/telemetry.rb

@@ -69,6 +69,7 @@ module Telemetry
         TAG_BROWSER_DRIVER = "browser_driver"
         TAG_IS_RUM = "is_rum"
         TAG_IS_RETRY = "is_retry"
+        TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason"
         TAG_IS_NEW = "is_new"
         TAG_LIBRARY = "library"
         TAG_ENDPOINT = "endpoint"
@@ -78,12 +79,18 @@ module Telemetry
         TAG_REQUEST_COMPRESSED = "rq_compressed"
         TAG_RESPONSE_COMPRESSED = "rs_compressed"
         TAG_COMMAND = "command"
+        # tags for git_requests.settings_response metric
         TAG_COVERAGE_ENABLED = "coverage_enabled"
+        TAG_ITR_ENABLED = "itr_enabled"
         TAG_ITR_SKIP_ENABLED = "itrskip_enabled"
+        TAG_REQUIRE_GIT = "require_git"
         TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled"
-        TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason"
+        TAG_FLAKY_TEST_RETRIES_ENABLED = "flaky_test_retries_enabled"
+        # tags for test_session metric
         TAG_PROVIDER = "provider"
         TAG_AUTO_INJECTED = "auto_injected"
+        TAG_AGENTLESS_LOG_SUBMISSION_ENABLED = "agentless_log_submission_enabled"
+        TAG_FAIL_FAST_TEST_ORDER_ENABLED = "fail_fast_test_order_enabled"
 
         module EventType
           TEST = "test"

lib/datadog/ci/remote/library_settings_client.rb

@@ -59,7 +59,10 @@ def fetch(test_session)
             {
               Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s,
               Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s,
-              Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s
+              Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s,
+              Ext::Telemetry::TAG_FLAKY_TEST_RETRIES_ENABLED => library_settings.flaky_test_retries_enabled?.to_s,
+              Ext::Telemetry::TAG_ITR_ENABLED => library_settings.itr_enabled?.to_s,
+              Ext::Telemetry::TAG_REQUIRE_GIT => library_settings.require_git?.to_s
             }
           )
 

sig/datadog/ci/ext/telemetry.rbs

@@ -126,16 +126,26 @@ module Datadog
 
         TAG_COVERAGE_ENABLED: "coverage_enabled"
 
+        TAG_ITR_ENABLED: "itr_enabled"
+
         TAG_ITR_SKIP_ENABLED: "itrskip_enabled"
 
+        TAG_REQUIRE_GIT: "require_git"
+
         TAG_EARLY_FLAKE_DETECTION_ENABLED: "early_flake_detection_enabled"
 
+        TAG_FLAKY_TEST_RETRIES_ENABLED: "flaky_test_retries_enabled"
+
         TAG_EARLY_FLAKE_DETECTION_ABORT_REASON: "early_flake_detection_abort_reason"
 
         TAG_PROVIDER: "provider"
 
         TAG_AUTO_INJECTED: "auto_injected"
 
+        TAG_AGENTLESS_LOG_SUBMISSION_ENABLED: "agentless_log_submission_enabled"
+
+        TAG_FAIL_FAST_TEST_ORDER_ENABLED: "fail_fast_test_order_enabled"
+
         module EventType
           TEST: "test"
           SUITE: "suite"

spec/datadog/ci/remote/library_settings_client_spec.rb

@@ -5,14 +5,13 @@
 RSpec.describe Datadog::CI::Remote::LibrarySettingsClient do
   include_context "Telemetry spy"
 
-  let(:api) { spy("api") }
   let(:dd_env) { "ci" }
   let(:config_tags) { {} }
 
   subject(:client) { described_class.new(api: api, dd_env: dd_env, config_tags: config_tags) }
 
   describe "#fetch" do
-    subject { client.fetch(test_session) }
+    subject(:response) { client.fetch(test_session) }
 
     let(:service) { "service" }
     let(:tracer_span) do
@@ -31,82 +30,82 @@
     end
     let(:test_session) { Datadog::CI::TestSession.new(tracer_span) }
 
-    let(:path) { Datadog::CI::Ext::Transport::DD_API_SETTINGS_PATH }
+    context "when api is present" do
+      let(:api) { spy("api") }
+      before do
+        allow(api).to receive(:api_request).and_return(http_response)
+      end
 
-    it "requests the settings" do
-      subject
+      let(:attributes) do
+        {
+          "code_coverage" => "1",
+          "tests_skipping" => "false",
+          "itr_enabled" => "True",
+          "require_git" => require_git,
+          "flaky_test_retries_enabled" => "true",
+          "early_flake_detection" => {
+            "enabled" => "true",
+            "slow_test_retries" => {
+              "5s" => 10,
+              "10s" => 5,
+              "30s" => 3,
+              "5m" => 2
+            },
+            "faulty_session_threshold" => 30
+          }
+        }
+      end
 
-      expect(api).to have_received(:api_request) do |args|
-        expect(args[:path]).to eq(path)
+      let(:http_response) do
+        double(
+          "http_response",
+          ok?: true,
+          payload: {
+            "data" => {
+              "id" => "123",
+              "type" => Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE,
+              "attributes" => attributes
+            }
+          }.to_json,
+          request_compressed: false,
+          duration_ms: 1.2
+        )
+      end
 
-        data = JSON.parse(args[:payload])["data"]
+      let(:require_git) { false }
 
-        expect(data["id"]).to eq(Datadog::Core::Environment::Identity.id)
-        expect(data["type"]).to eq(Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE)
+      let(:path) { Datadog::CI::Ext::Transport::DD_API_SETTINGS_PATH }
 
-        attributes = data["attributes"]
-        expect(attributes["service"]).to eq(service)
-        expect(attributes["env"]).to eq(dd_env)
-        expect(attributes["test_level"]).to eq("test")
-        expect(attributes["repository_url"]).to eq("repository_url")
-        expect(attributes["branch"]).to eq("branch")
-        expect(attributes["sha"]).to eq("commit_sha")
+      it "requests the settings" do
+        subject
 
-        configurations = attributes["configurations"]
-        expect(configurations["os.platform"]).to eq("platform")
-        expect(configurations["os.architecture"]).to eq("arch")
-        expect(configurations["os.version"]).to eq("version")
-        expect(configurations["runtime.name"]).to eq("runtime_name")
-        expect(configurations["runtime.version"]).to eq("runtime_version")
-      end
-    end
+        expect(api).to have_received(:api_request) do |args|
+          expect(args[:path]).to eq(path)
 
-    context "parsing response" do
-      subject(:response) { client.fetch(test_session) }
+          data = JSON.parse(args[:payload])["data"]
 
-      context "when api is present" do
-        before do
-          allow(api).to receive(:api_request).and_return(http_response)
-        end
+          expect(data["id"]).to eq(Datadog::Core::Environment::Identity.id)
+          expect(data["type"]).to eq(Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE)
 
-        context "when response is OK" do
-          let(:attributes) do
-            {
-              "code_coverage" => "1",
-              "tests_skipping" => "false",
-              "itr_enabled" => "True",
-              "require_git" => require_git,
-              "flaky_test_retries_enabled" => "true",
-              "early_flake_detection" => {
-                "enabled" => "true",
-                "slow_test_retries" => {
-                  "5s" => 10,
-                  "10s" => 5,
-                  "30s" => 3,
-                  "5m" => 2
-                },
-                "faulty_session_threshold" => 30
-              }
-            }
-          end
+          attributes = data["attributes"]
+          expect(attributes["service"]).to eq(service)
+          expect(attributes["env"]).to eq(dd_env)
+          expect(attributes["test_level"]).to eq("test")
+          expect(attributes["repository_url"]).to eq("repository_url")
+          expect(attributes["branch"]).to eq("branch")
+          expect(attributes["sha"]).to eq("commit_sha")
 
-          let(:http_response) do
-            double(
-              "http_response",
-              ok?: true,
-              payload: {
-                "data" => {
-                  "id" => "123",
-                  "type" => Datadog::CI::Ext::Transport::DD_API_SETTINGS_TYPE,
-                  "attributes" => attributes
-                }
-              }.to_json,
-              request_compressed: false,
-              duration_ms: 1.2
-            )
-          end
-          let(:require_git) { false }
+          configurations = attributes["configurations"]
+          expect(configurations["os.platform"]).to eq("platform")
+          expect(configurations["os.architecture"]).to eq("arch")
+          expect(configurations["os.version"]).to eq("version")
+          expect(configurations["runtime.name"]).to eq("runtime_name")
+          expect(configurations["runtime.version"]).to eq("runtime_version")
+        end
+      end
 
+      context "parsing response" do
+        context "when response is OK" do
           it "parses the response" do
             expect(response.ok?).to be true
             expect(response.payload).to eq(attributes)
@@ -128,7 +127,12 @@
 
             metric = telemetry_metric(:inc, "git_requests.settings_response")
             expect(metric.tags).to eq(
-              "coverage_enabled" => "true", "itrskip_enabled" => "false", "early_flake_detection_enabled" => "true"
+              "coverage_enabled" => "true",
+              "itr_enabled" => "true",
+              "itrskip_enabled" => "false",
+              "early_flake_detection_enabled" => "true",
+              "flaky_test_retries_enabled" => "true",
+              "require_git" => "false"
             )
           end
 
@@ -143,6 +147,22 @@
               expect(response.require_git?).to be true
             end
           end
+
+          context "with custom configuration" do
+            let(:config_tags) { {"key" => "value"} }
+
+            it "requests the settings" do
+              subject
+
+              expect(api).to have_received(:api_request) do |args|
+                data = JSON.parse(args[:payload])["data"]
+
+                attributes = data["attributes"]
+                configurations = attributes["configurations"]
+                expect(configurations["custom"]).to eq("key" => "value")
+              end
+            end
+          end
         end
 
         context "when response is not OK" do
@@ -218,31 +238,15 @@
           end
         end
       end
-
-      context "when there is no api" do
-        let(:api) { nil }
-
-        it "returns an empty response" do
-          expect(response.ok?).to be false
-          expect(response.payload).to eq("itr_enabled" => false)
-          expect(response.require_git?).to be false
-        end
-      end
     end
 
-    context "with custom configuration" do
-      let(:config_tags) { {"key" => "value"} }
-
-      it "requests the settings" do
-        subject
-
-        expect(api).to have_received(:api_request) do |args|
-          data = JSON.parse(args[:payload])["data"]
+    context "when there is no api" do
+      let(:api) { nil }
 
-          attributes = data["attributes"]
-          configurations = attributes["configurations"]
-          expect(configurations["custom"]).to eq("key" => "value")
-        end
+      it "returns an empty response" do
+        expect(response.ok?).to be false
+        expect(response.payload).to eq("itr_enabled" => false)
+        expect(response.require_git?).to be false
       end
     end
   end