Merge pull request #207 from DataDog/anmarchenko/telemetry_test_sessi…

…on_metric

[SDTEST-415] Add test_session metric

lib/datadog/ci/ext/environment.rb

@@ -23,6 +23,23 @@ module Environment
         TAG_NODE_NAME = "ci.node.name"
         TAG_CI_ENV_VARS = "_dd.ci.env_vars"
 
+        module Provider
+          APPVEYOR = "appveyor"
+          AZURE = "azurepipelines"
+          AWS = "awscodepipeline"
+          BITBUCKET = "bitbucket"
+          BITRISE = "bitrise"
+          BUDDYCI = "buddy"
+          BUILDKITE = "buildkite"
+          CIRCLECI = "circleci"
+          CODEFRESH = "codefresh"
+          GITHUB = "github"
+          GITLAB = "gitlab"
+          JENKINS = "jenkins"
+          TEAMCITY = "teamcity"
+          TRAVISCI = "travisci"
+        end
+
         POSSIBLE_BUNDLE_LOCATIONS = %w[vendor/bundle .bundle].freeze
 
         module_function

lib/datadog/ci/ext/environment/providers/appveyor.rb

@@ -15,7 +15,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "appveyor"
+              Provider::APPVEYOR
             end
 
             def pipeline_url

lib/datadog/ci/ext/environment/providers/aws_code_pipeline.rb

@@ -17,7 +17,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "awscodepipeline"
+              Provider::AWS
             end
 
             def pipeline_id

lib/datadog/ci/ext/environment/providers/azure.rb

@@ -17,7 +17,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "azurepipelines"
+              Provider::AZURE
             end
 
             def pipeline_url

lib/datadog/ci/ext/environment/providers/bitbucket.rb

@@ -16,7 +16,7 @@ def self.handles?(env)
 
             # overridden methods
             def provider_name
-              "bitbucket"
+              Provider::BITBUCKET
             end
 
             def pipeline_id

lib/datadog/ci/ext/environment/providers/bitrise.rb

@@ -15,7 +15,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "bitrise"
+              Provider::BITRISE
             end
 
             def pipeline_id

lib/datadog/ci/ext/environment/providers/buddy.rb

@@ -15,7 +15,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "buddy"
+              Provider::BUDDYCI
             end
 
             def pipeline_id

lib/datadog/ci/ext/environment/providers/buildkite.rb

@@ -17,7 +17,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "buildkite"
+              Provider::BUILDKITE
             end
 
             def job_url

lib/datadog/ci/ext/environment/providers/circleci.rb

@@ -17,7 +17,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "circleci"
+              Provider::CIRCLECI
             end
 
             def job_url

lib/datadog/ci/ext/environment/providers/codefresh.rb

@@ -17,7 +17,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "codefresh"
+              Provider::CODEFRESH
             end
 
             def job_name

lib/datadog/ci/ext/environment/providers/github_actions.rb

@@ -19,7 +19,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "github"
+              Provider::GITHUB
             end
 
             def job_name

lib/datadog/ci/ext/environment/providers/gitlab.rb

@@ -15,7 +15,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "gitlab"
+              Provider::GITLAB
             end
 
             def job_name

lib/datadog/ci/ext/environment/providers/jenkins.rb

@@ -18,7 +18,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "jenkins"
+              Provider::JENKINS
             end
 
             def pipeline_id

lib/datadog/ci/ext/environment/providers/teamcity.rb

@@ -15,7 +15,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "teamcity"
+              Provider::TEAMCITY
             end
 
             def job_name

lib/datadog/ci/ext/environment/providers/travis.rb

@@ -15,7 +15,7 @@ def self.handles?(env)
             end
 
             def provider_name
-              "travisci"
+              Provider::TRAVISCI
             end
 
             def job_url

lib/datadog/ci/ext/telemetry.rb

@@ -110,6 +110,24 @@ module Command
           GET_OBJECTS = "get_objects"
           PACK_OBJECTS = "pack_objects"
         end
+
+        module Provider
+          APPVEYOR = "appveyor"
+          AWS = "aws"
+          AZURE = "azp"
+          BITBUCKET = "bitbucket"
+          BITRISE = "bitrise"
+          BUDDYCI = "buddyci"
+          BUILDKITE = "buildkite"
+          CIRCLECI = "circleci"
+          CODEFRESH = "codefresh"
+          GITHUB = "githubactions"
+          GITLAB = "gitlab"
+          JENKINS = "jenkins"
+          TEAMCITY = "teamcity"
+          TRAVISCI = "travisci"
+          UNSUPPORTED = "unsupported"
+        end
       end
     end
   end

lib/datadog/ci/test_visibility/component.rb

@@ -144,6 +144,7 @@ def itr_enabled?
         # DOMAIN EVENTS
         def on_test_session_started(test_session)
           Telemetry.event_created(test_session)
+          Telemetry.test_session_started(test_session)
 
           # finds and instruments additional test libraries that we support (ex: selenium-webdriver)
           Contrib.auto_instrument_on_session_start!

lib/datadog/ci/test_visibility/telemetry.rb

@@ -18,6 +18,23 @@ module Telemetry
           Ext::AppTypes::TYPE_TEST_SESSION => Ext::Telemetry::EventType::SESSION
         }.freeze
 
+        PROVIDER_TAG_TO_TELEMETRY_PROVIDER_TAG = {
+          Ext::Environment::Provider::APPVEYOR => Ext::Telemetry::Provider::APPVEYOR,
+          Ext::Environment::Provider::AWS => Ext::Telemetry::Provider::AWS,
+          Ext::Environment::Provider::AZURE => Ext::Telemetry::Provider::AZURE,
+          Ext::Environment::Provider::BITBUCKET => Ext::Telemetry::Provider::BITBUCKET,
+          Ext::Environment::Provider::BITRISE => Ext::Telemetry::Provider::BITRISE,
+          Ext::Environment::Provider::BUDDYCI => Ext::Telemetry::Provider::BUDDYCI,
+          Ext::Environment::Provider::BUILDKITE => Ext::Telemetry::Provider::BUILDKITE,
+          Ext::Environment::Provider::CIRCLECI => Ext::Telemetry::Provider::CIRCLECI,
+          Ext::Environment::Provider::CODEFRESH => Ext::Telemetry::Provider::CODEFRESH,
+          Ext::Environment::Provider::GITHUB => Ext::Telemetry::Provider::GITHUB,
+          Ext::Environment::Provider::GITLAB => Ext::Telemetry::Provider::GITLAB,
+          Ext::Environment::Provider::JENKINS => Ext::Telemetry::Provider::JENKINS,
+          Ext::Environment::Provider::TEAMCITY => Ext::Telemetry::Provider::TEAMCITY,
+          Ext::Environment::Provider::TRAVISCI => Ext::Telemetry::Provider::TRAVISCI
+        }.freeze
+
         def self.event_created(span)
           Utils::Telemetry.inc(Ext::Telemetry::METRIC_EVENT_CREATED, 1, event_tags_from_span(span))
         end
@@ -28,6 +45,20 @@ def self.event_finished(span)
           Utils::Telemetry.inc(Ext::Telemetry::METRIC_EVENT_FINISHED, 1, tags)
         end
 
+        def self.test_session_started(test_session)
+          Utils::Telemetry.inc(
+            Ext::Telemetry::METRIC_TEST_SESSION,
+            1,
+            {
+              Ext::Telemetry::TAG_AUTO_INJECTED => "false", # ruby doesn't support auto injection yet
+              Ext::Telemetry::TAG_PROVIDER => PROVIDER_TAG_TO_TELEMETRY_PROVIDER_TAG.fetch(
+                test_session.get_tag(Ext::Environment::TAG_PROVIDER_NAME),
+                Ext::Telemetry::Provider::UNSUPPORTED
+              )
+            }
+          )
+        end
+
         def self.event_tags_from_span(span)
           # base tags for span
           # @type var tags: Hash[String, String]

sig/datadog/ci/ext/environment.rbs

@@ -31,6 +31,23 @@ module Datadog
 
         PROVIDERS: ::Array[Array[String | Symbol]]
 
+        module Provider
+          APPVEYOR: "appveyor"
+          AWS: "awscodepipeline"
+          AZURE: "azurepipelines"
+          BITBUCKET: "bitbucket"
+          BITRISE: "bitrise"
+          BUDDYCI: "buddy"
+          BUILDKITE: "buildkite"
+          CIRCLECI: "circleci"
+          CODEFRESH: "codefresh"
+          GITHUB: "github"
+          GITLAB: "gitlab"
+          JENKINS: "jenkins"
+          TEAMCITY: "teamcity"
+          TRAVISCI: "travisci"
+        end
+
         def self?.tags: (untyped env) -> Hash[String, String]
 
         def self?.ensure_post_conditions: (Hash[String, String] tags) -> void

sig/datadog/ci/ext/telemetry.rbs

@@ -162,6 +162,24 @@ module Datadog
 
           PACK_OBJECTS: "pack_objects"
         end
+
+        module Provider
+          APPVEYOR: "appveyor"
+          AWS: "aws"
+          AZURE: "azp"
+          BITBUCKET: "bitbucket"
+          BITRISE: "bitrise"
+          BUDDYCI: "buddyci"
+          BUILDKITE: "buildkite"
+          CIRCLECI: "circleci"
+          CODEFRESH: "codefresh"
+          GITHUB: "githubactions"
+          GITLAB: "gitlab"
+          JENKINS: "jenkins"
+          TEAMCITY: "teamcity"
+          TRAVISCI: "travisci"
+          UNSUPPORTED: "unsupported"
+        end
       end
     end
   end

sig/datadog/ci/test_visibility/telemetry.rbs

@@ -4,10 +4,14 @@ module Datadog
       module Telemetry
         SPAN_TYPE_TO_TELEMETRY_EVENT_TYPE: ::Hash[String, String]
 
+        PROVIDER_TAG_TO_TELEMETRY_PROVIDER_TAG: ::Hash[String, String]
+
         def self.event_created: (Datadog::CI::Span span) -> void
 
         def self.event_finished: (Datadog::CI::Span span) -> void
 
+        def self.test_session_started: (Datadog::CI::TestSession span) -> void
+
         def self.event_tags_from_span: (Datadog::CI::Span span) -> ::Hash[String, String]
 
         def self.add_browser_tags!: (Datadog::CI::Span span, ::Hash[String, String] tags) -> void

spec/datadog/ci/test_visibility/component_spec.rb

@@ -436,6 +436,7 @@
           end
         end
         it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_EVENT_CREATED
+        it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_TEST_SESSION
       end
 
       describe "#start_test_module" do

spec/datadog/ci/test_visibility/telemetry_spec.rb

@@ -206,4 +206,39 @@
       it { event_finished }
     end
   end
+
+  describe ".test_session_started" do
+    subject(:test_session_started) { described_class.test_session_started(test_session) }
+
+    let(:provider_tag) { "github" }
+    let(:expected_provider_telemetry_tag) { Datadog::CI::Ext::Telemetry::Provider::GITHUB }
+
+    let(:test_session) do
+      instance_double(
+        Datadog::CI::TestSession,
+        get_tag: provider_tag
+      )
+    end
+
+    before do
+      expect(Datadog::CI::Utils::Telemetry).to receive(:inc)
+        .with(
+          Datadog::CI::Ext::Telemetry::METRIC_TEST_SESSION,
+          1,
+          {
+            Datadog::CI::Ext::Telemetry::TAG_AUTO_INJECTED => "false",
+            Datadog::CI::Ext::Telemetry::TAG_PROVIDER => expected_provider_telemetry_tag
+          }
+        )
+    end
+
+    it { test_session_started }
+
+    context "when provider is not supported" do
+      let(:provider_tag) { nil }
+      let(:expected_provider_telemetry_tag) { Datadog::CI::Ext::Telemetry::Provider::UNSUPPORTED }
+
+      it { test_session_started }
+    end
+  end
 end