Merge pull request #106 from DataDog/anmarchenko/manual_api_cleaning_up

[CIVIS-8333] better handling of test.type and span.kind tags

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -37,8 +37,7 @@ def on_test_run_started(event)
             test_session = CI.start_test_session(
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s
               },
               service: configuration[:service_name]
             )
@@ -59,7 +58,6 @@ def on_test_case_started(event)
             tags = {
               CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
               CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-              CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
               CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file),
               CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s
             }

lib/datadog/ci/contrib/minitest/hooks.rb

@@ -30,7 +30,6 @@ def before_setup
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
                 CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(source_file),
                 CI::Ext::Test::TAG_SOURCE_START => line_number.to_s
               },

lib/datadog/ci/contrib/minitest/plugin.rb

@@ -50,8 +50,7 @@ def plugin_datadog_ci_init(*)
               test_session = CI.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
                 },
                 service: datadog_configuration[:service_name]
               )

lib/datadog/ci/contrib/rspec/example.rb

@@ -36,7 +36,6 @@ def run(*)
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
                   CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(metadata[:file_path]),
                   CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
                 },

lib/datadog/ci/contrib/rspec/runner.rb

@@ -20,8 +20,7 @@ def run_specs(*)
               test_session = CI.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
                 },
                 service: datadog_configuration[:service_name]
               )

lib/datadog/ci/ext/test.rb

@@ -23,30 +23,38 @@ module Test
         TAG_CODEOWNERS = "test.codeowners"
         TAG_PARAMETERS = "test.parameters"
 
-        TEST_TYPE = "test"
-
-        # those tags are special and they are used to correlate tests with the test sessions, suites, and modules
+        # those tags are special and used to correlate tests with the test sessions, suites, and modules
+        # they are transient and not sent to the backend
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"
         TAG_TEST_SUITE_ID = "_test.suite_id"
-        SPECIAL_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID, TAG_TEST_SUITE_ID].freeze
+        TRANSIENT_TAGS = [TAG_TEST_SESSION_ID, TAG_TEST_MODULE_ID, TAG_TEST_SUITE_ID].freeze
 
-        # tags that can be inherited from the test session
-        INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze
+        # tags that are common for the whole session and can be inherited from the test session
+        INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION].freeze
 
         # Environment runtime tags
         TAG_OS_ARCHITECTURE = "os.architecture"
         TAG_OS_PLATFORM = "os.platform"
         TAG_RUNTIME_NAME = "runtime.name"
         TAG_RUNTIME_VERSION = "runtime.version"
 
+        # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
+        SPAN_KIND_TEST = "test"
 
+        # test status as recognized by Datadog
         module Status
           PASS = "pass"
           FAIL = "fail"
           SKIP = "skip"
         end
+
+        # test types (e.g. test, benchmark, browser)
+        module Type
+          TEST = "test"
+          BENCHMARK = "benchmark" # DEV: not used yet, will be used when benchmarks are supported
+        end
       end
     end
   end

lib/datadog/ci/span.rb

@@ -130,7 +130,7 @@ def set_environment_runtime_tags
       end
 
       def set_default_tags
-        tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TYPE_TEST)
+        tracer_span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::Test::SPAN_KIND_TEST)
       end
 
       # Sets the parameters for this span for parametrized tests (e.g. Cucumber examples or RSpec shared specs).

lib/datadog/ci/test_visibility/recorder.rb

@@ -94,6 +94,7 @@ def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
           set_suite_context(tags, name: test_suite_name)
 
           tags[Ext::Test::TAG_NAME] = test_name
+          tags[Ext::Test::TAG_TYPE] ||= Ext::Test::Type::TEST
 
           span_options = build_span_options(
             service,

lib/datadog/ci/test_visibility/serializers/base.rb

@@ -34,7 +34,7 @@ def initialize(trace, span)
             @trace = trace
             @span = span
 
-            @meta = @span.meta.reject { |key, _| Ext::Test::SPECIAL_TAGS.include?(key) }
+            @meta = @span.meta.reject { |key, _| Ext::Test::TRANSIENT_TAGS.include?(key) }
 
             @errors = {}
             @validated = false

sig/datadog/ci/ext/test.rbs

@@ -2,63 +2,70 @@ module Datadog
   module CI
     module Ext
       module Test
-        CONTEXT_ORIGIN: String
+        CONTEXT_ORIGIN: "ciapp-test"
 
-        TAG_ARGUMENTS: String
+        TAG_ARGUMENTS: "test.arguments"
 
-        TAG_FRAMEWORK: String
+        TAG_FRAMEWORK: "test.framework"
 
-        TAG_FRAMEWORK_VERSION: String
+        TAG_FRAMEWORK_VERSION: "test.framework_version"
 
-        TAG_NAME: String
+        TAG_NAME: "test.name"
 
-        TAG_SKIP_REASON: String
+        TAG_SKIP_REASON: "test.skip_reason"
 
-        TAG_STATUS: String
+        TAG_STATUS: "test.status"
 
-        TAG_SUITE: String
+        TAG_SUITE: "test.suite"
 
-        TAG_MODULE: String
+        TAG_MODULE: "test.module"
 
-        TAG_TYPE: String
+        TAG_TYPE: "test.type"
 
-        TAG_COMMAND: String
+        TAG_COMMAND: "test.command"
 
-        TAG_SOURCE_FILE: String
+        TAG_SOURCE_FILE: "test.source.file"
 
-        TAG_SOURCE_START: String
+        TAG_SOURCE_START: "test.source.start"
 
-        TAG_CODEOWNERS: String
+        TAG_CODEOWNERS: "test.codeowners"
 
-        TAG_PARAMETERS: String
+        TAG_PARAMETERS: "test.parameters"
 
-        TAG_TEST_SESSION_ID: String
+        TAG_TEST_SESSION_ID: "_test.session_id"
 
-        TAG_TEST_MODULE_ID: String
+        TAG_TEST_MODULE_ID: "_test.module_id"
 
-        TAG_TEST_SUITE_ID: String
+        TAG_TEST_SUITE_ID: "_test.suite_id"
 
-        TEST_TYPE: String
-
-        SPECIAL_TAGS: Array[String]
+        TRANSIENT_TAGS: Array[String]
 
         INHERITABLE_TAGS: Array[String]
 
-        TAG_OS_ARCHITECTURE: String
+        TAG_OS_ARCHITECTURE: "os.architecture"
+
+        TAG_OS_PLATFORM: "os.platform"
+
+        TAG_RUNTIME_NAME: "runtime.name"
 
-        TAG_OS_PLATFORM: String
+        TAG_RUNTIME_VERSION: "runtime.version"
 
-        TAG_RUNTIME_NAME: String
+        TAG_SPAN_KIND: "span.kind"
 
-        TAG_RUNTIME_VERSION: String
-        TAG_SPAN_KIND: String
+        SPAN_KIND_TEST: "test"
 
         module Status
-          PASS: String
+          PASS: "pass"
+
+          FAIL: "fail"
+
+          SKIP: "skip"
+        end
 
-          FAIL: String
+        module Type
+          TEST: "test"
 
-          SKIP: String
+          BENCHMARK: "benchmark"
         end
       end
     end

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -77,12 +77,12 @@
       expect(scenario_span.resource).to eq("cucumber scenario")
       expect(scenario_span.service).to eq("jalapenos")
 
-      expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
+      expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::Test::SPAN_KIND_TEST)
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("cucumber scenario")
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
         "Datadog integration at spec/datadog/ci/contrib/cucumber/features/passing.feature"
       )
-      expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
+      expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::Type::TEST)
       expect(scenario_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
         Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK
       )
@@ -112,17 +112,14 @@
       expect(test_session_span).not_to be_nil
       expect(test_session_span.service).to eq("jalapenos")
       expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
+        Datadog::CI::Ext::Test::SPAN_KIND_TEST
       )
       expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
         Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK
       )
       expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
         Datadog::CI::Contrib::Cucumber::Integration.version.to_s
       )
-      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
-      )
       expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS)
     end
 
@@ -131,17 +128,14 @@
       expect(test_module_span.name).to eq(test_command)
       expect(test_module_span.service).to eq("jalapenos")
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
+        Datadog::CI::Ext::Test::SPAN_KIND_TEST
       )
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
         Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK
       )
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
         Datadog::CI::Contrib::Cucumber::Integration.version.to_s
       )
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
-      )
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS)
     end
 
@@ -150,17 +144,14 @@
       expect(test_suite_span.name).to eq("Datadog integration at spec/datadog/ci/contrib/cucumber/features/passing.feature")
       expect(test_suite_span.service).to eq("jalapenos")
       expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
+        Datadog::CI::Ext::Test::SPAN_KIND_TEST
       )
       expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
         Datadog::CI::Contrib::Cucumber::Ext::FRAMEWORK
       )
       expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
         Datadog::CI::Contrib::Cucumber::Integration.version.to_s
       )
-      expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
-      )
       expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq(Datadog::CI::Ext::Test::Status::PASS)
     end
 

spec/datadog/ci/contrib/minitest/instrumentation_spec.rb

@@ -58,8 +58,8 @@ def test_foo
       expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq(
         "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb"
       )
-      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
-      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
+      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::Test::SPAN_KIND_TEST)
+      expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::Type::TEST)
       expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK)
       expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
         Datadog::CI::Contrib::Minitest::Integration.version.to_s
@@ -409,10 +409,7 @@ def test_pass_other
           expect(test_session_span).not_to be_nil
           expect(test_session_span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION)
           expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-            Datadog::CI::Ext::AppTypes::TYPE_TEST
-          )
-          expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-            Datadog::CI::Ext::Test::TEST_TYPE
+            Datadog::CI::Ext::Test::SPAN_KIND_TEST
           )
           expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
             Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
@@ -432,10 +429,7 @@ def test_pass_other
           expect(test_module_span.name).to eq(test_command)
 
           expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-            Datadog::CI::Ext::AppTypes::TYPE_TEST
-          )
-          expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-            Datadog::CI::Ext::Test::TEST_TYPE
+            Datadog::CI::Ext::Test::SPAN_KIND_TEST
           )
           expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
             Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK
@@ -455,10 +449,7 @@ def test_pass_other
           expect(test_suite_span.name).to eq("SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb")
 
           expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-            Datadog::CI::Ext::AppTypes::TYPE_TEST
-          )
-          expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-            Datadog::CI::Ext::Test::TEST_TYPE
+            Datadog::CI::Ext::Test::SPAN_KIND_TEST
           )
           expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
             Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK

spec/datadog/ci/contrib/rspec/instrumentation_spec.rb

@@ -37,8 +37,8 @@ def with_new_rspec_environment
     expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("foo")
 
     expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq("some test at #{spec.file_path}")
-    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
-    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::Test::SPAN_KIND_TEST)
+    expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::Type::TEST)
     expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK)
     expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq(
       Datadog::CI::Contrib::RSpec::Integration.version.to_s
@@ -269,10 +269,7 @@ def rspec_session_run(with_failed_test: false, with_shared_test: false)
 
       expect(test_session_span.type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION)
       expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
-      )
-      expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
+        Datadog::CI::Ext::Test::SPAN_KIND_TEST
       )
       expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
         Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK
@@ -294,10 +291,7 @@ def rspec_session_run(with_failed_test: false, with_shared_test: false)
       expect(test_module_span.name).to eq(test_command)
 
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
-      )
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
+        Datadog::CI::Ext::Test::SPAN_KIND_TEST
       )
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
         Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK
@@ -319,10 +313,7 @@ def rspec_session_run(with_failed_test: false, with_shared_test: false)
       expect(test_suite_span.name).to eq("SomeTest at #{spec.file_path}")
 
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(
-        Datadog::CI::Ext::AppTypes::TYPE_TEST
-      )
-      expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(
-        Datadog::CI::Ext::Test::TEST_TYPE
+        Datadog::CI::Ext::Test::SPAN_KIND_TEST
       )
       expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(
         Datadog::CI::Contrib::RSpec::Ext::FRAMEWORK

spec/datadog/ci/test_visibility/recorder_spec.rb

@@ -24,7 +24,7 @@
     it "span.kind is equal to test" do
       expect(
         span_under_test.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)
-      ).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST)
+      ).to eq(Datadog::CI::Ext::Test::SPAN_KIND_TEST)
     end
   end