diff --git a/.standard_todo.yml b/.standard_todo.yml index 6c3441cc..d72dcc59 100644 --- a/.standard_todo.yml +++ b/.standard_todo.yml @@ -2,15 +2,17 @@ # Remove from this list as you refactor files. --- ignore: -- lib/datadog/ci/contrib/minitest/integration.rb: - - Style/SafeNavigation -- lib/datadog/ci/contrib/cucumber/integration.rb: - - Style/SafeNavigation -- lib/datadog/ci/contrib/rspec/integration.rb: - - Style/SafeNavigation -- lib/datadog/ci/ext/environment.rb: - - Style/SafeNavigation -- spec/support/log_helpers.rb: - - Performance/UnfreezeString -- Appraisals: - - Style/Alias + - lib/datadog/ci/test_visibility/serializers/base.rb: + - Style/HashExcept + - lib/datadog/ci/contrib/minitest/integration.rb: + - Style/SafeNavigation + - lib/datadog/ci/contrib/cucumber/integration.rb: + - Style/SafeNavigation + - lib/datadog/ci/contrib/rspec/integration.rb: + - Style/SafeNavigation + - lib/datadog/ci/ext/environment.rb: + - Style/SafeNavigation + - spec/support/log_helpers.rb: + - Performance/UnfreezeString + - Appraisals: + - Style/Alias diff --git a/lib/datadog/ci.rb b/lib/datadog/ci.rb index 7bfd382e..e89d30cf 100644 --- a/lib/datadog/ci.rb +++ b/lib/datadog/ci.rb @@ -10,8 +10,59 @@ module Datadog # @public_api module CI class << self + # Return a {Datadog::CI::TestSesstion ci_test_session} that represents the whole test session run. + # Raises an error if a session is already active. + # + # The {#start_test_session} method is used to mark the start of the test session: + # ``` + # Datadog::CI.start_test_session( + # service: "my-web-site-tests", + # tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" } + # ) + # + # # Somewhere else after test run has ended + # Datadog::CI.active_test_session.finish + # ``` + # + # Remember that calling {Datadog::CI::TestSession#finish} is mandatory. + # + # @param [String] service_name the service name for this session + # @param [Hash] tags extra tags which should be added to the test. + # @return [Datadog::CI::TestSession] returns the active, running {Datadog::CI::TestSession}. + # @return [nil] if test suite level visibility is disabled (old Datadog agent detected) + # + # @public_api + def start_test_session(service_name: nil, tags: {}) + recorder.start_test_session(service_name: service_name, tags: tags) + end + + # The active, unfinished test session span. + # + # Usage: + # + # ``` + # # start a test session + # Datadog::CI.start_test_session( + # service: "my-web-site-tests", + # tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" } + # ) + # + # # somewhere else, access the session + # test_session = Datadog::CI.active_test_session + # test_session.finish + # ``` + # + # @return [Datadog::CI::TestSession] the active test session + # @return [nil] if no test session is active + def active_test_session + recorder.active_test_session + end + # Return a {Datadog::CI::Test ci_test} that will trace a test called `test_name`. # Raises an error if a test is already active. + # If there is an active test session, the new test will be connected to the session. + # The test will inherit service name and tags from the running test session if not provided + # in parameters. # # You could trace your test using a do-block like: # @@ -189,6 +240,11 @@ def deactivate_test(test) recorder.deactivate_test(test) end + # Internal only, to finish a test session use Datadog::CI::TestSession#finish + def deactivate_test_session + recorder.deactivate_test_session + end + private def components diff --git a/lib/datadog/ci/concurrent_span.rb b/lib/datadog/ci/concurrent_span.rb new file mode 100644 index 00000000..994e5038 --- /dev/null +++ b/lib/datadog/ci/concurrent_span.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require_relative "span" + +module Datadog + module CI + # Represents a single part of a test run that can be safely shared between threads. + # Examples of shared objects are: TestSession, TestModule, TestSpan. + # + # @public_api + class ConcurrentSpan < Span + def initialize(tracer_span) + super + + @mutex = Mutex.new + end + + # Gets tag value by key. This method is thread-safe. + # @param [String] key the key of the tag. + # @return [String] the value of the tag. + def get_tag(key) + synchronize { super } + end + + # Sets tag value by key. This method is thread-safe. + # @param [String] key the key of the tag. + # @param [String] value the value of the tag. + # @return [void] + def set_tag(key, value) + synchronize { super } + end + + # Sets metric value by key. This method is thread-safe. + # @param [String] key the key of the metric. + # @param [Numeric] value the value of the metric. + # @return [void] + def set_metric(key, value) + synchronize { super } + end + + # Finishes the span. This method is thread-safe. + # @return [void] + def finish + synchronize { super } + end + + # Sets multiple tags at once. This method is thread-safe. + # @param [Hash[String, String]] tags the tags to set. + # @return [void] + def set_tags(tags) + synchronize { super } + end + + def synchronize + @mutex.synchronize { yield } + end + end + end +end diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 2bdb1cfb..870dbb10 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -6,6 +6,8 @@ require_relative "../ext/transport" require_relative "../test_visibility/flush" require_relative "../test_visibility/transport" +require_relative "../test_visibility/serializers/factories/test_level" +require_relative "../test_visibility/serializers/factories/test_suite_level" require_relative "../transport/api/builder" require_relative "../recorder" @@ -48,7 +50,7 @@ def activate_ci!(settings) settings.tracing.test_mode.enabled = true # Choose user defined TraceFlush or default to CI TraceFlush - settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Finished.new + settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Partial.new writer_options = settings.ci.writer_options if test_visibility_transport @@ -60,7 +62,9 @@ def activate_ci!(settings) settings.tracing.test_mode.writer_options = writer_options - @ci_recorder = Recorder.new + @ci_recorder = Recorder.new( + test_suite_level_visibility_enabled: settings.ci.experimental_test_suite_level_visibility_enabled + ) end def can_use_evp_proxy?(settings, agent_settings) @@ -89,6 +93,7 @@ def build_agentless_transport(settings) Datadog::CI::TestVisibility::Transport.new( api: Transport::Api::Builder.build_ci_test_cycle_api(settings), + serializers_factory: serializers_factory(settings), dd_env: settings.env ) end @@ -99,9 +104,18 @@ def build_evp_proxy_transport(settings, agent_settings) Datadog::CI::TestVisibility::Transport.new( api: Transport::Api::Builder.build_evp_proxy_api(agent_settings), + serializers_factory: serializers_factory(settings), dd_env: settings.env ) end + + def serializers_factory(settings) + if settings.ci.experimental_test_suite_level_visibility_enabled + Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel + else + Datadog::CI::TestVisibility::Serializers::Factories::TestLevel + end + end end end end diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index eb92f4b9..c3ac21a2 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -34,6 +34,12 @@ def self.add_settings!(base) o.env CI::Ext::Settings::ENV_AGENTLESS_URL end + option :experimental_test_suite_level_visibility_enabled do |o| + o.type :bool + o.env CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED + o.default false + end + define_method(:instrument) do |integration_name, options = {}, &block| return unless enabled diff --git a/lib/datadog/ci/context/global.rb b/lib/datadog/ci/context/global.rb new file mode 100644 index 00000000..0f7ebfdd --- /dev/null +++ b/lib/datadog/ci/context/global.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Context + # This context is shared between threads and represents the current test session. + class Global + def initialize + @mutex = Mutex.new + @test_session = nil + end + + def active_test_session + @test_session + end + + def activate_test_session!(test_session) + @mutex.synchronize do + raise "Nested test sessions are not supported. Currently active test session: #{@test_session}" unless @test_session.nil? + + @test_session = test_session + end + end + + def deactivate_test_session! + @mutex.synchronize { @test_session = nil } + end + end + end + end +end diff --git a/lib/datadog/ci/ext/app_types.rb b/lib/datadog/ci/ext/app_types.rb index e83ad3c3..a4a36529 100644 --- a/lib/datadog/ci/ext/app_types.rb +++ b/lib/datadog/ci/ext/app_types.rb @@ -5,6 +5,9 @@ module CI module Ext module AppTypes TYPE_TEST = "test" + TYPE_TEST_SESSION = "test_session_end" + + CI_SPAN_TYPES = [TYPE_TEST, TYPE_TEST_SESSION].freeze end end end diff --git a/lib/datadog/ci/ext/settings.rb b/lib/datadog/ci/ext/settings.rb index c08a5f56..3102e18a 100644 --- a/lib/datadog/ci/ext/settings.rb +++ b/lib/datadog/ci/ext/settings.rb @@ -8,6 +8,7 @@ module Settings ENV_MODE_ENABLED = "DD_TRACE_CI_ENABLED" ENV_AGENTLESS_MODE_ENABLED = "DD_CIVISIBILITY_AGENTLESS_ENABLED" ENV_AGENTLESS_URL = "DD_CIVISIBILITY_AGENTLESS_URL" + ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED" end end end diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index af7199fb..6da4c46f 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -16,6 +16,14 @@ module Test TAG_SUITE = "test.suite" TAG_TRAITS = "test.traits" TAG_TYPE = "test.type" + TAG_COMMAND = "test.command" + + # those tags are special and they are used to conrrelate tests with the test sessions, suites, and modules + TAG_TEST_SESSION_ID = "_test.session_id" + SPECIAL_TAGS = [TAG_TEST_SESSION_ID].freeze + + # tags that can be inherited from the test session + INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze # Environment runtime tags TAG_OS_ARCHITECTURE = "os.architecture" diff --git a/lib/datadog/ci/recorder.rb b/lib/datadog/ci/recorder.rb index 0d41bd9a..c163d88e 100644 --- a/lib/datadog/ci/recorder.rb +++ b/lib/datadog/ci/recorder.rb @@ -8,24 +8,57 @@ require_relative "ext/test" require_relative "ext/environment" +require_relative "context/global" require_relative "context/local" require_relative "span" require_relative "test" +require_relative "test_session" module Datadog module CI # Common behavior for CI tests class Recorder - attr_reader :environment_tags + attr_reader :environment_tags, :test_suite_level_visibility_enabled + + def initialize(test_suite_level_visibility_enabled: false) + @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled - def initialize @environment_tags = Ext::Environment.tags(ENV).freeze @local_context = Context::Local.new + @global_context = Context::Global.new + end + + def start_test_session(service_name: nil, tags: {}) + return nil unless @test_suite_level_visibility_enabled + + span_options = { + service: service_name, + span_type: Ext::AppTypes::TYPE_TEST_SESSION + } + + tracer_span = Datadog::Tracing.trace("test.session", **span_options) + trace = Datadog::Tracing.active_trace + + set_trace_origin(trace) + + tags[Ext::Test::TAG_TEST_SESSION_ID] = tracer_span.id + + test_session = build_test_session(tracer_span, tags) + @global_context.activate_test_session!(test_session) + + test_session end # Creates a new span for a CI test def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, &block) + test_session = active_test_session + if test_session + service_name ||= test_session.service + + tags = test_session.inheritable_tags.merge(tags) + end + span_options = { resource: test_name, service: service_name, @@ -33,6 +66,7 @@ def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, & } tags[Ext::Test::TAG_NAME] = test_name + tags[Ext::Test::TAG_TEST_SESSION_ID] = test_session.id if test_session if block Datadog::Tracing.trace(operation_name, **span_options) do |tracer_span, trace| @@ -73,17 +107,26 @@ def trace(span_type, span_name, tags: {}, &block) end end + def active_span + tracer_span = Datadog::Tracing.active_span + Span.new(tracer_span) if tracer_span + end + def active_test @local_context.active_test end + def active_test_session + @global_context.active_test_session + end + + # TODO: does it make sense to have a paramter here? def deactivate_test(test) @local_context.deactivate_test!(test) end - def active_span - tracer_span = Datadog::Tracing.active_span - Span.new(tracer_span) if tracer_span + def deactivate_test_session + @global_context.deactivate_test_session! end private @@ -93,26 +136,30 @@ def set_trace_origin(trace) trace.origin = Ext::Test::CONTEXT_ORIGIN if trace end + def build_test_session(tracer_span, tags) + test_session = TestSession.new(tracer_span) + set_initial_tags(test_session, tags) + test_session + end + def build_test(tracer_span, tags) test = Test.new(tracer_span) - - test.set_default_tags - test.set_environment_runtime_tags - - test.set_tags(tags) - test.set_tags(environment_tags) - + set_initial_tags(test, tags) test end def build_span(tracer_span, tags) span = Span.new(tracer_span) + set_initial_tags(span, tags) + span + end - span.set_default_tags - span.set_environment_runtime_tags - span.set_tags(tags) + def set_initial_tags(ci_span, tags) + ci_span.set_default_tags + ci_span.set_environment_runtime_tags - span + ci_span.set_tags(tags) + ci_span.set_tags(environment_tags) end end end diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb index 19e45a85..08f8cffe 100644 --- a/lib/datadog/ci/span.rb +++ b/lib/datadog/ci/span.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "ext/test" +require_relative "utils/test_run" module Datadog module CI @@ -15,11 +16,21 @@ def initialize(tracer_span) @tracer_span = tracer_span end + # @return [Integer] the ID of the span. + def id + tracer_span.id + end + # @return [String] the name of the span. def name tracer_span.name end + # @return [String] the service name of the span. + def service + tracer_span.service + end + # @return [String] the type of the span (for example "test" or type that was provided to [Datadog::CI.trace]). def span_type tracer_span.type @@ -83,9 +94,7 @@ def finish # @param [Hash[String, String]] tags the tags to set. # @return [void] def set_tags(tags) - tags.each do |key, value| - tracer_span.set_tag(key, value) - end + tracer_span.set_tags(tags) end def set_environment_runtime_tags @@ -93,6 +102,7 @@ def set_environment_runtime_tags tracer_span.set_tag(Ext::Test::TAG_OS_PLATFORM, ::RbConfig::CONFIG["host_os"]) tracer_span.set_tag(Ext::Test::TAG_RUNTIME_NAME, Core::Environment::Ext::LANG_ENGINE) tracer_span.set_tag(Ext::Test::TAG_RUNTIME_VERSION, Core::Environment::Ext::ENGINE_VERSION) + tracer_span.set_tag(Ext::Test::TAG_COMMAND, Utils::TestRun.command) end def set_default_tags diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb index 6cf43de2..d0cc5187 100644 --- a/lib/datadog/ci/test.rb +++ b/lib/datadog/ci/test.rb @@ -5,7 +5,6 @@ module Datadog module CI # Represents a single part of a test run. - # Could be a session, suite, test, or any custom span. # # @public_api class Test < Span diff --git a/lib/datadog/ci/test_session.rb b/lib/datadog/ci/test_session.rb new file mode 100644 index 00000000..baf4e3b6 --- /dev/null +++ b/lib/datadog/ci/test_session.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require_relative "concurrent_span" +require_relative "ext/test" + +module Datadog + module CI + # Represents the whole test session process. + # This object can be shared between multiple threads. + # + # @public_api + class TestSession < ConcurrentSpan + # Finishes the current test session. + # @return [void] + def finish + super + + CI.deactivate_test_session + end + + def inheritable_tags + return @inheritable_tags if defined?(@inheritable_tags) + + # this method is not synchronized because it does not iterate over the tags, but rather + # uses synchronized method to get each tag value + res = {} + Ext::Test::INHERITABLE_TAGS.each do |tag| + res[tag] = get_tag(tag) + end + @inheritable_tags = res + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/serializers/base.rb b/lib/datadog/ci/test_visibility/serializers/base.rb index 04e20e72..1c68c8a3 100644 --- a/lib/datadog/ci/test_visibility/serializers/base.rb +++ b/lib/datadog/ci/test_visibility/serializers/base.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative "../../ext/test" + module Datadog module CI module TestVisibility @@ -9,11 +11,13 @@ class Base MINIMUM_DURATION_NANO = 0 MAXIMUM_DURATION_NANO = 9223372036854775807 - attr_reader :trace, :span + attr_reader :trace, :span, :meta def initialize(trace, span) @trace = trace @span = span + + @meta = @span.meta.reject { |key, _| Ext::Test::SPECIAL_TAGS.include?(key) } end def to_msgpack(packer = nil) @@ -67,6 +71,10 @@ def parent_id @span.parent_id end + def test_session_id + @span.get_tag(Ext::Test::TAG_TEST_SESSION_ID) + end + def type end @@ -98,10 +106,6 @@ def duration @duration ||= duration_nano(@span.duration) end - def meta - @span.meta - end - def metrics @span.metrics end diff --git a/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb b/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb new file mode 100644 index 00000000..1675fe7b --- /dev/null +++ b/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require_relative "../test_v2" +require_relative "../test_session" +require_relative "../span" + +module Datadog + module CI + module TestVisibility + module Serializers + module Factories + # This factory takes care of creating citestcycle serializers when test-suite-level visibility is enabled + module TestSuiteLevel + module_function + + def serializer(trace, span) + case span.type + when Datadog::CI::Ext::AppTypes::TYPE_TEST + Serializers::TestV2.new(trace, span) + when Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION + Serializers::TestSession.new(trace, span) + else + Serializers::Span.new(trace, span) + end + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/serializers/test_session.rb b/lib/datadog/ci/test_visibility/serializers/test_session.rb new file mode 100644 index 00000000..4e4ed7d2 --- /dev/null +++ b/lib/datadog/ci/test_visibility/serializers/test_session.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require_relative "base" +require_relative "../../ext/test" + +module Datadog + module CI + module TestVisibility + module Serializers + class TestSession < Base + CONTENT_FIELDS = [ + "test_session_id", + "name", "resource", "service", + "error", "start", "duration", + "meta", "metrics", + "type" => "span_type" + ].freeze + + CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS) + + REQUIRED_FIELDS = [ + "test_session_id", + "error", + "name", + "resource", + "start", + "duration" + ].freeze + + def content_fields + CONTENT_FIELDS + end + + def content_map_size + CONTENT_MAP_SIZE + end + + def type + Ext::AppTypes::TYPE_TEST_SESSION + end + + def name + "#{@span.get_tag(Ext::Test::TAG_FRAMEWORK)}.test_session" + end + + def resource + "#{@span.get_tag(Ext::Test::TAG_FRAMEWORK)}.test_session.#{@span.get_tag(Ext::Test::TAG_COMMAND)}" + end + + private + + def required_fields + REQUIRED_FIELDS + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/serializers/test_v1.rb b/lib/datadog/ci/test_visibility/serializers/test_v1.rb index c2547234..ed9fb740 100644 --- a/lib/datadog/ci/test_visibility/serializers/test_v1.rb +++ b/lib/datadog/ci/test_visibility/serializers/test_v1.rb @@ -37,7 +37,7 @@ def content_map_size end def type - "test" + Ext::AppTypes::TYPE_TEST end def name diff --git a/lib/datadog/ci/test_visibility/serializers/test_v2.rb b/lib/datadog/ci/test_visibility/serializers/test_v2.rb new file mode 100644 index 00000000..1a08c1d0 --- /dev/null +++ b/lib/datadog/ci/test_visibility/serializers/test_v2.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require_relative "test_v1" +require_relative "../../ext/test" + +module Datadog + module CI + module TestVisibility + module Serializers + class TestV2 < TestV1 + CONTENT_FIELDS = [ + "trace_id", "span_id", + "name", "resource", "service", + "error", "start", "duration", + "meta", "metrics", "test_session_id", + "type" => "span_type" + ].freeze + + CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS) + + REQUIRED_FIELDS = [ + "test_session_id", + "trace_id", + "span_id", + "error", + "name", + "resource", + "start", + "duration" + ].freeze + + def content_fields + CONTENT_FIELDS + end + + def content_map_size + CONTENT_MAP_SIZE + end + + def version + 2 + end + + private + + def required_fields + REQUIRED_FIELDS + end + end + end + end + end +end diff --git a/lib/datadog/ci/utils/test_run.rb b/lib/datadog/ci/utils/test_run.rb new file mode 100644 index 00000000..1e41715e --- /dev/null +++ b/lib/datadog/ci/utils/test_run.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Utils + module TestRun + def self.command + return @command if defined?(@command) + + @command = "#{$0} #{ARGV.join(" ")}" + end + end + end + end +end diff --git a/sig/datadog/ci.rbs b/sig/datadog/ci.rbs index 9a1b76b4..fc1b3abc 100644 --- a/sig/datadog/ci.rbs +++ b/sig/datadog/ci.rbs @@ -4,14 +4,20 @@ module Datadog def self.start_test: (String span_name, ?service_name: String?, ?operation_name: String, ?tags: Hash[untyped, untyped]) -> Datadog::CI::Test + def self.start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession? + def self.trace: (String span_type, String span_name, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped + def self.active_test_session: () -> Datadog::CI::TestSession? + def self.active_test: () -> Datadog::CI::Test? def self.active_span: (String span_type) -> Datadog::CI::Span? def self.deactivate_test: (Datadog::CI::Test test) -> void + def self.deactivate_test_session: () -> void + def self.components: () -> Datadog::CI::Configuration::Components def self.recorder: () -> Datadog::CI::Recorder diff --git a/sig/datadog/ci/concurrent_span.rbs b/sig/datadog/ci/concurrent_span.rbs new file mode 100644 index 00000000..ff834140 --- /dev/null +++ b/sig/datadog/ci/concurrent_span.rbs @@ -0,0 +1,23 @@ +module Datadog + module CI + class ConcurrentSpan < Span + @mutex: Thread::Mutex + + def initialize: (Datadog::Tracing::SpanOperation tracer_span) -> void + def passed!: () -> void + def failed!: (?exception: untyped?) -> void + def skipped!: (?exception: untyped?, ?reason: String?) -> void + def get_tag: (String key) -> untyped? + def set_tag: (String key, untyped? value) -> void + def set_metric: (String key, untyped value) -> void + def finish: () -> void + def set_tags: (Hash[untyped, untyped] tags) -> void + + def set_environment_runtime_tags: () -> void + + def set_default_tags: () -> void + + def synchronize: () { () -> untyped } -> untyped + end + end +end diff --git a/sig/datadog/ci/configuration/components.rbs b/sig/datadog/ci/configuration/components.rbs index bae3e55a..5a1e0cad 100644 --- a/sig/datadog/ci/configuration/components.rbs +++ b/sig/datadog/ci/configuration/components.rbs @@ -13,6 +13,7 @@ module Datadog def build_agentless_transport: (untyped settings) -> Datadog::CI::TestVisibility::Transport? def build_evp_proxy_transport: (untyped settings, untyped agent_settings) -> Datadog::CI::TestVisibility::Transport def can_use_evp_proxy?: (untyped settings, untyped agent_settings) -> bool + def serializers_factory: (untyped settings) -> (singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel)) end end end diff --git a/sig/datadog/ci/context/global.rbs b/sig/datadog/ci/context/global.rbs new file mode 100644 index 00000000..ce319b46 --- /dev/null +++ b/sig/datadog/ci/context/global.rbs @@ -0,0 +1,19 @@ +module Datadog + module CI + module Context + class Global + @mutex: Thread::Mutex + + @test_session: Datadog::CI::TestSession? + + def initialize: () -> void + + def active_test_session: () -> Datadog::CI::TestSession? + + def activate_test_session!: (Datadog::CI::TestSession test_session) -> void + + def deactivate_test_session!: () -> void + end + end + end +end diff --git a/sig/datadog/ci/ext/app_types.rbs b/sig/datadog/ci/ext/app_types.rbs index 8188d06c..1f01f77b 100644 --- a/sig/datadog/ci/ext/app_types.rbs +++ b/sig/datadog/ci/ext/app_types.rbs @@ -2,7 +2,10 @@ module Datadog module CI module Ext module AppTypes - TYPE_TEST: String + TYPE_TEST: "test" + TYPE_TEST_SESSION: "test_session_end" + + CI_SPAN_TYPES: Array[String] end end end diff --git a/sig/datadog/ci/ext/settings.rbs b/sig/datadog/ci/ext/settings.rbs index 5d76a0cd..aef6d2f8 100644 --- a/sig/datadog/ci/ext/settings.rbs +++ b/sig/datadog/ci/ext/settings.rbs @@ -5,6 +5,7 @@ module Datadog ENV_MODE_ENABLED: String ENV_AGENTLESS_MODE_ENABLED: String ENV_AGENTLESS_URL: String + ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED: String end end end diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index 6f0c0c0c..ba7d6855 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -21,6 +21,15 @@ module Datadog TAG_TRAITS: String TAG_TYPE: String + + TAG_COMMAND: String + + TAG_TEST_SESSION_ID: String + + SPECIAL_TAGS: Array[String] + + INHERITABLE_TAGS: Array[String] + TAG_OS_ARCHITECTURE: String TAG_OS_PLATFORM: String diff --git a/sig/datadog/ci/recorder.rbs b/sig/datadog/ci/recorder.rbs index ff06b0c7..edb90fa7 100644 --- a/sig/datadog/ci/recorder.rbs +++ b/sig/datadog/ci/recorder.rbs @@ -1,21 +1,31 @@ module Datadog module CI class Recorder + @test_suite_level_visibility_enabled: bool @environment_tags: Hash[String, String] @local_context: Datadog::CI::Context::Local + @global_context: Datadog::CI::Context::Global attr_reader environment_tags: Hash[String, String] + def initialize: (?test_suite_level_visibility_enabled: bool) -> void + def trace_test: (String span_name, ?service_name: String?, ?operation_name: String, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Test span) -> untyped } -> untyped def trace: (String span_type, String span_name, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped + def start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession? + + def active_test_session: () -> Datadog::CI::TestSession? + def active_test: () -> Datadog::CI::Test? def active_span: () -> Datadog::CI::Span? def deactivate_test: (Datadog::CI::Test test) -> void + def deactivate_test_session: () -> void + def create_datadog_span: (String span_name, ?span_options: Hash[untyped, untyped], ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped def set_trace_origin: (Datadog::Tracing::TraceOperation trace) -> untyped @@ -24,7 +34,11 @@ module Datadog def build_test: (Datadog::Tracing::SpanOperation tracer_span, Hash[untyped, untyped] tags) -> Datadog::CI::Test + def build_test_session: (Datadog::Tracing::SpanOperation tracer_span, Hash[untyped, untyped] tags) -> Datadog::CI::TestSession + def build_span: (Datadog::Tracing::SpanOperation tracer_span, Hash[untyped, untyped] tags) -> Datadog::CI::Span + + def set_initial_tags: (Datadog::CI::Span ci_span, Hash[untyped, untyped] tags) -> void end end end diff --git a/sig/datadog/ci/span.rbs b/sig/datadog/ci/span.rbs index 32ae69a9..6265f0d6 100644 --- a/sig/datadog/ci/span.rbs +++ b/sig/datadog/ci/span.rbs @@ -7,8 +7,12 @@ module Datadog def initialize: (Datadog::Tracing::SpanOperation tracer_span) -> void + def id: () -> Integer + def name: () -> String + def service: () -> String + def passed!: () -> void def failed!: (?exception: untyped?) -> void diff --git a/sig/datadog/ci/test_session.rbs b/sig/datadog/ci/test_session.rbs new file mode 100644 index 00000000..3d369772 --- /dev/null +++ b/sig/datadog/ci/test_session.rbs @@ -0,0 +1,9 @@ +module Datadog + module CI + class TestSession < ConcurrentSpan + @inheritable_tags: Hash[untyped, untyped] + + def inheritable_tags: () -> Hash[untyped, untyped] + end + end +end diff --git a/sig/datadog/ci/test_visibility/serializers/base.rbs b/sig/datadog/ci/test_visibility/serializers/base.rbs index 908b19d6..80c5f6bc 100644 --- a/sig/datadog/ci/test_visibility/serializers/base.rbs +++ b/sig/datadog/ci/test_visibility/serializers/base.rbs @@ -10,6 +10,7 @@ module Datadog @content_fields_count: Integer @start: Integer @duration: Integer + @meta: Hash[untyped, untyped] attr_reader trace: Datadog::Tracing::TraceSegment attr_reader span: Datadog::Tracing::Span diff --git a/sig/datadog/ci/test_visibility/serializers/factories/test_suite_level.rbs b/sig/datadog/ci/test_visibility/serializers/factories/test_suite_level.rbs new file mode 100644 index 00000000..8ebc424f --- /dev/null +++ b/sig/datadog/ci/test_visibility/serializers/factories/test_suite_level.rbs @@ -0,0 +1,13 @@ +module Datadog + module CI + module TestVisibility + module Serializers + module Factories + module TestSuiteLevel + def self?.serializer: (Datadog::Tracing::TraceSegment trace, Datadog::Tracing::Span span) -> Datadog::CI::TestVisibility::Serializers::Base + end + end + end + end + end +end diff --git a/sig/datadog/ci/test_visibility/serializers/test_session.rbs b/sig/datadog/ci/test_visibility/serializers/test_session.rbs new file mode 100644 index 00000000..98df8ac0 --- /dev/null +++ b/sig/datadog/ci/test_visibility/serializers/test_session.rbs @@ -0,0 +1,26 @@ +module Datadog + module CI + module TestVisibility + module Serializers + class TestSession < Base + CONTENT_FIELDS: Array[String | Hash[String, String]] + CONTENT_MAP_SIZE: Integer + REQUIRED_FIELDS: Array[String] + + def content_fields: () -> Array[String | Hash[String, String]] + def content_map_size: () -> Integer + + def type: () -> ::String + + def name: () -> ::String + + def resource: () -> ::String + + private + + def required_fields: () -> Array[String] + end + end + end + end +end diff --git a/sig/datadog/ci/test_visibility/serializers/test_v1.rbs b/sig/datadog/ci/test_visibility/serializers/test_v1.rbs index a590d687..3c365885 100644 --- a/sig/datadog/ci/test_visibility/serializers/test_v1.rbs +++ b/sig/datadog/ci/test_visibility/serializers/test_v1.rbs @@ -11,7 +11,7 @@ module Datadog def content_fields: () -> Array[String | Hash[String, String]] def content_map_size: () -> Integer - def type: () -> "test" + def type: () -> ::String def name: () -> ::String diff --git a/sig/datadog/ci/test_visibility/serializers/test_v2.rbs b/sig/datadog/ci/test_visibility/serializers/test_v2.rbs new file mode 100644 index 00000000..1ba034be --- /dev/null +++ b/sig/datadog/ci/test_visibility/serializers/test_v2.rbs @@ -0,0 +1,25 @@ +module Datadog + module CI + module TestVisibility + module Serializers + class TestV2 < TestV1 + CONTENT_FIELDS: ::Array[String | ::Hash[::String, String]] + + CONTENT_MAP_SIZE: Integer + + REQUIRED_FIELDS: ::Array[String] + + def content_fields: () -> ::Array[String | ::Hash[::String, String]] + + def content_map_size: () -> Integer + + def version: () -> 2 + + private + + def required_fields: () -> Array[String] + end + end + end + end +end diff --git a/sig/datadog/ci/test_visibility/transport.rbs b/sig/datadog/ci/test_visibility/transport.rbs index f94a68bf..20822d81 100644 --- a/sig/datadog/ci/test_visibility/transport.rbs +++ b/sig/datadog/ci/test_visibility/transport.rbs @@ -4,19 +4,19 @@ module Datadog class Transport DEFAULT_MAX_PAYLOAD_SIZE: Integer - attr_reader serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) + attr_reader serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) attr_reader dd_env: String? attr_reader api: Datadog::CI::Transport::Api::Base attr_reader max_payload_size: Integer @dd_env: String? - @serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) + @serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) @max_payload_size: Integer def initialize: ( api: Datadog::CI::Transport::Api::Base, ?dd_env: ::String?, - ?serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel), + ?serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel), ?max_payload_size: Integer ) -> void diff --git a/sig/datadog/ci/utils/test_run.rbs b/sig/datadog/ci/utils/test_run.rbs new file mode 100644 index 00000000..c985f99b --- /dev/null +++ b/sig/datadog/ci/utils/test_run.rbs @@ -0,0 +1,11 @@ +module Datadog + module CI + module Utils + module TestRun + self.@command: String + + def self.command: () -> String + end + end + end +end diff --git a/spec/datadog/ci/concurrent_span_spec.rb b/spec/datadog/ci/concurrent_span_spec.rb new file mode 100644 index 00000000..88074fb6 --- /dev/null +++ b/spec/datadog/ci/concurrent_span_spec.rb @@ -0,0 +1,118 @@ +THREADS_COUNT = 10 +REPEAT_COUNT = 20 + +RSpec.describe Datadog::CI::ConcurrentSpan do + describe "#finish" do + it "calls SpanOperation#stop once" do + REPEAT_COUNT.times do + tracer_span = Datadog::Tracing::SpanOperation.new("operation") + ci_span = described_class.new(tracer_span) + + expect(tracer_span).to receive(:stop).once + + (1..THREADS_COUNT).map do + Thread.new do + ci_span.finish + end + end.map(&:join) + end + end + end + + # the following tests make sure that ConcurrentSpan works exactly like Span + let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, name: "span_name", type: "test") } + subject(:span) { described_class.new(tracer_span) } + + describe "#passed!" do + it "sets the status to PASS" do + expect(tracer_span).to receive(:set_tag).with("test.status", "pass") + + span.passed! + end + end + + describe "#failed!" do + context "when exception is nil" do + it "sets the status to FAIL" do + expect(tracer_span).to receive(:status=).with(1) + expect(tracer_span).to receive(:set_tag).with("test.status", "fail") + + span.failed! + end + end + + context "when exception is provided" do + it "sets the status to FAIL" do + expect(tracer_span).to receive(:status=).with(1) + expect(tracer_span).to receive(:set_tag).with("test.status", "fail") + expect(tracer_span).to receive(:set_error).with("error") + + span.failed!(exception: "error") + end + end + end + + describe "#skipped!" do + context "when exception is nil" do + it "sets the status to SKIP" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).not_to receive(:set_error) + + span.skipped! + end + end + + context "when exception is provided" do + it "sets the status to SKIP and sets error" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).to receive(:set_error).with("error") + + span.skipped!(exception: "error") + end + end + + context "when reason is nil" do + it "doesn't set the skip reason tag" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).not_to receive(:set_tag).with("test.skip_reason", "reason") + + span + + span.skipped! + end + end + + context "when reason is provided" do + it "sets the skip reason tag" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).to receive(:set_tag).with("test.skip_reason", "reason") + + span.skipped!(reason: "reason") + end + end + end + + describe "#set_tag" do + it "sets the tag" do + expect(tracer_span).to receive(:set_tag).with("foo", "bar") + + span.set_tag("foo", "bar") + end + end + + describe "#set_tags" do + it "sets the tags" do + expect(tracer_span).to receive(:set_tags).with({"foo" => "bar", "baz" => "qux"}) + + span.set_tags("foo" => "bar", "baz" => "qux") + end + end + + describe "#set_metric" do + it "sets the metric" do + expect(tracer_span).to receive(:set_metric).with("foo", "bar") + + span.set_metric("foo", "bar") + end + end +end diff --git a/spec/datadog/ci/configuration/components_spec.rb b/spec/datadog/ci/configuration/components_spec.rb index 68719ee6..67d00c8d 100644 --- a/spec/datadog/ci/configuration/components_spec.rb +++ b/spec/datadog/ci/configuration/components_spec.rb @@ -42,6 +42,10 @@ .to receive(:agentless_mode_enabled) .and_return(agentless_enabled) + allow(settings.ci) + .to receive(:experimental_test_suite_level_visibility_enabled) + .and_return(experimental_test_suite_level_visibility_enabled) + allow(settings.ci) .to receive(:agentless_url) .and_return(agentless_url) @@ -90,13 +94,28 @@ let(:agentless_url) { nil } let(:dd_site) { nil } let(:agentless_enabled) { false } + let(:experimental_test_suite_level_visibility_enabled) { false } let(:evp_proxy_supported) { false } context "is enabled" do let(:enabled) { true } - it "creates a CI recorder" do - expect(components.ci_recorder).to be_kind_of(Datadog::CI::Recorder) + context "when #experimental_test_suite_level_visibility_enabled" do + context "is false" do + it "creates a CI recorder with test_suite_level_visibility_enabled=false" do + expect(components.ci_recorder).to be_kind_of(Datadog::CI::Recorder) + expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(false) + end + end + + context "is true" do + let(:experimental_test_suite_level_visibility_enabled) { true } + + it "creates a CI recorder with test_suite_level_visibility_enabled=false" do + expect(components.ci_recorder).to be_kind_of(Datadog::CI::Recorder) + expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(true) + end + end end context "and when #agentless_mode" do @@ -129,7 +148,7 @@ expect(settings.tracing.test_mode) .to have_received(:trace_flush=) - .with(settings.ci.trace_flush || kind_of(Datadog::CI::TestVisibility::Flush::Finished)) + .with(settings.ci.trace_flush || kind_of(Datadog::CI::TestVisibility::Flush::Partial)) expect(settings.tracing.test_mode).to have_received(:writer_options=) do |options| expect(options[:transport]).to be_nil diff --git a/spec/datadog/ci/configuration/settings_spec.rb b/spec/datadog/ci/configuration/settings_spec.rb index e58fcab7..17963a2a 100644 --- a/spec/datadog/ci/configuration/settings_spec.rb +++ b/spec/datadog/ci/configuration/settings_spec.rb @@ -172,6 +172,51 @@ def patcher end end + describe "#experimental_test_suite_level_visibility_enabled" do + subject(:experimental_test_suite_level_visibility_enabled) do + settings.ci.experimental_test_suite_level_visibility_enabled + end + + it { is_expected.to be false } + + context "when #{Datadog::CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED}" do + around do |example| + ClimateControl.modify( + Datadog::CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED => enable + ) do + example.run + end + end + + context "is not defined" do + let(:enable) { nil } + + it { is_expected.to be false } + end + + context "is set to true" do + let(:enable) { "true" } + + it { is_expected.to be true } + end + + context "is set to false" do + let(:enable) { "false" } + + it { is_expected.to be false } + end + end + end + + describe "#experimental_test_suite_level_visibility_enabled=" do + it "updates the #enabled setting" do + expect { settings.ci.experimental_test_suite_level_visibility_enabled = true } + .to change { settings.ci.experimental_test_suite_level_visibility_enabled } + .from(false) + .to(true) + end + end + describe "#instrument" do let(:integration_name) { :fake } diff --git a/spec/datadog/ci/context/global_spec.rb b/spec/datadog/ci/context/global_spec.rb new file mode 100644 index 00000000..a9330479 --- /dev/null +++ b/spec/datadog/ci/context/global_spec.rb @@ -0,0 +1,51 @@ +RSpec.describe Datadog::CI::Context::Global do + subject { described_class.new } + + let(:tracer_span) { double(Datadog::Tracing::SpanOperation, name: "test.session") } + let(:session) { Datadog::CI::TestSession.new(tracer_span) } + + describe "#activate_test_session!" do + context "when a test session is already active" do + before do + subject.activate_test_session!(Datadog::CI::TestSession.new(tracer_span)) + end + + it "raises an error" do + expect { subject.activate_test_session!(session) }.to( + raise_error( + RuntimeError, + "Nested test sessions are not supported. Currently active test session: " \ + "#{session}" + ) + ) + end + end + + context "when no test session is active" do + it "activates the test session" do + subject.activate_test_session!(session) + expect(subject.active_test_session).to be(session) + end + end + end + + describe "#deactivate_test_session!" do + context "when a test session is active" do + before do + subject.activate_test_session!(session) + end + + it "deactivates the test session" do + subject.deactivate_test_session! + expect(subject.active_test_session).to be_nil + end + end + + context "when no test session is active" do + it "does nothing" do + subject.deactivate_test_session! + expect(subject.active_test_session).to be_nil + end + end + end +end diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index 7f4ee337..58b186f2 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -4,9 +4,15 @@ let(:operation_name) { "span name" } let(:test_name) { "test name" } let(:tags) { {} } + let(:expected_tags) { {} } let(:environment_tags) { Datadog::CI::Ext::Environment.tags(ENV) } + let(:test_suite_level_visibility_enabled) { true } - subject(:recorder) { described_class.new } + let(:ci_span) do + spy("CI object spy") + end + + subject(:recorder) { described_class.new(test_suite_level_visibility_enabled: test_suite_level_visibility_enabled) } before do allow(Datadog::Tracing).to receive(:active_trace).and_return(trace_op) @@ -21,14 +27,17 @@ end end - describe "#trace_test" do - def expect_initialized_test - allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_test) - expect(ci_test).to receive(:set_default_tags) - expect(ci_test).to receive(:set_environment_runtime_tags) - expect(ci_test).to receive(:set_tags).with(tags) - expect(ci_test).to receive(:set_tags).with(environment_tags) + shared_examples_for "initialize ci span with tags" do + it do + expect(ci_span).to have_received(:set_default_tags) + expect(ci_span).to have_received(:set_environment_runtime_tags) + expect(ci_span).to have_received(:set_tags).with(expected_tags) + expect(ci_span).to have_received(:set_tags).with(environment_tags) end + end + + describe "#trace_test" do + let(:expected_tags) { {"test.name" => test_name} } context "when given a block" do subject(:trace) do @@ -42,7 +51,6 @@ def expect_initialized_test end let(:span_op) { Datadog::Tracing::SpanOperation.new(operation_name) } - let(:ci_test) { instance_double(Datadog::CI::Test) } let(:block) { proc { |s| block_spy.call(s) } } let(:block_result) { double("result") } let(:block_spy) { spy("block") } @@ -61,7 +69,7 @@ def expect_initialized_test } ) - expect_initialized_test + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) trace_block.call(span_op, trace_op) end @@ -70,59 +78,147 @@ def expect_initialized_test end it_behaves_like "internal tracing context" - it { expect(block_spy).to have_received(:call).with(ci_test) } + it_behaves_like "initialize ci span with tags" + + it { expect(block_spy).to have_received(:call).with(ci_span) } it { is_expected.to be(block_result) } end context "when not given a block" do - subject(:trace) do - recorder.trace_test( - test_name, - service_name: service, - operation_name: operation_name, - tags: tags - ) - end let(:span_op) { Datadog::Tracing::SpanOperation.new(operation_name) } - let(:ci_test) { instance_double(Datadog::CI::Test) } - before do - allow(Datadog::Tracing) - .to receive(:trace) - .with( - operation_name, - { - span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, - resource: test_name, - service: service - } + context "without active test session" do + subject(:trace) do + recorder.trace_test( + test_name, + service_name: service, + operation_name: operation_name, + tags: tags ) - .and_return(span_op) + end - expect_initialized_test + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, + resource: test_name, + service: service + } + ) + .and_return(span_op) - trace + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) + + trace + end + + it_behaves_like "internal tracing context" + it_behaves_like "initialize ci span with tags" + it { is_expected.to be(ci_span) } end - it_behaves_like "internal tracing context" - it { is_expected.to be(ci_test) } + context "when test session is active" do + let(:test_session_id) { 42 } + let(:inheritable_tags) { {"test.framework" => "my framework"} } + + let(:test_session_service) { "my-test-service" } + let(:test_session) do + instance_double( + Datadog::CI::TestSession, + service: test_session_service, + inheritable_tags: inheritable_tags, + id: test_session_id + ) + end + + before do + allow(recorder).to receive(:active_test_session).and_return(test_session) + end + + context "when service name and tags are not given" do + let(:expected_tags) do + {"test.framework" => "my framework", "test.name" => test_name, "_test.session_id" => test_session_id} + end + + subject(:trace) do + recorder.trace_test( + test_name, + operation_name: operation_name, + tags: tags + ) + end + + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, + resource: test_name, + service: test_session_service + } + ) + .and_return(span_op) + + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) + + trace + end + + it_behaves_like "initialize ci span with tags" + it { is_expected.to be(ci_span) } + end + + context "when service name and tags are given" do + let(:tags) { {"test.framework" => "special test framework"} } + let(:expected_tags) do + {"test.framework" => "special test framework", "test.name" => test_name, "_test.session_id" => test_session_id} + end + + subject(:trace) do + recorder.trace_test( + test_name, + service_name: service, + operation_name: operation_name, + tags: tags + ) + end + + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, + resource: test_name, + service: service + } + ) + .and_return(span_op) + + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) + + trace + end + + it_behaves_like "initialize ci span with tags" + it { is_expected.to be(ci_span) } + end + end end end describe "#trace" do - def expect_initialized_span - allow(Datadog::CI::Span).to receive(:new).with(span_op).and_return(ci_span) - expect(ci_span).to receive(:set_default_tags) - expect(ci_span).to receive(:set_environment_runtime_tags) - expect(ci_span).to receive(:set_tags).with(tags) - end - let(:tags) { {"my_tag" => "my_value"} } + let(:expected_tags) { {"my_tag" => "my_value"} } let(:span_type) { "step" } let(:span_name) { "span name" } - let(:expected_tags) { tags } - context "when given a block" do subject(:trace) do recorder.trace( @@ -134,7 +230,6 @@ def expect_initialized_span end let(:span_op) { Datadog::Tracing::SpanOperation.new(span_name) } - let(:ci_span) { instance_double(Datadog::CI::Span) } let(:block) { proc { |s| block_spy.call(s) } } let(:block_result) { double("result") } let(:block_spy) { spy("block") } @@ -154,11 +249,12 @@ def expect_initialized_span trace_block.call(span_op, trace_op) end - expect_initialized_span + allow(Datadog::CI::Span).to receive(:new).with(span_op).and_return(ci_span) trace end + it_behaves_like "initialize ci span with tags" it { expect(block_spy).to have_received(:call).with(ci_span) } it { is_expected.to be(block_result) } end @@ -173,7 +269,6 @@ def expect_initialized_span end let(:span_op) { Datadog::Tracing::SpanOperation.new(span_name) } - let(:ci_span) { instance_double(Datadog::CI::Span) } before do allow(Datadog::Tracing) @@ -187,11 +282,12 @@ def expect_initialized_span ) .and_return(span_op) - expect_initialized_span + allow(Datadog::CI::Span).to receive(:new).with(span_op).and_return(ci_span) trace end + it_behaves_like "initialize ci span with tags" it { is_expected.to be(ci_span) } end end @@ -251,4 +347,74 @@ def expect_initialized_span it { is_expected.to be_nil } end end + + describe "#start_test_session" do + context "when test suite level visibility is enabled" do + subject(:start_test_session) { recorder.start_test_session(service_name: service) } + + let(:session_operation_name) { "test.session" } + let(:span_op) { Datadog::Tracing::SpanOperation.new(session_operation_name) } + let(:expected_tags) { {"_test.session_id" => span_op.id} } + + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + session_operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION, + service: service + } + ) + .and_return(span_op) + + allow(Datadog::CI::TestSession).to receive(:new).with(span_op).and_return(ci_span) + + start_test_session + end + + it_behaves_like "internal tracing context" + it_behaves_like "initialize ci span with tags" + + it { is_expected.to be(ci_span) } + end + + context "when test suite level visibility is disabled" do + subject(:start_test_session) { recorder.start_test_session(service_name: service) } + + let(:test_suite_level_visibility_enabled) { false } + + before { start_test_session } + + it { is_expected.to be_nil } + end + end + + describe "#active_test_session" do + subject(:active_test_session) { recorder.active_test_session } + + let(:ci_session) do + recorder.start_test_session(service_name: service) + end + + before { ci_session } + + it { is_expected.to be(ci_session) } + end + + describe "#deactivate_test_session" do + subject(:deactivate_test_session) { recorder.deactivate_test_session } + + let(:ci_session) do + recorder.start_test_session(service_name: service) + end + + before do + ci_session + + deactivate_test_session + end + + it { expect(recorder.active_test_session).to be_nil } + end end diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index fe4983c3..6519d90c 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -87,8 +87,7 @@ describe "#set_tags" do it "sets the tags" do - expect(tracer_span).to receive(:set_tag).with("foo", "bar") - expect(tracer_span).to receive(:set_tag).with("baz", "qux") + expect(tracer_span).to receive(:set_tags).with({"foo" => "bar", "baz" => "qux"}) span.set_tags("foo" => "bar", "baz" => "qux") end @@ -111,11 +110,18 @@ end describe "#set_environment_runtime_tags" do + let(:test_command) { "command" } + + before do + allow(Datadog::CI::Utils::TestRun).to receive(:command).and_return(test_command) + end + it "sets the environment runtime tags" do expect(tracer_span).to receive(:set_tag).with("os.architecture", ::RbConfig::CONFIG["host_cpu"]) expect(tracer_span).to receive(:set_tag).with("os.platform", ::RbConfig::CONFIG["host_os"]) expect(tracer_span).to receive(:set_tag).with("runtime.name", Datadog::Core::Environment::Ext::LANG_ENGINE) expect(tracer_span).to receive(:set_tag).with("runtime.version", Datadog::Core::Environment::Ext::ENGINE_VERSION) + expect(tracer_span).to receive(:set_tag).with("test.command", test_command) span.set_environment_runtime_tags end diff --git a/spec/datadog/ci/test_session_spec.rb b/spec/datadog/ci/test_session_spec.rb new file mode 100644 index 00000000..97429d32 --- /dev/null +++ b/spec/datadog/ci/test_session_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +RSpec.describe Datadog::CI::TestSession do + let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, finish: true) } + + describe "#finish" do + subject(:ci_test_session) { described_class.new(tracer_span) } + + before { allow(Datadog::CI).to receive(:deactivate_test_session) } + + it "deactivates the test session" do + ci_test_session.finish + + expect(Datadog::CI).to have_received(:deactivate_test_session) + end + end + + describe "#inheritable_tags" do + subject(:inheritable_tags) { ci_test_session.inheritable_tags } + + let(:ci_test_session) { described_class.new(tracer_span) } + + before do + Datadog::CI::Ext::Test::INHERITABLE_TAGS.each do |tag| + allow(tracer_span).to receive(:get_tag).with(tag).and_return("value for #{tag}") + end + end + + it "returns a hash of inheritable tags" do + is_expected.to eq( + Datadog::CI::Ext::Test::INHERITABLE_TAGS.each_with_object({}) do |tag, memo| + memo[tag] = "value for #{tag}" + end + ) + end + end +end diff --git a/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb b/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb new file mode 100644 index 00000000..3bd9c945 --- /dev/null +++ b/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb @@ -0,0 +1,33 @@ +require_relative "../../../../../../lib/datadog/ci/test_visibility/serializers/factories/test_suite_level" +require_relative "../../../../../../lib/datadog/ci/test_visibility/serializers/test_v2" +require_relative "../../../../../../lib/datadog/ci/test_visibility/serializers/test_session" +require_relative "../../../../../../lib/datadog/ci/recorder" + +RSpec.describe Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + end + + before do + produce_test_session_trace(with_http_span: true) + end + + subject { described_class.serializer(trace, ci_span) } + + describe ".convert_trace_to_serializable_events" do + context "with a session span" do + let(:ci_span) { test_session_span } + it { is_expected.to be_kind_of(Datadog::CI::TestVisibility::Serializers::TestSession) } + end + + context "with a test span" do + let(:ci_span) { first_test_span } + it { is_expected.to be_kind_of(Datadog::CI::TestVisibility::Serializers::TestV2) } + end + + context "with a http request span" do + let(:ci_span) { first_other_span } + it { is_expected.to be_kind_of(Datadog::CI::TestVisibility::Serializers::Span) } + end + end +end diff --git a/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb new file mode 100644 index 00000000..bb05f5e9 --- /dev/null +++ b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb @@ -0,0 +1,83 @@ +require_relative "../../../../../lib/datadog/ci/test_visibility/serializers/test_session" +require_relative "../../../../../lib/datadog/ci/recorder" +require_relative "../../../../../lib/datadog/ci/ext/app_types" + +RSpec.describe Datadog::CI::TestVisibility::Serializers::TestSession do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + end + + include_context "Test visibility event serialized" do + subject { described_class.new(trace, test_session_span) } + end + + describe "#to_msgpack" do + context "traced a single test execution with Recorder" do + before do + produce_test_session_trace + end + + it "serializes test event to messagepack" do + expect_event_header(type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION) + + expect(content).to include( + { + "test_session_id" => test_session_span.id.to_s, + "name" => "rspec.test_session", + "service" => "rspec-test-suite", + "type" => Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION, + "resource" => "rspec.test_session.#{test_command}" + } + ) + + expect(meta).to include( + { + "test.command" => test_command, + "test.framework" => "rspec", + "test.status" => "pass", + "_dd.origin" => "ciapp-test" + } + ) + + expect(meta["_test.session_id"]).to be_nil + end + end + + context "trace a failed test" do + before do + produce_test_session_trace(result: "FAILED", exception: StandardError.new("1 + 2 are not equal to 5")) + end + + it "has error" do + expect_event_header(type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION) + + expect(content).to include({"error" => 1}) + expect(meta).to include({"test.status" => "fail"}) + end + end + end + + describe "#valid?" do + context "test_session_id" do + before do + produce_test_session_trace + end + + context "when test_session_id is not nil" do + it "returns true" do + expect(subject.valid?).to eq(true) + end + end + + context "when test_session_id is nil" do + before do + test_session_span.clear_tag("_test.session_id") + end + + it "returns false" do + expect(subject.valid?).to eq(false) + end + end + end + end +end diff --git a/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb b/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb new file mode 100644 index 00000000..46732848 --- /dev/null +++ b/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb @@ -0,0 +1,101 @@ +require_relative "../../../../../lib/datadog/ci/test_visibility/serializers/test_v2" +require_relative "../../../../../lib/datadog/ci/recorder" + +RSpec.describe Datadog::CI::TestVisibility::Serializers::TestV2 do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + end + + include_context "Test visibility event serialized" do + subject { described_class.new(trace, first_test_span) } + end + + describe "#to_msgpack" do + context "traced a single test execution with Recorder" do + before do + produce_test_session_trace + end + + it "serializes test event to messagepack" do + expect_event_header(version: 2) + + expect(content).to include( + { + "trace_id" => trace.id, + "span_id" => first_test_span.id, + "name" => "rspec.test", + "service" => "rspec-test-suite", + "type" => "test", + "resource" => "calculator_tests.test_add", + "test_session_id" => test_session_span.id.to_s + } + ) + + expect(meta).to include( + { + "test.framework" => "rspec", + "test.status" => "pass", + "_dd.origin" => "ciapp-test", + "test_owner" => "my_team" + } + ) + expect(meta["_test.session_id"]).to be_nil + + expect(metrics).to eq({"memory_allocations" => 16}) + end + end + + context "trace a failed test" do + before do + produce_test_session_trace(result: "FAILED", exception: StandardError.new("1 + 2 are not equal to 5")) + end + + it "has error" do + expect_event_header(version: 2) + + expect(content).to include({"error" => 1}) + expect(meta).to include({"test.status" => "fail"}) + end + end + + context "with time and duration expectations" do + let(:start_time) { Time.now } + let(:duration_seconds) { 3 } + + before do + produce_test_session_trace(start_time: start_time, duration_seconds: duration_seconds) + end + + it "correctly serializes start and duration in nanoseconds" do + expect(content).to include({ + "start" => start_time.to_i * 1_000_000_000 + start_time.nsec, + "duration" => 3 * 1_000_000_000 + }) + end + end + end + + describe "#valid?" do + context "test_session_id" do + before do + produce_test_session_trace + end + + context "when test_session_id is not nil" do + it "returns true" do + expect(subject.valid?).to eq(true) + end + end + + context "when test_session_id is nil" do + before do + first_test_span.clear_tag("_test.session_id") + end + + it "returns false" do + expect(subject.valid?).to eq(false) + end + end + end + end +end diff --git a/spec/datadog/ci/utils/test_run_spec.rb b/spec/datadog/ci/utils/test_run_spec.rb new file mode 100644 index 00000000..7f84b65e --- /dev/null +++ b/spec/datadog/ci/utils/test_run_spec.rb @@ -0,0 +1,9 @@ +require_relative "../../../../lib/datadog/ci/utils/test_run" + +RSpec.describe ::Datadog::CI::Utils::TestRun do + describe ".command" do + subject { described_class.command } + + it { is_expected.to eq("#{$0} #{ARGV.join(" ")}") } + end +end diff --git a/spec/datadog/ci_spec.rb b/spec/datadog/ci_spec.rb index bee49fbd..d8a71aa3 100644 --- a/spec/datadog/ci_spec.rb +++ b/spec/datadog/ci_spec.rb @@ -98,4 +98,38 @@ it { is_expected.to be_nil } end end + + describe "::start_test_session" do + subject(:start_test_session) { described_class.start_test_session } + + let(:ci_test_session) { instance_double(Datadog::CI::TestSession) } + + before do + allow(recorder).to receive(:start_test_session).and_return(ci_test_session) + end + + it { is_expected.to be(ci_test_session) } + end + + describe "::active_test_session" do + subject(:active_test_session) { described_class.active_test_session } + + let(:ci_test_session) { instance_double(Datadog::CI::TestSession) } + + before do + allow(recorder).to receive(:active_test_session).and_return(ci_test_session) + end + + it { is_expected.to be(ci_test_session) } + end + + describe "::deactivate_test_session" do + subject(:deactivate_test_session) { described_class.deactivate_test_session } + + before do + allow(recorder).to receive(:deactivate_test_session) + end + + it { is_expected.to be_nil } + end end diff --git a/spec/support/ci_mode_helpers.rb b/spec/support/ci_mode_helpers.rb index 9d8d678c..90739634 100644 --- a/spec/support/ci_mode_helpers.rb +++ b/spec/support/ci_mode_helpers.rb @@ -1,4 +1,5 @@ RSpec.shared_context "CI mode activated" do + let(:test_command) { "command" } let(:integration_name) { :override_me } let(:integration_options) { {} } @@ -7,8 +8,11 @@ receive(:endpoint?).with("/evp_proxy/v2/").and_return(true) ) + allow(Datadog::CI::Utils::TestRun).to receive(:command).and_return(test_command) + Datadog.configure do |c| c.ci.enabled = true + c.ci.experimental_test_suite_level_visibility_enabled = true c.ci.instrument integration_name, integration_options end end diff --git a/spec/support/tracer_helpers.rb b/spec/support/tracer_helpers.rb index 9eabe73a..a1ca5bd6 100644 --- a/spec/support/tracer_helpers.rb +++ b/spec/support/tracer_helpers.rb @@ -43,14 +43,7 @@ def produce_test_trace( Datadog::CI.active_test.set_tag("test_owner", "my_team") Datadog::CI.active_test.set_metric("memory_allocations", 16) - case result - when "FAILED" - test.failed!(exception: exception) - when "SKIPPED" - test.skipped!(exception: exception, reason: skip_reason) - else - test.passed! - end + set_result(test, result: result, exception: exception, skip_reason: skip_reason) Timecop.travel(start_time + duration_seconds) end @@ -58,12 +51,50 @@ def produce_test_trace( Timecop.return end + def produce_test_session_trace( + framework: "rspec", operation: "rspec.example", + test_name: "test_add", test_suite: "calculator_tests", + service: "rspec-test-suite", result: "PASSED", exception: nil, + skip_reason: nil, start_time: Time.now, duration_seconds: 2, + with_http_span: false + ) + allow(Process).to receive(:clock_gettime).and_return( + 0, duration_seconds, 2 * duration_seconds, 3 * duration_seconds + ) + + test_session = Datadog::CI.start_test_session( + service_name: service, + tags: { + Datadog::CI::Ext::Test::TAG_FRAMEWORK => framework, + Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION => "1.0.0", + Datadog::CI::Ext::Test::TAG_TYPE => "test" + } + ) + + produce_test_trace( + framework: framework, operation: operation, + test_name: test_name, test_suite: test_suite, + # service is inherited from test_session + service: nil, + result: result, exception: exception, skip_reason: skip_reason, + start_time: start_time, duration_seconds: duration_seconds, + with_http_span: with_http_span + ) + + set_result(test_session, result: result, exception: exception, skip_reason: skip_reason) + test_session.finish + end + + def test_session_span + spans.find { |span| span.type == "test_session_end" } + end + def first_test_span spans.find { |span| span.type == "test" } end def first_other_span - spans.find { |span| span.type != "test" } + spans.find { |span| !Datadog::CI::Ext::AppTypes::CI_SPAN_TYPES.include?(span.type) } end # Returns spans and caches it (similar to +let(:spans)+). @@ -127,6 +158,17 @@ def fetch_spans(tracer = self.tracer) end end + def set_result(span, result: "PASSED", exception: nil, skip_reason: nil) + case result + when "FAILED" + span.failed!(exception: exception) + when "SKIPPED" + span.skipped!(exception: exception, reason: skip_reason) + else + span.passed! + end + end + # Remove all traces from the current tracer instance and # busts cache of +#spans+ and +#span+. def clear_traces!