From 4ee83ae751e88bea36fd9f9c12e7d3c641a7694d Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 21 Nov 2023 13:35:32 +0100 Subject: [PATCH 01/16] add test session public API --- lib/datadog/ci.rb | 53 +++++++++ lib/datadog/ci/context/global.rb | 31 ++++++ lib/datadog/ci/ext/app_types.rb | 1 + lib/datadog/ci/recorder.rb | 61 ++++++++--- lib/datadog/ci/test.rb | 1 - lib/datadog/ci/test_session.rb | 27 +++++ .../ci/test_visibility/serializers/test_v1.rb | 2 +- sig/datadog/ci.rbs | 6 ++ sig/datadog/ci/context/global.rbs | 19 ++++ sig/datadog/ci/ext/app_types.rbs | 3 +- sig/datadog/ci/recorder.rbs | 11 ++ sig/datadog/ci/test_session.rbs | 7 ++ .../test_visibility/serializers/test_v1.rbs | 2 +- spec/datadog/ci/context/global_spec.rb | 51 +++++++++ spec/datadog/ci/recorder_spec.rb | 101 +++++++++++++----- spec/datadog/ci/test_session_spec.rb | 17 +++ spec/datadog/ci_spec.rb | 34 ++++++ 17 files changed, 385 insertions(+), 42 deletions(-) create mode 100644 lib/datadog/ci/context/global.rb create mode 100644 lib/datadog/ci/test_session.rb create mode 100644 sig/datadog/ci/context/global.rbs create mode 100644 sig/datadog/ci/test_session.rbs create mode 100644 spec/datadog/ci/context/global_spec.rb create mode 100644 spec/datadog/ci/test_session_spec.rb diff --git a/lib/datadog/ci.rb b/lib/datadog/ci.rb index 7bfd382e..96dc9347 100644 --- a/lib/datadog/ci.rb +++ b/lib/datadog/ci.rb @@ -10,6 +10,54 @@ module Datadog # @public_api module CI class << self + # Return a {Datadog::CI::TestSesstion ci_test_session} that represents the whole test session run. + # Raises an error if a session is already active. + # + # + # The {#start_test_session} method is used to mark the start : + # ``` + # Datadog::CI.start_test_session( + # service: "my-web-site-tests", + # tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" } + # ) + # + # # Somewhere else after test run has ended + # Datadog::CI.active_test_session.finish + # ``` + # + # Remember that calling {Datadog::CI::TestSession#finish} is mandatory. + # + # @param [String] service_name the service name for this session + # @param [Hash] tags extra tags which should be added to the test. + # @return [Datadog::CI::TestSession] returns the active, running {Datadog::CI::TestSession}. + # + # @public_api + def start_test_session(service_name: nil, tags: {}) + recorder.start_test_session(service_name: service_name, tags: tags) + end + + # The active, unfinished test session span. + # + # Usage: + # + # ``` + # # start a test session + # Datadog::CI.start_test_session( + # service: "my-web-site-tests", + # tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" } + # ) + # + # # somewhere else, access the session + # test_session = Datadog::CI.active_test_session + # test_session.finish + # ``` + # + # @return [Datadog::CI::TestSession] the active test session + # @return [nil] if no test session is active + def active_test_session + recorder.active_test_session + end + # Return a {Datadog::CI::Test ci_test} that will trace a test called `test_name`. # Raises an error if a test is already active. # @@ -189,6 +237,11 @@ def deactivate_test(test) recorder.deactivate_test(test) end + # Internal only, to finish a test session use Datadog::CI::TestSession#finish + def deactivate_test_session + recorder.deactivate_test_session + end + private def components diff --git a/lib/datadog/ci/context/global.rb b/lib/datadog/ci/context/global.rb new file mode 100644 index 00000000..0f7ebfdd --- /dev/null +++ b/lib/datadog/ci/context/global.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Context + # This context is shared between threads and represents the current test session. + class Global + def initialize + @mutex = Mutex.new + @test_session = nil + end + + def active_test_session + @test_session + end + + def activate_test_session!(test_session) + @mutex.synchronize do + raise "Nested test sessions are not supported. Currently active test session: #{@test_session}" unless @test_session.nil? + + @test_session = test_session + end + end + + def deactivate_test_session! + @mutex.synchronize { @test_session = nil } + end + end + end + end +end diff --git a/lib/datadog/ci/ext/app_types.rb b/lib/datadog/ci/ext/app_types.rb index e83ad3c3..f28230a7 100644 --- a/lib/datadog/ci/ext/app_types.rb +++ b/lib/datadog/ci/ext/app_types.rb @@ -5,6 +5,7 @@ module CI module Ext module AppTypes TYPE_TEST = "test" + TYPE_TEST_SESSION = "test_session_end" end end end diff --git a/lib/datadog/ci/recorder.rb b/lib/datadog/ci/recorder.rb index 0d41bd9a..e6fe12dd 100644 --- a/lib/datadog/ci/recorder.rb +++ b/lib/datadog/ci/recorder.rb @@ -8,10 +8,12 @@ require_relative "ext/test" require_relative "ext/environment" +require_relative "context/global" require_relative "context/local" require_relative "span" require_relative "test" +require_relative "test_session" module Datadog module CI @@ -22,6 +24,24 @@ class Recorder def initialize @environment_tags = Ext::Environment.tags(ENV).freeze @local_context = Context::Local.new + @global_context = Context::Global.new + end + + def start_test_session(service_name: nil, tags: {}) + span_options = { + service: service_name, + span_type: Ext::AppTypes::TYPE_TEST_SESSION + } + + tracer_span = Datadog::Tracing.trace("test.session", **span_options) + trace = Datadog::Tracing.active_trace + + set_trace_origin(trace) + + test_session = build_test_session(tracer_span, tags) + @global_context.activate_test_session!(test_session) + + test_session end # Creates a new span for a CI test @@ -73,17 +93,26 @@ def trace(span_type, span_name, tags: {}, &block) end end + def active_span + tracer_span = Datadog::Tracing.active_span + Span.new(tracer_span) if tracer_span + end + def active_test @local_context.active_test end + def active_test_session + @global_context.active_test_session + end + + # TODO: does it make sense to have a paramter here? def deactivate_test(test) @local_context.deactivate_test!(test) end - def active_span - tracer_span = Datadog::Tracing.active_span - Span.new(tracer_span) if tracer_span + def deactivate_test_session + @global_context.deactivate_test_session! end private @@ -93,26 +122,30 @@ def set_trace_origin(trace) trace.origin = Ext::Test::CONTEXT_ORIGIN if trace end + def build_test_session(tracer_span, tags) + test_session = TestSession.new(tracer_span) + set_initial_tags(test_session, tags) + test_session + end + def build_test(tracer_span, tags) test = Test.new(tracer_span) - - test.set_default_tags - test.set_environment_runtime_tags - - test.set_tags(tags) - test.set_tags(environment_tags) - + set_initial_tags(test, tags) test end def build_span(tracer_span, tags) span = Span.new(tracer_span) + set_initial_tags(span, tags) + span + end - span.set_default_tags - span.set_environment_runtime_tags - span.set_tags(tags) + def set_initial_tags(ci_span, tags) + ci_span.set_default_tags + ci_span.set_environment_runtime_tags - span + ci_span.set_tags(tags) + ci_span.set_tags(environment_tags) end end end diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb index 6cf43de2..d0cc5187 100644 --- a/lib/datadog/ci/test.rb +++ b/lib/datadog/ci/test.rb @@ -5,7 +5,6 @@ module Datadog module CI # Represents a single part of a test run. - # Could be a session, suite, test, or any custom span. # # @public_api class Test < Span diff --git a/lib/datadog/ci/test_session.rb b/lib/datadog/ci/test_session.rb new file mode 100644 index 00000000..c6e49217 --- /dev/null +++ b/lib/datadog/ci/test_session.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require_relative "span" + +module Datadog + module CI + # Represents the whole test session process. + # This object can be shared between multiple threads. + # + # @public_api + class TestSession < Span + def initialize(tracer_span) + super + + @mutex = Mutex.new + end + + # Finishes the current test session. + # @return [void] + def finish + super + + CI.deactivate_test_session + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/serializers/test_v1.rb b/lib/datadog/ci/test_visibility/serializers/test_v1.rb index c2547234..ed9fb740 100644 --- a/lib/datadog/ci/test_visibility/serializers/test_v1.rb +++ b/lib/datadog/ci/test_visibility/serializers/test_v1.rb @@ -37,7 +37,7 @@ def content_map_size end def type - "test" + Ext::AppTypes::TYPE_TEST end def name diff --git a/sig/datadog/ci.rbs b/sig/datadog/ci.rbs index 9a1b76b4..2340fccd 100644 --- a/sig/datadog/ci.rbs +++ b/sig/datadog/ci.rbs @@ -4,14 +4,20 @@ module Datadog def self.start_test: (String span_name, ?service_name: String?, ?operation_name: String, ?tags: Hash[untyped, untyped]) -> Datadog::CI::Test + def self.start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession + def self.trace: (String span_type, String span_name, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped + def self.active_test_session: () -> Datadog::CI::TestSession? + def self.active_test: () -> Datadog::CI::Test? def self.active_span: (String span_type) -> Datadog::CI::Span? def self.deactivate_test: (Datadog::CI::Test test) -> void + def self.deactivate_test_session: () -> void + def self.components: () -> Datadog::CI::Configuration::Components def self.recorder: () -> Datadog::CI::Recorder diff --git a/sig/datadog/ci/context/global.rbs b/sig/datadog/ci/context/global.rbs new file mode 100644 index 00000000..ce319b46 --- /dev/null +++ b/sig/datadog/ci/context/global.rbs @@ -0,0 +1,19 @@ +module Datadog + module CI + module Context + class Global + @mutex: Thread::Mutex + + @test_session: Datadog::CI::TestSession? + + def initialize: () -> void + + def active_test_session: () -> Datadog::CI::TestSession? + + def activate_test_session!: (Datadog::CI::TestSession test_session) -> void + + def deactivate_test_session!: () -> void + end + end + end +end diff --git a/sig/datadog/ci/ext/app_types.rbs b/sig/datadog/ci/ext/app_types.rbs index 8188d06c..83cd3e79 100644 --- a/sig/datadog/ci/ext/app_types.rbs +++ b/sig/datadog/ci/ext/app_types.rbs @@ -2,7 +2,8 @@ module Datadog module CI module Ext module AppTypes - TYPE_TEST: String + TYPE_TEST: "test" + TYPE_TEST_SESSION: "test_session_end" end end end diff --git a/sig/datadog/ci/recorder.rbs b/sig/datadog/ci/recorder.rbs index ff06b0c7..6bafc806 100644 --- a/sig/datadog/ci/recorder.rbs +++ b/sig/datadog/ci/recorder.rbs @@ -3,6 +3,7 @@ module Datadog class Recorder @environment_tags: Hash[String, String] @local_context: Datadog::CI::Context::Local + @global_context: Datadog::CI::Context::Global attr_reader environment_tags: Hash[String, String] @@ -10,12 +11,18 @@ module Datadog def trace: (String span_type, String span_name, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped + def start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession + + def active_test_session: () -> Datadog::CI::TestSession? + def active_test: () -> Datadog::CI::Test? def active_span: () -> Datadog::CI::Span? def deactivate_test: (Datadog::CI::Test test) -> void + def deactivate_test_session: () -> void + def create_datadog_span: (String span_name, ?span_options: Hash[untyped, untyped], ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped def set_trace_origin: (Datadog::Tracing::TraceOperation trace) -> untyped @@ -24,7 +31,11 @@ module Datadog def build_test: (Datadog::Tracing::SpanOperation tracer_span, Hash[untyped, untyped] tags) -> Datadog::CI::Test + def build_test_session: (Datadog::Tracing::SpanOperation tracer_span, Hash[untyped, untyped] tags) -> Datadog::CI::TestSession + def build_span: (Datadog::Tracing::SpanOperation tracer_span, Hash[untyped, untyped] tags) -> Datadog::CI::Span + + def set_initial_tags: (Datadog::CI::Span ci_span, Hash[untyped, untyped] tags) -> void end end end diff --git a/sig/datadog/ci/test_session.rbs b/sig/datadog/ci/test_session.rbs new file mode 100644 index 00000000..9f439e47 --- /dev/null +++ b/sig/datadog/ci/test_session.rbs @@ -0,0 +1,7 @@ +module Datadog + module CI + class TestSession < Span + @mutex: Thread::Mutex + end + end +end diff --git a/sig/datadog/ci/test_visibility/serializers/test_v1.rbs b/sig/datadog/ci/test_visibility/serializers/test_v1.rbs index a590d687..3c365885 100644 --- a/sig/datadog/ci/test_visibility/serializers/test_v1.rbs +++ b/sig/datadog/ci/test_visibility/serializers/test_v1.rbs @@ -11,7 +11,7 @@ module Datadog def content_fields: () -> Array[String | Hash[String, String]] def content_map_size: () -> Integer - def type: () -> "test" + def type: () -> ::String def name: () -> ::String diff --git a/spec/datadog/ci/context/global_spec.rb b/spec/datadog/ci/context/global_spec.rb new file mode 100644 index 00000000..a9330479 --- /dev/null +++ b/spec/datadog/ci/context/global_spec.rb @@ -0,0 +1,51 @@ +RSpec.describe Datadog::CI::Context::Global do + subject { described_class.new } + + let(:tracer_span) { double(Datadog::Tracing::SpanOperation, name: "test.session") } + let(:session) { Datadog::CI::TestSession.new(tracer_span) } + + describe "#activate_test_session!" do + context "when a test session is already active" do + before do + subject.activate_test_session!(Datadog::CI::TestSession.new(tracer_span)) + end + + it "raises an error" do + expect { subject.activate_test_session!(session) }.to( + raise_error( + RuntimeError, + "Nested test sessions are not supported. Currently active test session: " \ + "#{session}" + ) + ) + end + end + + context "when no test session is active" do + it "activates the test session" do + subject.activate_test_session!(session) + expect(subject.active_test_session).to be(session) + end + end + end + + describe "#deactivate_test_session!" do + context "when a test session is active" do + before do + subject.activate_test_session!(session) + end + + it "deactivates the test session" do + subject.deactivate_test_session! + expect(subject.active_test_session).to be_nil + end + end + + context "when no test session is active" do + it "does nothing" do + subject.deactivate_test_session! + expect(subject.active_test_session).to be_nil + end + end + end +end diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index 7f4ee337..c14f514a 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -6,6 +6,10 @@ let(:tags) { {} } let(:environment_tags) { Datadog::CI::Ext::Environment.tags(ENV) } + let(:ci_span) do + spy("CI object spy") + end + subject(:recorder) { described_class.new } before do @@ -21,15 +25,16 @@ end end - describe "#trace_test" do - def expect_initialized_test - allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_test) - expect(ci_test).to receive(:set_default_tags) - expect(ci_test).to receive(:set_environment_runtime_tags) - expect(ci_test).to receive(:set_tags).with(tags) - expect(ci_test).to receive(:set_tags).with(environment_tags) + shared_examples_for "initialize ci span with tags" do + it do + expect(ci_span).to have_received(:set_default_tags) + expect(ci_span).to have_received(:set_environment_runtime_tags) + expect(ci_span).to have_received(:set_tags).with(tags) + expect(ci_span).to have_received(:set_tags).with(environment_tags) end + end + describe "#trace_test" do context "when given a block" do subject(:trace) do recorder.trace_test( @@ -42,7 +47,6 @@ def expect_initialized_test end let(:span_op) { Datadog::Tracing::SpanOperation.new(operation_name) } - let(:ci_test) { instance_double(Datadog::CI::Test) } let(:block) { proc { |s| block_spy.call(s) } } let(:block_result) { double("result") } let(:block_spy) { spy("block") } @@ -61,7 +65,7 @@ def expect_initialized_test } ) - expect_initialized_test + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) trace_block.call(span_op, trace_op) end @@ -70,7 +74,9 @@ def expect_initialized_test end it_behaves_like "internal tracing context" - it { expect(block_spy).to have_received(:call).with(ci_test) } + it_behaves_like "initialize ci span with tags" + + it { expect(block_spy).to have_received(:call).with(ci_span) } it { is_expected.to be(block_result) } end @@ -84,7 +90,6 @@ def expect_initialized_test ) end let(:span_op) { Datadog::Tracing::SpanOperation.new(operation_name) } - let(:ci_test) { instance_double(Datadog::CI::Test) } before do allow(Datadog::Tracing) @@ -99,24 +104,18 @@ def expect_initialized_test ) .and_return(span_op) - expect_initialized_test + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) trace end it_behaves_like "internal tracing context" - it { is_expected.to be(ci_test) } + it_behaves_like "initialize ci span with tags" + it { is_expected.to be(ci_span) } end end describe "#trace" do - def expect_initialized_span - allow(Datadog::CI::Span).to receive(:new).with(span_op).and_return(ci_span) - expect(ci_span).to receive(:set_default_tags) - expect(ci_span).to receive(:set_environment_runtime_tags) - expect(ci_span).to receive(:set_tags).with(tags) - end - let(:tags) { {"my_tag" => "my_value"} } let(:span_type) { "step" } let(:span_name) { "span name" } @@ -134,7 +133,6 @@ def expect_initialized_span end let(:span_op) { Datadog::Tracing::SpanOperation.new(span_name) } - let(:ci_span) { instance_double(Datadog::CI::Span) } let(:block) { proc { |s| block_spy.call(s) } } let(:block_result) { double("result") } let(:block_spy) { spy("block") } @@ -154,11 +152,12 @@ def expect_initialized_span trace_block.call(span_op, trace_op) end - expect_initialized_span + allow(Datadog::CI::Span).to receive(:new).with(span_op).and_return(ci_span) trace end + it_behaves_like "initialize ci span with tags" it { expect(block_spy).to have_received(:call).with(ci_span) } it { is_expected.to be(block_result) } end @@ -173,7 +172,6 @@ def expect_initialized_span end let(:span_op) { Datadog::Tracing::SpanOperation.new(span_name) } - let(:ci_span) { instance_double(Datadog::CI::Span) } before do allow(Datadog::Tracing) @@ -187,11 +185,12 @@ def expect_initialized_span ) .and_return(span_op) - expect_initialized_span + allow(Datadog::CI::Span).to receive(:new).with(span_op).and_return(ci_span) trace end + it_behaves_like "initialize ci span with tags" it { is_expected.to be(ci_span) } end end @@ -251,4 +250,58 @@ def expect_initialized_span it { is_expected.to be_nil } end end + + describe "#start_test_session" do + subject(:start_test_session) { recorder.start_test_session(service_name: service) } + + let(:session_operation_name) { "test.session" } + + let(:span_op) { Datadog::Tracing::SpanOperation.new(session_operation_name) } + + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + session_operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION, + service: service + } + ) + .and_return(span_op) + + allow(Datadog::CI::TestSession).to receive(:new).with(span_op).and_return(ci_span) + + start_test_session + end + + it_behaves_like "internal tracing context" + it_behaves_like "initialize ci span with tags" + + it { is_expected.to be(ci_span) } + end + + describe "#active_test_session" do + subject(:active_test_session) { recorder.active_test_session } + + let(:ci_session) do + recorder.start_test_session(service_name: service) + end + + before { ci_session } + + it { is_expected.to be(ci_session) } + end + + describe "#deactivate_test_session" do + subject(:deactivate_test_session) { recorder.deactivate_test_session } + + let(:ci_session) do + recorder.start_test_session(service_name: service) + end + + before { deactivate_test_session } + + it { expect(recorder.active_test_session).to be_nil } + end end diff --git a/spec/datadog/ci/test_session_spec.rb b/spec/datadog/ci/test_session_spec.rb new file mode 100644 index 00000000..e2a6e148 --- /dev/null +++ b/spec/datadog/ci/test_session_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +RSpec.describe Datadog::CI::TestSession do + let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, finish: true) } + + describe "#finish" do + subject(:ci_test_session) { described_class.new(tracer_span) } + + before { allow(Datadog::CI).to receive(:deactivate_test_session) } + + it "deactivates the test" do + ci_test_session.finish + + expect(Datadog::CI).to have_received(:deactivate_test_session) + end + end +end diff --git a/spec/datadog/ci_spec.rb b/spec/datadog/ci_spec.rb index bee49fbd..d8a71aa3 100644 --- a/spec/datadog/ci_spec.rb +++ b/spec/datadog/ci_spec.rb @@ -98,4 +98,38 @@ it { is_expected.to be_nil } end end + + describe "::start_test_session" do + subject(:start_test_session) { described_class.start_test_session } + + let(:ci_test_session) { instance_double(Datadog::CI::TestSession) } + + before do + allow(recorder).to receive(:start_test_session).and_return(ci_test_session) + end + + it { is_expected.to be(ci_test_session) } + end + + describe "::active_test_session" do + subject(:active_test_session) { described_class.active_test_session } + + let(:ci_test_session) { instance_double(Datadog::CI::TestSession) } + + before do + allow(recorder).to receive(:active_test_session).and_return(ci_test_session) + end + + it { is_expected.to be(ci_test_session) } + end + + describe "::deactivate_test_session" do + subject(:deactivate_test_session) { described_class.deactivate_test_session } + + before do + allow(recorder).to receive(:deactivate_test_session) + end + + it { is_expected.to be_nil } + end end From 9b72294dcc045a0ae608ef3354da49628dbd7cfc Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 22 Nov 2023 17:23:16 +0100 Subject: [PATCH 02/16] introduce concurrency-safe Span class and inherit service name from the active test session in #trace_test --- lib/datadog/ci.rb | 6 +- lib/datadog/ci/concurrent_span.rb | 88 ++++++++++++++++++++ lib/datadog/ci/ext/test.rb | 3 + lib/datadog/ci/recorder.rb | 5 ++ lib/datadog/ci/span.rb | 5 ++ lib/datadog/ci/test_session.rb | 10 +-- sig/datadog/ci/concurrent_span.rbs | 23 ++++++ sig/datadog/ci/ext/test.rbs | 3 + sig/datadog/ci/span.rbs | 2 + sig/datadog/ci/test_session.rbs | 3 +- spec/datadog/ci/recorder_spec.rb | 125 ++++++++++++++++++++++------- 11 files changed, 234 insertions(+), 39 deletions(-) create mode 100644 lib/datadog/ci/concurrent_span.rb create mode 100644 sig/datadog/ci/concurrent_span.rbs diff --git a/lib/datadog/ci.rb b/lib/datadog/ci.rb index 96dc9347..0d7a1a9f 100644 --- a/lib/datadog/ci.rb +++ b/lib/datadog/ci.rb @@ -14,7 +14,7 @@ class << self # Raises an error if a session is already active. # # - # The {#start_test_session} method is used to mark the start : + # The {#start_test_session} method is used to mark the start of the test session: # ``` # Datadog::CI.start_test_session( # service: "my-web-site-tests", @@ -59,7 +59,9 @@ def active_test_session end # Return a {Datadog::CI::Test ci_test} that will trace a test called `test_name`. - # Raises an error if a test is already active. + # Raises an error if a test is already active. If there is an active test session, + # the new test will be connected to the session. The test will inherit service name and tags from + # the running test session if none provided. # # You could trace your test using a do-block like: # diff --git a/lib/datadog/ci/concurrent_span.rb b/lib/datadog/ci/concurrent_span.rb new file mode 100644 index 00000000..9910d0dc --- /dev/null +++ b/lib/datadog/ci/concurrent_span.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require_relative "span" + +module Datadog + module CI + # Represents a single part of a test run that can be safely shared between threads. + # Examples of shared objects are: TestSession, TestModule, TestSpan. + # + # @public_api + class ConcurrentSpan < Span + def initialize(tracer_span) + super + + @mutex = Mutex.new + end + + # Sets the status of the span to "pass". This method is thread-safe. + # @return [void] + def passed! + synchronize { super } + end + + # Sets the status of the span to "fail". This method is thread-safe. + # @param [Exception] exception the exception that caused the test to fail. + # @return [void] + def failed!(exception: nil) + synchronize { super } + end + + # Sets the status of the span to "skip". This method is thread-safe. + # @param [Exception] exception the exception that caused the test to fail. + # @param [String] reason the reason why the test was skipped. + # @return [void] + def skipped!(exception: nil, reason: nil) + synchronize { super } + end + + # Gets tag value by key. This method is thread-safe. + # @param [String] key the key of the tag. + # @return [String] the value of the tag. + def get_tag(key) + synchronize { super } + end + + # Sets tag value by key. This method is thread-safe. + # @param [String] key the key of the tag. + # @param [String] value the value of the tag. + # @return [void] + def set_tag(key, value) + synchronize { super } + end + + # Sets metric value by key. This method is thread-safe. + # @param [String] key the key of the metric. + # @param [Numeric] value the value of the metric. + # @return [void] + def set_metric(key, value) + synchronize { super } + end + + # Finishes the span. This method is thread-safe. + # @return [void] + def finish + synchronize { super } + end + + # Sets multiple tags at once. This method is thread-safe. + # @param [Hash[String, String]] tags the tags to set. + # @return [void] + def set_tags(tags) + synchronize { super } + end + + def set_environment_runtime_tags + synchronize { super } + end + + def set_default_tags + synchronize { super } + end + + def synchronize + @mutex.synchronize { yield } + end + end + end +end diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index af7199fb..5f062705 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -17,6 +17,9 @@ module Test TAG_TRAITS = "test.traits" TAG_TYPE = "test.type" + # tags that are inherited from the test session + INHERITED_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze + # Environment runtime tags TAG_OS_ARCHITECTURE = "os.architecture" TAG_OS_PLATFORM = "os.platform" diff --git a/lib/datadog/ci/recorder.rb b/lib/datadog/ci/recorder.rb index e6fe12dd..b076b38f 100644 --- a/lib/datadog/ci/recorder.rb +++ b/lib/datadog/ci/recorder.rb @@ -46,6 +46,11 @@ def start_test_session(service_name: nil, tags: {}) # Creates a new span for a CI test def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, &block) + test_session = active_test_session + if test_session + service_name ||= test_session.service + end + span_options = { resource: test_name, service: service_name, diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb index 19e45a85..4c77fbb1 100644 --- a/lib/datadog/ci/span.rb +++ b/lib/datadog/ci/span.rb @@ -20,6 +20,11 @@ def name tracer_span.name end + # @return [String] the service name of the span. + def service + tracer_span.service + end + # @return [String] the type of the span (for example "test" or type that was provided to [Datadog::CI.trace]). def span_type tracer_span.type diff --git a/lib/datadog/ci/test_session.rb b/lib/datadog/ci/test_session.rb index c6e49217..2adede5d 100644 --- a/lib/datadog/ci/test_session.rb +++ b/lib/datadog/ci/test_session.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "span" +require_relative "concurrent_span" module Datadog module CI @@ -8,13 +8,7 @@ module CI # This object can be shared between multiple threads. # # @public_api - class TestSession < Span - def initialize(tracer_span) - super - - @mutex = Mutex.new - end - + class TestSession < ConcurrentSpan # Finishes the current test session. # @return [void] def finish diff --git a/sig/datadog/ci/concurrent_span.rbs b/sig/datadog/ci/concurrent_span.rbs new file mode 100644 index 00000000..ff834140 --- /dev/null +++ b/sig/datadog/ci/concurrent_span.rbs @@ -0,0 +1,23 @@ +module Datadog + module CI + class ConcurrentSpan < Span + @mutex: Thread::Mutex + + def initialize: (Datadog::Tracing::SpanOperation tracer_span) -> void + def passed!: () -> void + def failed!: (?exception: untyped?) -> void + def skipped!: (?exception: untyped?, ?reason: String?) -> void + def get_tag: (String key) -> untyped? + def set_tag: (String key, untyped? value) -> void + def set_metric: (String key, untyped value) -> void + def finish: () -> void + def set_tags: (Hash[untyped, untyped] tags) -> void + + def set_environment_runtime_tags: () -> void + + def set_default_tags: () -> void + + def synchronize: () { () -> untyped } -> untyped + end + end +end diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index 6f0c0c0c..dada2f87 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -21,6 +21,9 @@ module Datadog TAG_TRAITS: String TAG_TYPE: String + + INHERITED_TAGS: Array[String] + TAG_OS_ARCHITECTURE: String TAG_OS_PLATFORM: String diff --git a/sig/datadog/ci/span.rbs b/sig/datadog/ci/span.rbs index 32ae69a9..b81df6ca 100644 --- a/sig/datadog/ci/span.rbs +++ b/sig/datadog/ci/span.rbs @@ -9,6 +9,8 @@ module Datadog def name: () -> String + def service: () -> String + def passed!: () -> void def failed!: (?exception: untyped?) -> void diff --git a/sig/datadog/ci/test_session.rbs b/sig/datadog/ci/test_session.rbs index 9f439e47..32d99305 100644 --- a/sig/datadog/ci/test_session.rbs +++ b/sig/datadog/ci/test_session.rbs @@ -1,7 +1,6 @@ module Datadog module CI - class TestSession < Span - @mutex: Thread::Mutex + class TestSession < ConcurrentSpan end end end diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index c14f514a..f511e4e9 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -4,6 +4,7 @@ let(:operation_name) { "span name" } let(:test_name) { "test name" } let(:tags) { {} } + let(:expected_tags) { tags } let(:environment_tags) { Datadog::CI::Ext::Environment.tags(ENV) } let(:ci_span) do @@ -29,7 +30,7 @@ it do expect(ci_span).to have_received(:set_default_tags) expect(ci_span).to have_received(:set_environment_runtime_tags) - expect(ci_span).to have_received(:set_tags).with(tags) + expect(ci_span).to have_received(:set_tags).with(expected_tags) expect(ci_span).to have_received(:set_tags).with(environment_tags) end end @@ -81,37 +82,109 @@ end context "when not given a block" do - subject(:trace) do - recorder.trace_test( - test_name, - service_name: service, - operation_name: operation_name, - tags: tags - ) - end let(:span_op) { Datadog::Tracing::SpanOperation.new(operation_name) } - before do - allow(Datadog::Tracing) - .to receive(:trace) - .with( - operation_name, - { - span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, - resource: test_name, - service: service - } + context "without active test session" do + subject(:trace) do + recorder.trace_test( + test_name, + service_name: service, + operation_name: operation_name, + tags: tags ) - .and_return(span_op) + end - allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, + resource: test_name, + service: service + } + ) + .and_return(span_op) - trace + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) + + trace + end + + it_behaves_like "internal tracing context" + it_behaves_like "initialize ci span with tags" + it { is_expected.to be(ci_span) } end - it_behaves_like "internal tracing context" - it_behaves_like "initialize ci span with tags" - it { is_expected.to be(ci_span) } + context "when test session is active" do + let(:test_session_service) { "my-test-service" } + let(:test_session) { instance_double(Datadog::CI::TestSession, service: test_session_service) } + before do + allow(recorder).to receive(:active_test_session).and_return(test_session) + end + + context "when service name is not given" do + subject(:trace) do + recorder.trace_test( + test_name, + operation_name: operation_name, + tags: tags + ) + end + + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, + resource: test_name, + service: test_session_service + } + ) + .and_return(span_op) + + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) + + trace + end + + it { is_expected.to be(ci_span) } + end + + context "when service name is given" do + subject(:trace) do + recorder.trace_test( + test_name, + service_name: service, + operation_name: operation_name, + tags: tags + ) + end + + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST, + resource: test_name, + service: service + } + ) + .and_return(span_op) + + allow(Datadog::CI::Test).to receive(:new).with(span_op).and_return(ci_span) + + trace + end + + it { is_expected.to be(ci_span) } + end + end end end @@ -120,8 +193,6 @@ let(:span_type) { "step" } let(:span_name) { "span name" } - let(:expected_tags) { tags } - context "when given a block" do subject(:trace) do recorder.trace( From 7fa3b9ff34c9523d7edc5afc24422599eaf3bf27 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 22 Nov 2023 17:42:50 +0100 Subject: [PATCH 03/16] inherit tags from the running test session when starting a test --- lib/datadog/ci/ext/test.rb | 4 ++-- lib/datadog/ci/recorder.rb | 2 ++ lib/datadog/ci/test_session.rb | 9 +++++++++ sig/datadog/ci/ext/test.rbs | 2 +- sig/datadog/ci/test_session.rbs | 1 + spec/datadog/ci/recorder_spec.rb | 22 ++++++++++++++++++---- 6 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index 5f062705..06f450c0 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -17,8 +17,8 @@ module Test TAG_TRAITS = "test.traits" TAG_TYPE = "test.type" - # tags that are inherited from the test session - INHERITED_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze + # tags that can be inherited from the test session + INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze # Environment runtime tags TAG_OS_ARCHITECTURE = "os.architecture" diff --git a/lib/datadog/ci/recorder.rb b/lib/datadog/ci/recorder.rb index b076b38f..b38d5f45 100644 --- a/lib/datadog/ci/recorder.rb +++ b/lib/datadog/ci/recorder.rb @@ -49,6 +49,8 @@ def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, & test_session = active_test_session if test_session service_name ||= test_session.service + + tags = test_session.inheritable_tags.merge(tags) end span_options = { diff --git a/lib/datadog/ci/test_session.rb b/lib/datadog/ci/test_session.rb index 2adede5d..a8ca98cc 100644 --- a/lib/datadog/ci/test_session.rb +++ b/lib/datadog/ci/test_session.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "concurrent_span" +require_relative "ext/test" module Datadog module CI @@ -16,6 +17,14 @@ def finish CI.deactivate_test_session end + + def inheritable_tags + res = {} + Ext::Test::INHERITABLE_TAGS.each do |tag| + res[tag] = get_tag(tag) + end + res + end end end end diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index dada2f87..9916f363 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -22,7 +22,7 @@ module Datadog TAG_TYPE: String - INHERITED_TAGS: Array[String] + INHERITABLE_TAGS: Array[String] TAG_OS_ARCHITECTURE: String diff --git a/sig/datadog/ci/test_session.rbs b/sig/datadog/ci/test_session.rbs index 32d99305..4f89722d 100644 --- a/sig/datadog/ci/test_session.rbs +++ b/sig/datadog/ci/test_session.rbs @@ -1,6 +1,7 @@ module Datadog module CI class TestSession < ConcurrentSpan + def inheritable_tags: () -> Hash[untyped, untyped] end end end diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index f511e4e9..55f51188 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -4,7 +4,7 @@ let(:operation_name) { "span name" } let(:test_name) { "test name" } let(:tags) { {} } - let(:expected_tags) { tags } + let(:expected_tags) { {} } let(:environment_tags) { Datadog::CI::Ext::Environment.tags(ENV) } let(:ci_span) do @@ -36,6 +36,8 @@ end describe "#trace_test" do + let(:expected_tags) { {"test.name" => test_name} } + context "when given a block" do subject(:trace) do recorder.trace_test( @@ -118,13 +120,19 @@ end context "when test session is active" do + let(:inheritable_tags) { {"test.framework" => "my framework"} } let(:test_session_service) { "my-test-service" } - let(:test_session) { instance_double(Datadog::CI::TestSession, service: test_session_service) } + let(:test_session) do + instance_double(Datadog::CI::TestSession, service: test_session_service, inheritable_tags: inheritable_tags) + end + before do allow(recorder).to receive(:active_test_session).and_return(test_session) end - context "when service name is not given" do + context "when service name and tags are not given" do + let(:expected_tags) { {"test.framework" => "my framework", "test.name" => test_name} } + subject(:trace) do recorder.trace_test( test_name, @@ -151,10 +159,14 @@ trace end + it_behaves_like "initialize ci span with tags" it { is_expected.to be(ci_span) } end - context "when service name is given" do + context "when service name and tags are given" do + let(:tags) { {"test.framework" => "special test framework"} } + let(:expected_tags) { {"test.framework" => "special test framework", "test.name" => test_name} } + subject(:trace) do recorder.trace_test( test_name, @@ -182,6 +194,7 @@ trace end + it_behaves_like "initialize ci span with tags" it { is_expected.to be(ci_span) } end end @@ -190,6 +203,7 @@ describe "#trace" do let(:tags) { {"my_tag" => "my_value"} } + let(:expected_tags) { {"my_tag" => "my_value"} } let(:span_type) { "step" } let(:span_name) { "span name" } From 68d9935304bbd0ad886dbb8d5e6a09e8ccfb9f22 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 22 Nov 2023 17:52:42 +0100 Subject: [PATCH 04/16] test for TestSession#inheritable_tags --- lib/datadog/ci/test_session.rb | 2 ++ spec/datadog/ci/test_session_spec.rb | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/datadog/ci/test_session.rb b/lib/datadog/ci/test_session.rb index a8ca98cc..0dbd691e 100644 --- a/lib/datadog/ci/test_session.rb +++ b/lib/datadog/ci/test_session.rb @@ -19,6 +19,8 @@ def finish end def inheritable_tags + # this method is not synchronized because it does not iterate over the tags, but rather + # uses synchronized method to get each tag value res = {} Ext::Test::INHERITABLE_TAGS.each do |tag| res[tag] = get_tag(tag) diff --git a/spec/datadog/ci/test_session_spec.rb b/spec/datadog/ci/test_session_spec.rb index e2a6e148..97429d32 100644 --- a/spec/datadog/ci/test_session_spec.rb +++ b/spec/datadog/ci/test_session_spec.rb @@ -8,10 +8,30 @@ before { allow(Datadog::CI).to receive(:deactivate_test_session) } - it "deactivates the test" do + it "deactivates the test session" do ci_test_session.finish expect(Datadog::CI).to have_received(:deactivate_test_session) end end + + describe "#inheritable_tags" do + subject(:inheritable_tags) { ci_test_session.inheritable_tags } + + let(:ci_test_session) { described_class.new(tracer_span) } + + before do + Datadog::CI::Ext::Test::INHERITABLE_TAGS.each do |tag| + allow(tracer_span).to receive(:get_tag).with(tag).and_return("value for #{tag}") + end + end + + it "returns a hash of inheritable tags" do + is_expected.to eq( + Datadog::CI::Ext::Test::INHERITABLE_TAGS.each_with_object({}) do |tag, memo| + memo[tag] = "value for #{tag}" + end + ) + end + end end From 34118e09e8a914b68467c407edaca02aee58b651 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 22 Nov 2023 17:56:49 +0100 Subject: [PATCH 05/16] memoize inheritable tags in TestSession --- lib/datadog/ci/test_session.rb | 4 +++- sig/datadog/ci/test_session.rbs | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/datadog/ci/test_session.rb b/lib/datadog/ci/test_session.rb index 0dbd691e..baf4e3b6 100644 --- a/lib/datadog/ci/test_session.rb +++ b/lib/datadog/ci/test_session.rb @@ -19,13 +19,15 @@ def finish end def inheritable_tags + return @inheritable_tags if defined?(@inheritable_tags) + # this method is not synchronized because it does not iterate over the tags, but rather # uses synchronized method to get each tag value res = {} Ext::Test::INHERITABLE_TAGS.each do |tag| res[tag] = get_tag(tag) end - res + @inheritable_tags = res end end end diff --git a/sig/datadog/ci/test_session.rbs b/sig/datadog/ci/test_session.rbs index 4f89722d..3d369772 100644 --- a/sig/datadog/ci/test_session.rbs +++ b/sig/datadog/ci/test_session.rbs @@ -1,6 +1,8 @@ module Datadog module CI class TestSession < ConcurrentSpan + @inheritable_tags: Hash[untyped, untyped] + def inheritable_tags: () -> Hash[untyped, untyped] end end From 0daab7e38fcd3063ab74629ca5205979026e36c7 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 23 Nov 2023 10:41:53 +0100 Subject: [PATCH 06/16] connect Test to TestSession if present --- lib/datadog/ci.rb | 7 ++++--- lib/datadog/ci/ext/test.rb | 3 +++ lib/datadog/ci/recorder.rb | 1 + lib/datadog/ci/span.rb | 5 +++++ sig/datadog/ci/ext/test.rbs | 2 ++ sig/datadog/ci/span.rbs | 2 ++ spec/datadog/ci/recorder_spec.rb | 17 ++++++++++++++--- 7 files changed, 31 insertions(+), 6 deletions(-) diff --git a/lib/datadog/ci.rb b/lib/datadog/ci.rb index 0d7a1a9f..7b46155c 100644 --- a/lib/datadog/ci.rb +++ b/lib/datadog/ci.rb @@ -59,9 +59,10 @@ def active_test_session end # Return a {Datadog::CI::Test ci_test} that will trace a test called `test_name`. - # Raises an error if a test is already active. If there is an active test session, - # the new test will be connected to the session. The test will inherit service name and tags from - # the running test session if none provided. + # Raises an error if a test is already active. + # If there is an active test session, the new test will be connected to the session. + # The test will inherit service name and tags from the running test session if not provided + # in parameters. # # You could trace your test using a do-block like: # diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index 06f450c0..32302010 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -17,6 +17,9 @@ module Test TAG_TRAITS = "test.traits" TAG_TYPE = "test.type" + # those tags are special and they are used to conrrelate tests with the test sessions, suites, and modules + TAG_TEST_SESSION_ID = "_test.session_id" + # tags that can be inherited from the test session INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze diff --git a/lib/datadog/ci/recorder.rb b/lib/datadog/ci/recorder.rb index b38d5f45..08d10e36 100644 --- a/lib/datadog/ci/recorder.rb +++ b/lib/datadog/ci/recorder.rb @@ -60,6 +60,7 @@ def trace_test(test_name, service_name: nil, operation_name: "test", tags: {}, & } tags[Ext::Test::TAG_NAME] = test_name + tags[Ext::Test::TAG_TEST_SESSION_ID] = test_session.id if test_session if block Datadog::Tracing.trace(operation_name, **span_options) do |tracer_span, trace| diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb index 4c77fbb1..94df446f 100644 --- a/lib/datadog/ci/span.rb +++ b/lib/datadog/ci/span.rb @@ -15,6 +15,11 @@ def initialize(tracer_span) @tracer_span = tracer_span end + # @return [Integer] the ID of the span. + def id + tracer_span.id + end + # @return [String] the name of the span. def name tracer_span.name diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index 9916f363..e3745fb4 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -22,6 +22,8 @@ module Datadog TAG_TYPE: String + TAG_TEST_SESSION_ID: String + INHERITABLE_TAGS: Array[String] TAG_OS_ARCHITECTURE: String diff --git a/sig/datadog/ci/span.rbs b/sig/datadog/ci/span.rbs index b81df6ca..6265f0d6 100644 --- a/sig/datadog/ci/span.rbs +++ b/sig/datadog/ci/span.rbs @@ -7,6 +7,8 @@ module Datadog def initialize: (Datadog::Tracing::SpanOperation tracer_span) -> void + def id: () -> Integer + def name: () -> String def service: () -> String diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index 55f51188..d29ea2bb 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -120,10 +120,17 @@ end context "when test session is active" do + let(:test_session_id) { 42 } let(:inheritable_tags) { {"test.framework" => "my framework"} } + let(:test_session_service) { "my-test-service" } let(:test_session) do - instance_double(Datadog::CI::TestSession, service: test_session_service, inheritable_tags: inheritable_tags) + instance_double( + Datadog::CI::TestSession, + service: test_session_service, + inheritable_tags: inheritable_tags, + id: test_session_id + ) end before do @@ -131,7 +138,9 @@ end context "when service name and tags are not given" do - let(:expected_tags) { {"test.framework" => "my framework", "test.name" => test_name} } + let(:expected_tags) do + {"test.framework" => "my framework", "test.name" => test_name, "_test.session_id" => test_session_id} + end subject(:trace) do recorder.trace_test( @@ -165,7 +174,9 @@ context "when service name and tags are given" do let(:tags) { {"test.framework" => "special test framework"} } - let(:expected_tags) { {"test.framework" => "special test framework", "test.name" => test_name} } + let(:expected_tags) do + {"test.framework" => "special test framework", "test.name" => test_name, "_test.session_id" => test_session_id} + end subject(:trace) do recorder.trace_test( From 507049605cb6dde1b79597877f14ca68c6785db1 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 23 Nov 2023 13:00:50 +0100 Subject: [PATCH 07/16] TestSession and TestV2 serializers --- .standard_todo.yml | 26 ++++--- lib/datadog/ci/ext/test.rb | 2 + lib/datadog/ci/recorder.rb | 2 + .../ci/test_visibility/serializers/base.rb | 14 ++-- .../serializers/factories/test_suite_level.rb | 31 ++++++++ .../serializers/test_session.rb | 59 ++++++++++++++ .../ci/test_visibility/serializers/test_v2.rb | 53 +++++++++++++ sig/datadog/ci/ext/test.rbs | 4 + .../ci/test_visibility/serializers/base.rbs | 1 + .../factories/test_suite_level.rbs | 13 ++++ .../serializers/test_session.rbs | 26 +++++++ .../test_visibility/serializers/test_v2.rbs | 25 ++++++ spec/datadog/ci/recorder_spec.rb | 2 +- .../factories/test_suite_level_spec.rb | 28 +++++++ .../serializers/test_session_spec.rb | 58 ++++++++++++++ .../serializers/test_v2_spec.rb | 77 +++++++++++++++++++ spec/support/tracer_helpers.rb | 52 +++++++++++-- 17 files changed, 447 insertions(+), 26 deletions(-) create mode 100644 lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb create mode 100644 lib/datadog/ci/test_visibility/serializers/test_session.rb create mode 100644 lib/datadog/ci/test_visibility/serializers/test_v2.rb create mode 100644 sig/datadog/ci/test_visibility/serializers/factories/test_suite_level.rbs create mode 100644 sig/datadog/ci/test_visibility/serializers/test_session.rbs create mode 100644 sig/datadog/ci/test_visibility/serializers/test_v2.rbs create mode 100644 spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb create mode 100644 spec/datadog/ci/test_visibility/serializers/test_session_spec.rb create mode 100644 spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb diff --git a/.standard_todo.yml b/.standard_todo.yml index 6c3441cc..d72dcc59 100644 --- a/.standard_todo.yml +++ b/.standard_todo.yml @@ -2,15 +2,17 @@ # Remove from this list as you refactor files. --- ignore: -- lib/datadog/ci/contrib/minitest/integration.rb: - - Style/SafeNavigation -- lib/datadog/ci/contrib/cucumber/integration.rb: - - Style/SafeNavigation -- lib/datadog/ci/contrib/rspec/integration.rb: - - Style/SafeNavigation -- lib/datadog/ci/ext/environment.rb: - - Style/SafeNavigation -- spec/support/log_helpers.rb: - - Performance/UnfreezeString -- Appraisals: - - Style/Alias + - lib/datadog/ci/test_visibility/serializers/base.rb: + - Style/HashExcept + - lib/datadog/ci/contrib/minitest/integration.rb: + - Style/SafeNavigation + - lib/datadog/ci/contrib/cucumber/integration.rb: + - Style/SafeNavigation + - lib/datadog/ci/contrib/rspec/integration.rb: + - Style/SafeNavigation + - lib/datadog/ci/ext/environment.rb: + - Style/SafeNavigation + - spec/support/log_helpers.rb: + - Performance/UnfreezeString + - Appraisals: + - Style/Alias diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb index 32302010..6da4c46f 100644 --- a/lib/datadog/ci/ext/test.rb +++ b/lib/datadog/ci/ext/test.rb @@ -16,9 +16,11 @@ module Test TAG_SUITE = "test.suite" TAG_TRAITS = "test.traits" TAG_TYPE = "test.type" + TAG_COMMAND = "test.command" # those tags are special and they are used to conrrelate tests with the test sessions, suites, and modules TAG_TEST_SESSION_ID = "_test.session_id" + SPECIAL_TAGS = [TAG_TEST_SESSION_ID].freeze # tags that can be inherited from the test session INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION, TAG_TYPE].freeze diff --git a/lib/datadog/ci/recorder.rb b/lib/datadog/ci/recorder.rb index 08d10e36..2aac8bef 100644 --- a/lib/datadog/ci/recorder.rb +++ b/lib/datadog/ci/recorder.rb @@ -38,6 +38,8 @@ def start_test_session(service_name: nil, tags: {}) set_trace_origin(trace) + tags[Ext::Test::TAG_TEST_SESSION_ID] = tracer_span.id + test_session = build_test_session(tracer_span, tags) @global_context.activate_test_session!(test_session) diff --git a/lib/datadog/ci/test_visibility/serializers/base.rb b/lib/datadog/ci/test_visibility/serializers/base.rb index 04e20e72..1c68c8a3 100644 --- a/lib/datadog/ci/test_visibility/serializers/base.rb +++ b/lib/datadog/ci/test_visibility/serializers/base.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative "../../ext/test" + module Datadog module CI module TestVisibility @@ -9,11 +11,13 @@ class Base MINIMUM_DURATION_NANO = 0 MAXIMUM_DURATION_NANO = 9223372036854775807 - attr_reader :trace, :span + attr_reader :trace, :span, :meta def initialize(trace, span) @trace = trace @span = span + + @meta = @span.meta.reject { |key, _| Ext::Test::SPECIAL_TAGS.include?(key) } end def to_msgpack(packer = nil) @@ -67,6 +71,10 @@ def parent_id @span.parent_id end + def test_session_id + @span.get_tag(Ext::Test::TAG_TEST_SESSION_ID) + end + def type end @@ -98,10 +106,6 @@ def duration @duration ||= duration_nano(@span.duration) end - def meta - @span.meta - end - def metrics @span.metrics end diff --git a/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb b/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb new file mode 100644 index 00000000..1675fe7b --- /dev/null +++ b/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require_relative "../test_v2" +require_relative "../test_session" +require_relative "../span" + +module Datadog + module CI + module TestVisibility + module Serializers + module Factories + # This factory takes care of creating citestcycle serializers when test-suite-level visibility is enabled + module TestSuiteLevel + module_function + + def serializer(trace, span) + case span.type + when Datadog::CI::Ext::AppTypes::TYPE_TEST + Serializers::TestV2.new(trace, span) + when Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION + Serializers::TestSession.new(trace, span) + else + Serializers::Span.new(trace, span) + end + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/serializers/test_session.rb b/lib/datadog/ci/test_visibility/serializers/test_session.rb new file mode 100644 index 00000000..4e4ed7d2 --- /dev/null +++ b/lib/datadog/ci/test_visibility/serializers/test_session.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require_relative "base" +require_relative "../../ext/test" + +module Datadog + module CI + module TestVisibility + module Serializers + class TestSession < Base + CONTENT_FIELDS = [ + "test_session_id", + "name", "resource", "service", + "error", "start", "duration", + "meta", "metrics", + "type" => "span_type" + ].freeze + + CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS) + + REQUIRED_FIELDS = [ + "test_session_id", + "error", + "name", + "resource", + "start", + "duration" + ].freeze + + def content_fields + CONTENT_FIELDS + end + + def content_map_size + CONTENT_MAP_SIZE + end + + def type + Ext::AppTypes::TYPE_TEST_SESSION + end + + def name + "#{@span.get_tag(Ext::Test::TAG_FRAMEWORK)}.test_session" + end + + def resource + "#{@span.get_tag(Ext::Test::TAG_FRAMEWORK)}.test_session.#{@span.get_tag(Ext::Test::TAG_COMMAND)}" + end + + private + + def required_fields + REQUIRED_FIELDS + end + end + end + end + end +end diff --git a/lib/datadog/ci/test_visibility/serializers/test_v2.rb b/lib/datadog/ci/test_visibility/serializers/test_v2.rb new file mode 100644 index 00000000..1a08c1d0 --- /dev/null +++ b/lib/datadog/ci/test_visibility/serializers/test_v2.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require_relative "test_v1" +require_relative "../../ext/test" + +module Datadog + module CI + module TestVisibility + module Serializers + class TestV2 < TestV1 + CONTENT_FIELDS = [ + "trace_id", "span_id", + "name", "resource", "service", + "error", "start", "duration", + "meta", "metrics", "test_session_id", + "type" => "span_type" + ].freeze + + CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS) + + REQUIRED_FIELDS = [ + "test_session_id", + "trace_id", + "span_id", + "error", + "name", + "resource", + "start", + "duration" + ].freeze + + def content_fields + CONTENT_FIELDS + end + + def content_map_size + CONTENT_MAP_SIZE + end + + def version + 2 + end + + private + + def required_fields + REQUIRED_FIELDS + end + end + end + end + end +end diff --git a/sig/datadog/ci/ext/test.rbs b/sig/datadog/ci/ext/test.rbs index e3745fb4..ba7d6855 100644 --- a/sig/datadog/ci/ext/test.rbs +++ b/sig/datadog/ci/ext/test.rbs @@ -22,8 +22,12 @@ module Datadog TAG_TYPE: String + TAG_COMMAND: String + TAG_TEST_SESSION_ID: String + SPECIAL_TAGS: Array[String] + INHERITABLE_TAGS: Array[String] TAG_OS_ARCHITECTURE: String diff --git a/sig/datadog/ci/test_visibility/serializers/base.rbs b/sig/datadog/ci/test_visibility/serializers/base.rbs index 908b19d6..80c5f6bc 100644 --- a/sig/datadog/ci/test_visibility/serializers/base.rbs +++ b/sig/datadog/ci/test_visibility/serializers/base.rbs @@ -10,6 +10,7 @@ module Datadog @content_fields_count: Integer @start: Integer @duration: Integer + @meta: Hash[untyped, untyped] attr_reader trace: Datadog::Tracing::TraceSegment attr_reader span: Datadog::Tracing::Span diff --git a/sig/datadog/ci/test_visibility/serializers/factories/test_suite_level.rbs b/sig/datadog/ci/test_visibility/serializers/factories/test_suite_level.rbs new file mode 100644 index 00000000..8ebc424f --- /dev/null +++ b/sig/datadog/ci/test_visibility/serializers/factories/test_suite_level.rbs @@ -0,0 +1,13 @@ +module Datadog + module CI + module TestVisibility + module Serializers + module Factories + module TestSuiteLevel + def self?.serializer: (Datadog::Tracing::TraceSegment trace, Datadog::Tracing::Span span) -> Datadog::CI::TestVisibility::Serializers::Base + end + end + end + end + end +end diff --git a/sig/datadog/ci/test_visibility/serializers/test_session.rbs b/sig/datadog/ci/test_visibility/serializers/test_session.rbs new file mode 100644 index 00000000..98df8ac0 --- /dev/null +++ b/sig/datadog/ci/test_visibility/serializers/test_session.rbs @@ -0,0 +1,26 @@ +module Datadog + module CI + module TestVisibility + module Serializers + class TestSession < Base + CONTENT_FIELDS: Array[String | Hash[String, String]] + CONTENT_MAP_SIZE: Integer + REQUIRED_FIELDS: Array[String] + + def content_fields: () -> Array[String | Hash[String, String]] + def content_map_size: () -> Integer + + def type: () -> ::String + + def name: () -> ::String + + def resource: () -> ::String + + private + + def required_fields: () -> Array[String] + end + end + end + end +end diff --git a/sig/datadog/ci/test_visibility/serializers/test_v2.rbs b/sig/datadog/ci/test_visibility/serializers/test_v2.rbs new file mode 100644 index 00000000..1ba034be --- /dev/null +++ b/sig/datadog/ci/test_visibility/serializers/test_v2.rbs @@ -0,0 +1,25 @@ +module Datadog + module CI + module TestVisibility + module Serializers + class TestV2 < TestV1 + CONTENT_FIELDS: ::Array[String | ::Hash[::String, String]] + + CONTENT_MAP_SIZE: Integer + + REQUIRED_FIELDS: ::Array[String] + + def content_fields: () -> ::Array[String | ::Hash[::String, String]] + + def content_map_size: () -> Integer + + def version: () -> 2 + + private + + def required_fields: () -> Array[String] + end + end + end + end +end diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index d29ea2bb..3e062ad6 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -351,8 +351,8 @@ subject(:start_test_session) { recorder.start_test_session(service_name: service) } let(:session_operation_name) { "test.session" } - let(:span_op) { Datadog::Tracing::SpanOperation.new(session_operation_name) } + let(:expected_tags) { {"_test.session_id" => span_op.id} } before do allow(Datadog::Tracing) diff --git a/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb b/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb new file mode 100644 index 00000000..91ed48f9 --- /dev/null +++ b/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb @@ -0,0 +1,28 @@ +require_relative "../../../../../../lib/datadog/ci/test_visibility/serializers/factories/test_suite_level" +require_relative "../../../../../../lib/datadog/ci/test_visibility/serializers/test_v2" +require_relative "../../../../../../lib/datadog/ci/test_visibility/serializers/test_session" +require_relative "../../../../../../lib/datadog/ci/recorder" + +RSpec.describe Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + end + + before do + produce_test_session_trace + end + + subject { described_class.serializer(trace, ci_span) } + + describe ".convert_trace_to_serializable_events" do + context "with a session span" do + let(:ci_span) { test_session_span } + it { is_expected.to be_kind_of(Datadog::CI::TestVisibility::Serializers::TestSession) } + end + + context "with a test span" do + let(:ci_span) { first_test_span } + it { is_expected.to be_kind_of(Datadog::CI::TestVisibility::Serializers::TestV2) } + end + end +end diff --git a/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb new file mode 100644 index 00000000..6cc84b64 --- /dev/null +++ b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb @@ -0,0 +1,58 @@ +require_relative "../../../../../lib/datadog/ci/test_visibility/serializers/test_session" +require_relative "../../../../../lib/datadog/ci/recorder" +require_relative "../../../../../lib/datadog/ci/ext/app_types" + +RSpec.describe Datadog::CI::TestVisibility::Serializers::TestSession do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + end + + include_context "Test visibility event serialized" do + subject { described_class.new(trace, test_session_span) } + end + + describe "#to_msgpack" do + context "traced a single test execution with Recorder" do + before do + produce_test_session_trace + end + + it "serializes test event to messagepack" do + expect_event_header(type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION) + + expect(content).to include( + { + "test_session_id" => test_session_span.id.to_s, + "name" => "rspec.test_session", + "service" => "rspec-test-suite", + "type" => Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION, + "resource" => "rspec.test_session." + } + ) + + expect(meta).to include( + { + "test.framework" => "rspec", + "test.status" => "pass", + "_dd.origin" => "ciapp-test" + } + ) + + expect(meta["_test.session_id"]).to be_nil + end + end + + context "trace a failed test" do + before do + produce_test_session_trace(result: "FAILED", exception: StandardError.new("1 + 2 are not equal to 5")) + end + + it "has error" do + expect_event_header(type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION) + + expect(content).to include({"error" => 1}) + expect(meta).to include({"test.status" => "fail"}) + end + end + end +end diff --git a/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb b/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb new file mode 100644 index 00000000..32ccb4fc --- /dev/null +++ b/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb @@ -0,0 +1,77 @@ +require_relative "../../../../../lib/datadog/ci/test_visibility/serializers/test_v2" +require_relative "../../../../../lib/datadog/ci/recorder" + +RSpec.describe Datadog::CI::TestVisibility::Serializers::TestV2 do + include_context "CI mode activated" do + let(:integration_name) { :rspec } + end + + include_context "Test visibility event serialized" do + subject { described_class.new(trace, first_test_span) } + end + + describe "#to_msgpack" do + context "traced a single test execution with Recorder" do + before do + produce_test_session_trace + end + + it "serializes test event to messagepack" do + expect_event_header(version: 2) + + expect(content).to include( + { + "trace_id" => trace.id, + "span_id" => first_test_span.id, + "name" => "rspec.test", + "service" => "rspec-test-suite", + "type" => "test", + "resource" => "calculator_tests.test_add", + "test_session_id" => test_session_span.id.to_s + } + ) + + expect(meta).to include( + { + "test.framework" => "rspec", + "test.status" => "pass", + "_dd.origin" => "ciapp-test", + "test_owner" => "my_team" + } + ) + expect(meta["_test.session_id"]).to be_nil + + expect(metrics).to eq({"memory_allocations" => 16}) + end + end + + context "trace a failed test" do + before do + produce_test_session_trace(result: "FAILED", exception: StandardError.new("1 + 2 are not equal to 5")) + end + + it "has error" do + expect_event_header(version: 2) + + expect(content).to include({"error" => 1}) + expect(meta).to include({"test.status" => "fail"}) + end + end + + context "with time and duration expectations" do + let(:start_time) { Time.now } + let(:duration_seconds) { 3 } + + before do + produce_test_session_trace(start_time: start_time, duration_seconds: duration_seconds) + end + + it "correctly serializes start and duration in nanoseconds" do + expect(content).to include({ + "start" => start_time.to_i * 1_000_000_000 + start_time.nsec, + "duration" => 3 * 1_000_000_000 + }) + end + end + end +end diff --git a/spec/support/tracer_helpers.rb b/spec/support/tracer_helpers.rb index 9eabe73a..ceac8cb3 100644 --- a/spec/support/tracer_helpers.rb +++ b/spec/support/tracer_helpers.rb @@ -43,14 +43,7 @@ def produce_test_trace( Datadog::CI.active_test.set_tag("test_owner", "my_team") Datadog::CI.active_test.set_metric("memory_allocations", 16) - case result - when "FAILED" - test.failed!(exception: exception) - when "SKIPPED" - test.skipped!(exception: exception, reason: skip_reason) - else - test.passed! - end + set_result(test, result: result, exception: exception, skip_reason: skip_reason) Timecop.travel(start_time + duration_seconds) end @@ -58,6 +51,38 @@ def produce_test_trace( Timecop.return end + def produce_test_session_trace( + framework: "rspec", operation: "rspec.example", + test_name: "test_add", test_suite: "calculator_tests", + service: "rspec-test-suite", result: "PASSED", exception: nil, + skip_reason: nil, start_time: Time.now, duration_seconds: 2, + with_http_span: false + ) + test_session = Datadog::CI.start_test_session( + service_name: service, + tags: { + Datadog::CI::Ext::Test::TAG_FRAMEWORK => framework, + Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION => "1.0.0", + Datadog::CI::Ext::Test::TAG_TYPE => "test" + } + ) + + produce_test_trace( + framework: framework, operation: operation, + test_name: test_name, test_suite: test_suite, + service: service, result: result, exception: exception, skip_reason: skip_reason, + start_time: start_time, duration_seconds: duration_seconds, + with_http_span: with_http_span + ) + + set_result(test_session, result: result, exception: exception, skip_reason: skip_reason) + test_session.finish + end + + def test_session_span + spans.find { |span| span.type == "test_session_end" } + end + def first_test_span spans.find { |span| span.type == "test" } end @@ -127,6 +152,17 @@ def fetch_spans(tracer = self.tracer) end end + def set_result(span, result: "PASSED", exception: nil, skip_reason: nil) + case result + when "FAILED" + span.failed!(exception: exception) + when "SKIPPED" + span.skipped!(exception: exception, reason: skip_reason) + else + span.passed! + end + end + # Remove all traces from the current tracer instance and # busts cache of +#spans+ and +#span+. def clear_traces! From 0aed4c043fb9dea55740a91ba74b2b2081d80eaa Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 23 Nov 2023 13:49:29 +0100 Subject: [PATCH 08/16] add test.command tag --- lib/datadog/ci/span.rb | 2 ++ lib/datadog/ci/utils/test_run.rb | 15 +++++++++++++++ sig/datadog/ci/utils/test_run.rbs | 11 +++++++++++ spec/datadog/ci/span_spec.rb | 7 +++++++ .../serializers/test_session_spec.rb | 3 ++- spec/datadog/ci/utils/test_run_spec.rb | 9 +++++++++ spec/support/ci_mode_helpers.rb | 3 +++ 7 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 lib/datadog/ci/utils/test_run.rb create mode 100644 sig/datadog/ci/utils/test_run.rbs create mode 100644 spec/datadog/ci/utils/test_run_spec.rb diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb index 94df446f..df57eaa7 100644 --- a/lib/datadog/ci/span.rb +++ b/lib/datadog/ci/span.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "ext/test" +require_relative "utils/test_run" module Datadog module CI @@ -103,6 +104,7 @@ def set_environment_runtime_tags tracer_span.set_tag(Ext::Test::TAG_OS_PLATFORM, ::RbConfig::CONFIG["host_os"]) tracer_span.set_tag(Ext::Test::TAG_RUNTIME_NAME, Core::Environment::Ext::LANG_ENGINE) tracer_span.set_tag(Ext::Test::TAG_RUNTIME_VERSION, Core::Environment::Ext::ENGINE_VERSION) + tracer_span.set_tag(Ext::Test::TAG_COMMAND, Utils::TestRun.command) end def set_default_tags diff --git a/lib/datadog/ci/utils/test_run.rb b/lib/datadog/ci/utils/test_run.rb new file mode 100644 index 00000000..b3309a79 --- /dev/null +++ b/lib/datadog/ci/utils/test_run.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Utils + module TestRun + def self.command + return @command if defined?(@command) + + @command = "#{Process.argv0} #{ARGV.join(" ")}" + end + end + end + end +end diff --git a/sig/datadog/ci/utils/test_run.rbs b/sig/datadog/ci/utils/test_run.rbs new file mode 100644 index 00000000..c985f99b --- /dev/null +++ b/sig/datadog/ci/utils/test_run.rbs @@ -0,0 +1,11 @@ +module Datadog + module CI + module Utils + module TestRun + self.@command: String + + def self.command: () -> String + end + end + end +end diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index fe4983c3..639fb8d8 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -111,11 +111,18 @@ end describe "#set_environment_runtime_tags" do + let(:test_command) { "command" } + + before do + allow(Datadog::CI::Utils::TestRun).to receive(:command).and_return(test_command) + end + it "sets the environment runtime tags" do expect(tracer_span).to receive(:set_tag).with("os.architecture", ::RbConfig::CONFIG["host_cpu"]) expect(tracer_span).to receive(:set_tag).with("os.platform", ::RbConfig::CONFIG["host_os"]) expect(tracer_span).to receive(:set_tag).with("runtime.name", Datadog::Core::Environment::Ext::LANG_ENGINE) expect(tracer_span).to receive(:set_tag).with("runtime.version", Datadog::Core::Environment::Ext::ENGINE_VERSION) + expect(tracer_span).to receive(:set_tag).with("test.command", test_command) span.set_environment_runtime_tags end diff --git a/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb index 6cc84b64..8c0de0b0 100644 --- a/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb +++ b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb @@ -26,12 +26,13 @@ "name" => "rspec.test_session", "service" => "rspec-test-suite", "type" => Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION, - "resource" => "rspec.test_session." + "resource" => "rspec.test_session.#{test_command}" } ) expect(meta).to include( { + "test.command" => test_command, "test.framework" => "rspec", "test.status" => "pass", "_dd.origin" => "ciapp-test" diff --git a/spec/datadog/ci/utils/test_run_spec.rb b/spec/datadog/ci/utils/test_run_spec.rb new file mode 100644 index 00000000..34247408 --- /dev/null +++ b/spec/datadog/ci/utils/test_run_spec.rb @@ -0,0 +1,9 @@ +require_relative "../../../../lib/datadog/ci/utils/test_run" + +RSpec.describe ::Datadog::CI::Utils::TestRun do + describe ".command" do + subject { described_class.command } + + it { is_expected.to eq("#{Process.argv0} #{ARGV.join(" ")}") } + end +end diff --git a/spec/support/ci_mode_helpers.rb b/spec/support/ci_mode_helpers.rb index 9d8d678c..6841082f 100644 --- a/spec/support/ci_mode_helpers.rb +++ b/spec/support/ci_mode_helpers.rb @@ -1,4 +1,5 @@ RSpec.shared_context "CI mode activated" do + let(:test_command) { "command" } let(:integration_name) { :override_me } let(:integration_options) { {} } @@ -7,6 +8,8 @@ receive(:endpoint?).with("/evp_proxy/v2/").and_return(true) ) + allow(Datadog::CI::Utils::TestRun).to receive(:command).and_return(test_command) + Datadog.configure do |c| c.ci.enabled = true c.ci.instrument integration_name, integration_options From a9d1e3b159673a71852887decfae17277b51c979 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 23 Nov 2023 14:16:13 +0100 Subject: [PATCH 09/16] add experimental_test_suite_level_visibility_enabled config --- lib/datadog/ci.rb | 2 +- lib/datadog/ci/configuration/components.rb | 4 +- lib/datadog/ci/configuration/settings.rb | 6 ++ lib/datadog/ci/ext/settings.rb | 1 + lib/datadog/ci/recorder.rb | 8 ++- sig/datadog/ci.rbs | 2 +- sig/datadog/ci/ext/settings.rbs | 1 + sig/datadog/ci/recorder.rbs | 5 +- .../ci/configuration/components_spec.rb | 23 +++++++- .../datadog/ci/configuration/settings_spec.rb | 45 ++++++++++++++ spec/datadog/ci/recorder_spec.rb | 59 +++++++++++-------- spec/support/ci_mode_helpers.rb | 1 + 12 files changed, 126 insertions(+), 31 deletions(-) diff --git a/lib/datadog/ci.rb b/lib/datadog/ci.rb index 7b46155c..e89d30cf 100644 --- a/lib/datadog/ci.rb +++ b/lib/datadog/ci.rb @@ -13,7 +13,6 @@ class << self # Return a {Datadog::CI::TestSesstion ci_test_session} that represents the whole test session run. # Raises an error if a session is already active. # - # # The {#start_test_session} method is used to mark the start of the test session: # ``` # Datadog::CI.start_test_session( @@ -30,6 +29,7 @@ class << self # @param [String] service_name the service name for this session # @param [Hash] tags extra tags which should be added to the test. # @return [Datadog::CI::TestSession] returns the active, running {Datadog::CI::TestSession}. + # @return [nil] if test suite level visibility is disabled (old Datadog agent detected) # # @public_api def start_test_session(service_name: nil, tags: {}) diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 2bdb1cfb..6647d131 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -60,7 +60,9 @@ def activate_ci!(settings) settings.tracing.test_mode.writer_options = writer_options - @ci_recorder = Recorder.new + @ci_recorder = Recorder.new( + test_suite_level_visibility_enabled: settings.ci.experimental_test_suite_level_visibility_enabled + ) end def can_use_evp_proxy?(settings, agent_settings) diff --git a/lib/datadog/ci/configuration/settings.rb b/lib/datadog/ci/configuration/settings.rb index eb92f4b9..c3ac21a2 100644 --- a/lib/datadog/ci/configuration/settings.rb +++ b/lib/datadog/ci/configuration/settings.rb @@ -34,6 +34,12 @@ def self.add_settings!(base) o.env CI::Ext::Settings::ENV_AGENTLESS_URL end + option :experimental_test_suite_level_visibility_enabled do |o| + o.type :bool + o.env CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED + o.default false + end + define_method(:instrument) do |integration_name, options = {}, &block| return unless enabled diff --git a/lib/datadog/ci/ext/settings.rb b/lib/datadog/ci/ext/settings.rb index c08a5f56..3102e18a 100644 --- a/lib/datadog/ci/ext/settings.rb +++ b/lib/datadog/ci/ext/settings.rb @@ -8,6 +8,7 @@ module Settings ENV_MODE_ENABLED = "DD_TRACE_CI_ENABLED" ENV_AGENTLESS_MODE_ENABLED = "DD_CIVISIBILITY_AGENTLESS_ENABLED" ENV_AGENTLESS_URL = "DD_CIVISIBILITY_AGENTLESS_URL" + ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED" end end end diff --git a/lib/datadog/ci/recorder.rb b/lib/datadog/ci/recorder.rb index 2aac8bef..c163d88e 100644 --- a/lib/datadog/ci/recorder.rb +++ b/lib/datadog/ci/recorder.rb @@ -19,15 +19,19 @@ module Datadog module CI # Common behavior for CI tests class Recorder - attr_reader :environment_tags + attr_reader :environment_tags, :test_suite_level_visibility_enabled + + def initialize(test_suite_level_visibility_enabled: false) + @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled - def initialize @environment_tags = Ext::Environment.tags(ENV).freeze @local_context = Context::Local.new @global_context = Context::Global.new end def start_test_session(service_name: nil, tags: {}) + return nil unless @test_suite_level_visibility_enabled + span_options = { service: service_name, span_type: Ext::AppTypes::TYPE_TEST_SESSION diff --git a/sig/datadog/ci.rbs b/sig/datadog/ci.rbs index 2340fccd..fc1b3abc 100644 --- a/sig/datadog/ci.rbs +++ b/sig/datadog/ci.rbs @@ -4,7 +4,7 @@ module Datadog def self.start_test: (String span_name, ?service_name: String?, ?operation_name: String, ?tags: Hash[untyped, untyped]) -> Datadog::CI::Test - def self.start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession + def self.start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession? def self.trace: (String span_type, String span_name, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped diff --git a/sig/datadog/ci/ext/settings.rbs b/sig/datadog/ci/ext/settings.rbs index 5d76a0cd..aef6d2f8 100644 --- a/sig/datadog/ci/ext/settings.rbs +++ b/sig/datadog/ci/ext/settings.rbs @@ -5,6 +5,7 @@ module Datadog ENV_MODE_ENABLED: String ENV_AGENTLESS_MODE_ENABLED: String ENV_AGENTLESS_URL: String + ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED: String end end end diff --git a/sig/datadog/ci/recorder.rbs b/sig/datadog/ci/recorder.rbs index 6bafc806..edb90fa7 100644 --- a/sig/datadog/ci/recorder.rbs +++ b/sig/datadog/ci/recorder.rbs @@ -1,17 +1,20 @@ module Datadog module CI class Recorder + @test_suite_level_visibility_enabled: bool @environment_tags: Hash[String, String] @local_context: Datadog::CI::Context::Local @global_context: Datadog::CI::Context::Global attr_reader environment_tags: Hash[String, String] + def initialize: (?test_suite_level_visibility_enabled: bool) -> void + def trace_test: (String span_name, ?service_name: String?, ?operation_name: String, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Test span) -> untyped } -> untyped def trace: (String span_type, String span_name, ?tags: Hash[untyped, untyped]) ?{ (Datadog::CI::Span span) -> untyped } -> untyped - def start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession + def start_test_session: (?service_name: String?, ?tags: Hash[untyped, untyped]) -> Datadog::CI::TestSession? def active_test_session: () -> Datadog::CI::TestSession? diff --git a/spec/datadog/ci/configuration/components_spec.rb b/spec/datadog/ci/configuration/components_spec.rb index 68719ee6..401a48b2 100644 --- a/spec/datadog/ci/configuration/components_spec.rb +++ b/spec/datadog/ci/configuration/components_spec.rb @@ -42,6 +42,10 @@ .to receive(:agentless_mode_enabled) .and_return(agentless_enabled) + allow(settings.ci) + .to receive(:experimental_test_suite_level_visibility_enabled) + .and_return(experimental_test_suite_level_visibility_enabled) + allow(settings.ci) .to receive(:agentless_url) .and_return(agentless_url) @@ -90,13 +94,28 @@ let(:agentless_url) { nil } let(:dd_site) { nil } let(:agentless_enabled) { false } + let(:experimental_test_suite_level_visibility_enabled) { false } let(:evp_proxy_supported) { false } context "is enabled" do let(:enabled) { true } - it "creates a CI recorder" do - expect(components.ci_recorder).to be_kind_of(Datadog::CI::Recorder) + context "when #experimental_test_suite_level_visibility_enabled" do + context "is false" do + it "creates a CI recorder with test_suite_level_visibility_enabled=false" do + expect(components.ci_recorder).to be_kind_of(Datadog::CI::Recorder) + expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(false) + end + end + + context "is true" do + let(:experimental_test_suite_level_visibility_enabled) { true } + + it "creates a CI recorder with test_suite_level_visibility_enabled=false" do + expect(components.ci_recorder).to be_kind_of(Datadog::CI::Recorder) + expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(true) + end + end end context "and when #agentless_mode" do diff --git a/spec/datadog/ci/configuration/settings_spec.rb b/spec/datadog/ci/configuration/settings_spec.rb index e58fcab7..17963a2a 100644 --- a/spec/datadog/ci/configuration/settings_spec.rb +++ b/spec/datadog/ci/configuration/settings_spec.rb @@ -172,6 +172,51 @@ def patcher end end + describe "#experimental_test_suite_level_visibility_enabled" do + subject(:experimental_test_suite_level_visibility_enabled) do + settings.ci.experimental_test_suite_level_visibility_enabled + end + + it { is_expected.to be false } + + context "when #{Datadog::CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED}" do + around do |example| + ClimateControl.modify( + Datadog::CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED => enable + ) do + example.run + end + end + + context "is not defined" do + let(:enable) { nil } + + it { is_expected.to be false } + end + + context "is set to true" do + let(:enable) { "true" } + + it { is_expected.to be true } + end + + context "is set to false" do + let(:enable) { "false" } + + it { is_expected.to be false } + end + end + end + + describe "#experimental_test_suite_level_visibility_enabled=" do + it "updates the #enabled setting" do + expect { settings.ci.experimental_test_suite_level_visibility_enabled = true } + .to change { settings.ci.experimental_test_suite_level_visibility_enabled } + .from(false) + .to(true) + end + end + describe "#instrument" do let(:integration_name) { :fake } diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index 3e062ad6..80c92d66 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -6,12 +6,13 @@ let(:tags) { {} } let(:expected_tags) { {} } let(:environment_tags) { Datadog::CI::Ext::Environment.tags(ENV) } + let(:test_suite_level_visibility_enabled) { true } let(:ci_span) do spy("CI object spy") end - subject(:recorder) { described_class.new } + subject(:recorder) { described_class.new(test_suite_level_visibility_enabled: test_suite_level_visibility_enabled) } before do allow(Datadog::Tracing).to receive(:active_trace).and_return(trace_op) @@ -348,33 +349,45 @@ end describe "#start_test_session" do - subject(:start_test_session) { recorder.start_test_session(service_name: service) } - - let(:session_operation_name) { "test.session" } - let(:span_op) { Datadog::Tracing::SpanOperation.new(session_operation_name) } - let(:expected_tags) { {"_test.session_id" => span_op.id} } - - before do - allow(Datadog::Tracing) - .to receive(:trace) - .with( - session_operation_name, - { - span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION, - service: service - } - ) - .and_return(span_op) + context "when test suite level visibility is enabled" do + subject(:start_test_session) { recorder.start_test_session(service_name: service) } + + let(:session_operation_name) { "test.session" } + let(:span_op) { Datadog::Tracing::SpanOperation.new(session_operation_name) } + let(:expected_tags) { {"_test.session_id" => span_op.id} } + + before do + allow(Datadog::Tracing) + .to receive(:trace) + .with( + session_operation_name, + { + span_type: Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION, + service: service + } + ) + .and_return(span_op) - allow(Datadog::CI::TestSession).to receive(:new).with(span_op).and_return(ci_span) + allow(Datadog::CI::TestSession).to receive(:new).with(span_op).and_return(ci_span) - start_test_session + start_test_session + end + + it_behaves_like "internal tracing context" + it_behaves_like "initialize ci span with tags" + + it { is_expected.to be(ci_span) } end - it_behaves_like "internal tracing context" - it_behaves_like "initialize ci span with tags" + context "when test suite level visibility is disabled" do + subject(:start_test_session) { recorder.start_test_session(service_name: service) } + + let(:test_suite_level_visibility_enabled) { false } + + before { start_test_session } - it { is_expected.to be(ci_span) } + it { is_expected.to be_nil } + end end describe "#active_test_session" do diff --git a/spec/support/ci_mode_helpers.rb b/spec/support/ci_mode_helpers.rb index 6841082f..90739634 100644 --- a/spec/support/ci_mode_helpers.rb +++ b/spec/support/ci_mode_helpers.rb @@ -12,6 +12,7 @@ Datadog.configure do |c| c.ci.enabled = true + c.ci.experimental_test_suite_level_visibility_enabled = true c.ci.instrument integration_name, integration_options end end From b5c50ebaf08baccba9d2e2a2ec95813d65003830 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 23 Nov 2023 14:29:27 +0100 Subject: [PATCH 10/16] configure test visibility transport with correct serializers factory based on settings --- lib/datadog/ci/configuration/components.rb | 12 ++++++++++++ sig/datadog/ci/configuration/components.rbs | 1 + sig/datadog/ci/test_visibility/transport.rbs | 6 +++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 6647d131..2d485121 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -6,6 +6,8 @@ require_relative "../ext/transport" require_relative "../test_visibility/flush" require_relative "../test_visibility/transport" +require_relative "../test_visibility/serializers/factories/test_level" +require_relative "../test_visibility/serializers/factories/test_suite_level" require_relative "../transport/api/builder" require_relative "../recorder" @@ -91,6 +93,7 @@ def build_agentless_transport(settings) Datadog::CI::TestVisibility::Transport.new( api: Transport::Api::Builder.build_ci_test_cycle_api(settings), + serializers_factory: serializers_factory(settings), dd_env: settings.env ) end @@ -101,9 +104,18 @@ def build_evp_proxy_transport(settings, agent_settings) Datadog::CI::TestVisibility::Transport.new( api: Transport::Api::Builder.build_evp_proxy_api(agent_settings), + serializers_factory: serializers_factory(settings), dd_env: settings.env ) end + + def serializers_factory(settings) + if settings.ci.experimental_test_suite_level_visibility_enabled + Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel + else + Datadog::CI::TestVisibility::Serializers::Factories::TestLevel + end + end end end end diff --git a/sig/datadog/ci/configuration/components.rbs b/sig/datadog/ci/configuration/components.rbs index bae3e55a..5a1e0cad 100644 --- a/sig/datadog/ci/configuration/components.rbs +++ b/sig/datadog/ci/configuration/components.rbs @@ -13,6 +13,7 @@ module Datadog def build_agentless_transport: (untyped settings) -> Datadog::CI::TestVisibility::Transport? def build_evp_proxy_transport: (untyped settings, untyped agent_settings) -> Datadog::CI::TestVisibility::Transport def can_use_evp_proxy?: (untyped settings, untyped agent_settings) -> bool + def serializers_factory: (untyped settings) -> (singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel)) end end end diff --git a/sig/datadog/ci/test_visibility/transport.rbs b/sig/datadog/ci/test_visibility/transport.rbs index f94a68bf..20822d81 100644 --- a/sig/datadog/ci/test_visibility/transport.rbs +++ b/sig/datadog/ci/test_visibility/transport.rbs @@ -4,19 +4,19 @@ module Datadog class Transport DEFAULT_MAX_PAYLOAD_SIZE: Integer - attr_reader serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) + attr_reader serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) attr_reader dd_env: String? attr_reader api: Datadog::CI::Transport::Api::Base attr_reader max_payload_size: Integer @dd_env: String? - @serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) + @serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel) @max_payload_size: Integer def initialize: ( api: Datadog::CI::Transport::Api::Base, ?dd_env: ::String?, - ?serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel), + ?serializers_factory: singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestLevel) | singleton(Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel), ?max_payload_size: Integer ) -> void From b0537e36bee56d36a966891c9e827f584ddc9ad6 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 23 Nov 2023 16:40:59 +0100 Subject: [PATCH 11/16] use partial flush to send spans as they are finished within a session --- lib/datadog/ci/configuration/components.rb | 2 +- spec/datadog/ci/configuration/components_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 2d485121..870dbb10 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -50,7 +50,7 @@ def activate_ci!(settings) settings.tracing.test_mode.enabled = true # Choose user defined TraceFlush or default to CI TraceFlush - settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Finished.new + settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Partial.new writer_options = settings.ci.writer_options if test_visibility_transport diff --git a/spec/datadog/ci/configuration/components_spec.rb b/spec/datadog/ci/configuration/components_spec.rb index 401a48b2..67d00c8d 100644 --- a/spec/datadog/ci/configuration/components_spec.rb +++ b/spec/datadog/ci/configuration/components_spec.rb @@ -148,7 +148,7 @@ expect(settings.tracing.test_mode) .to have_received(:trace_flush=) - .with(settings.ci.trace_flush || kind_of(Datadog::CI::TestVisibility::Flush::Finished)) + .with(settings.ci.trace_flush || kind_of(Datadog::CI::TestVisibility::Flush::Partial)) expect(settings.tracing.test_mode).to have_received(:writer_options=) do |options| expect(options[:transport]).to be_nil From c24f2899596891b1d11398f41fdcef53d9677560 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 24 Nov 2023 08:35:06 +0100 Subject: [PATCH 12/16] Jruby doesn't have Process.argv0, replace with $0 --- lib/datadog/ci/utils/test_run.rb | 2 +- spec/datadog/ci/utils/test_run_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datadog/ci/utils/test_run.rb b/lib/datadog/ci/utils/test_run.rb index b3309a79..1e41715e 100644 --- a/lib/datadog/ci/utils/test_run.rb +++ b/lib/datadog/ci/utils/test_run.rb @@ -7,7 +7,7 @@ module TestRun def self.command return @command if defined?(@command) - @command = "#{Process.argv0} #{ARGV.join(" ")}" + @command = "#{$0} #{ARGV.join(" ")}" end end end diff --git a/spec/datadog/ci/utils/test_run_spec.rb b/spec/datadog/ci/utils/test_run_spec.rb index 34247408..7f84b65e 100644 --- a/spec/datadog/ci/utils/test_run_spec.rb +++ b/spec/datadog/ci/utils/test_run_spec.rb @@ -4,6 +4,6 @@ describe ".command" do subject { described_class.command } - it { is_expected.to eq("#{Process.argv0} #{ARGV.join(" ")}") } + it { is_expected.to eq("#{$0} #{ARGV.join(" ")}") } end end From c60cf731ccb243410828117b82a4a11f17608601 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 24 Nov 2023 08:45:50 +0100 Subject: [PATCH 13/16] test service propagation from session --- spec/support/tracer_helpers.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/support/tracer_helpers.rb b/spec/support/tracer_helpers.rb index ceac8cb3..729232bf 100644 --- a/spec/support/tracer_helpers.rb +++ b/spec/support/tracer_helpers.rb @@ -70,7 +70,9 @@ def produce_test_session_trace( produce_test_trace( framework: framework, operation: operation, test_name: test_name, test_suite: test_suite, - service: service, result: result, exception: exception, skip_reason: skip_reason, + # service is inherited from test_session + service: nil, + result: result, exception: exception, skip_reason: skip_reason, start_time: start_time, duration_seconds: duration_seconds, with_http_span: with_http_span ) From 93138cf6ccfac9dc871c4368146104198b4475a9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 24 Nov 2023 09:29:30 +0100 Subject: [PATCH 14/16] use SpanOperation#set_tags, do not synchronize methods that set multiple tags one by one --- lib/datadog/ci/concurrent_span.rb | 8 -------- lib/datadog/ci/span.rb | 4 +--- spec/datadog/ci/span_spec.rb | 3 +-- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/lib/datadog/ci/concurrent_span.rb b/lib/datadog/ci/concurrent_span.rb index 9910d0dc..19c52713 100644 --- a/lib/datadog/ci/concurrent_span.rb +++ b/lib/datadog/ci/concurrent_span.rb @@ -72,14 +72,6 @@ def set_tags(tags) synchronize { super } end - def set_environment_runtime_tags - synchronize { super } - end - - def set_default_tags - synchronize { super } - end - def synchronize @mutex.synchronize { yield } end diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb index df57eaa7..08f8cffe 100644 --- a/lib/datadog/ci/span.rb +++ b/lib/datadog/ci/span.rb @@ -94,9 +94,7 @@ def finish # @param [Hash[String, String]] tags the tags to set. # @return [void] def set_tags(tags) - tags.each do |key, value| - tracer_span.set_tag(key, value) - end + tracer_span.set_tags(tags) end def set_environment_runtime_tags diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index 639fb8d8..6519d90c 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -87,8 +87,7 @@ describe "#set_tags" do it "sets the tags" do - expect(tracer_span).to receive(:set_tag).with("foo", "bar") - expect(tracer_span).to receive(:set_tag).with("baz", "qux") + expect(tracer_span).to receive(:set_tags).with({"foo" => "bar", "baz" => "qux"}) span.set_tags("foo" => "bar", "baz" => "qux") end From 89ef625a19d903e3ce04c284193f7e18558d1585 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 24 Nov 2023 13:33:14 +0100 Subject: [PATCH 15/16] ConcurrentSpan tests, add missing test coverage --- lib/datadog/ci/ext/app_types.rb | 2 + sig/datadog/ci/ext/app_types.rbs | 2 + spec/datadog/ci/concurrent_span_spec.rb | 118 ++++++++++++++++++ spec/datadog/ci/recorder_spec.rb | 6 +- .../factories/test_suite_level_spec.rb | 7 +- .../serializers/test_session_spec.rb | 24 ++++ .../serializers/test_v2_spec.rb | 24 ++++ spec/support/tracer_helpers.rb | 6 +- 8 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 spec/datadog/ci/concurrent_span_spec.rb diff --git a/lib/datadog/ci/ext/app_types.rb b/lib/datadog/ci/ext/app_types.rb index f28230a7..a4a36529 100644 --- a/lib/datadog/ci/ext/app_types.rb +++ b/lib/datadog/ci/ext/app_types.rb @@ -6,6 +6,8 @@ module Ext module AppTypes TYPE_TEST = "test" TYPE_TEST_SESSION = "test_session_end" + + CI_SPAN_TYPES = [TYPE_TEST, TYPE_TEST_SESSION].freeze end end end diff --git a/sig/datadog/ci/ext/app_types.rbs b/sig/datadog/ci/ext/app_types.rbs index 83cd3e79..1f01f77b 100644 --- a/sig/datadog/ci/ext/app_types.rbs +++ b/sig/datadog/ci/ext/app_types.rbs @@ -4,6 +4,8 @@ module Datadog module AppTypes TYPE_TEST: "test" TYPE_TEST_SESSION: "test_session_end" + + CI_SPAN_TYPES: Array[String] end end end diff --git a/spec/datadog/ci/concurrent_span_spec.rb b/spec/datadog/ci/concurrent_span_spec.rb new file mode 100644 index 00000000..88074fb6 --- /dev/null +++ b/spec/datadog/ci/concurrent_span_spec.rb @@ -0,0 +1,118 @@ +THREADS_COUNT = 10 +REPEAT_COUNT = 20 + +RSpec.describe Datadog::CI::ConcurrentSpan do + describe "#finish" do + it "calls SpanOperation#stop once" do + REPEAT_COUNT.times do + tracer_span = Datadog::Tracing::SpanOperation.new("operation") + ci_span = described_class.new(tracer_span) + + expect(tracer_span).to receive(:stop).once + + (1..THREADS_COUNT).map do + Thread.new do + ci_span.finish + end + end.map(&:join) + end + end + end + + # the following tests make sure that ConcurrentSpan works exactly like Span + let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, name: "span_name", type: "test") } + subject(:span) { described_class.new(tracer_span) } + + describe "#passed!" do + it "sets the status to PASS" do + expect(tracer_span).to receive(:set_tag).with("test.status", "pass") + + span.passed! + end + end + + describe "#failed!" do + context "when exception is nil" do + it "sets the status to FAIL" do + expect(tracer_span).to receive(:status=).with(1) + expect(tracer_span).to receive(:set_tag).with("test.status", "fail") + + span.failed! + end + end + + context "when exception is provided" do + it "sets the status to FAIL" do + expect(tracer_span).to receive(:status=).with(1) + expect(tracer_span).to receive(:set_tag).with("test.status", "fail") + expect(tracer_span).to receive(:set_error).with("error") + + span.failed!(exception: "error") + end + end + end + + describe "#skipped!" do + context "when exception is nil" do + it "sets the status to SKIP" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).not_to receive(:set_error) + + span.skipped! + end + end + + context "when exception is provided" do + it "sets the status to SKIP and sets error" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).to receive(:set_error).with("error") + + span.skipped!(exception: "error") + end + end + + context "when reason is nil" do + it "doesn't set the skip reason tag" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).not_to receive(:set_tag).with("test.skip_reason", "reason") + + span + + span.skipped! + end + end + + context "when reason is provided" do + it "sets the skip reason tag" do + expect(tracer_span).to receive(:set_tag).with("test.status", "skip") + expect(tracer_span).to receive(:set_tag).with("test.skip_reason", "reason") + + span.skipped!(reason: "reason") + end + end + end + + describe "#set_tag" do + it "sets the tag" do + expect(tracer_span).to receive(:set_tag).with("foo", "bar") + + span.set_tag("foo", "bar") + end + end + + describe "#set_tags" do + it "sets the tags" do + expect(tracer_span).to receive(:set_tags).with({"foo" => "bar", "baz" => "qux"}) + + span.set_tags("foo" => "bar", "baz" => "qux") + end + end + + describe "#set_metric" do + it "sets the metric" do + expect(tracer_span).to receive(:set_metric).with("foo", "bar") + + span.set_metric("foo", "bar") + end + end +end diff --git a/spec/datadog/ci/recorder_spec.rb b/spec/datadog/ci/recorder_spec.rb index 80c92d66..58b186f2 100644 --- a/spec/datadog/ci/recorder_spec.rb +++ b/spec/datadog/ci/recorder_spec.rb @@ -409,7 +409,11 @@ recorder.start_test_session(service_name: service) end - before { deactivate_test_session } + before do + ci_session + + deactivate_test_session + end it { expect(recorder.active_test_session).to be_nil } end diff --git a/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb b/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb index 91ed48f9..3bd9c945 100644 --- a/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb +++ b/spec/datadog/ci/test_visibility/serializers/factories/test_suite_level_spec.rb @@ -9,7 +9,7 @@ end before do - produce_test_session_trace + produce_test_session_trace(with_http_span: true) end subject { described_class.serializer(trace, ci_span) } @@ -24,5 +24,10 @@ let(:ci_span) { first_test_span } it { is_expected.to be_kind_of(Datadog::CI::TestVisibility::Serializers::TestV2) } end + + context "with a http request span" do + let(:ci_span) { first_other_span } + it { is_expected.to be_kind_of(Datadog::CI::TestVisibility::Serializers::Span) } + end end end diff --git a/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb index 8c0de0b0..bb05f5e9 100644 --- a/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb +++ b/spec/datadog/ci/test_visibility/serializers/test_session_spec.rb @@ -56,4 +56,28 @@ end end end + + describe "#valid?" do + context "test_session_id" do + before do + produce_test_session_trace + end + + context "when test_session_id is not nil" do + it "returns true" do + expect(subject.valid?).to eq(true) + end + end + + context "when test_session_id is nil" do + before do + test_session_span.clear_tag("_test.session_id") + end + + it "returns false" do + expect(subject.valid?).to eq(false) + end + end + end + end end diff --git a/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb b/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb index 32ccb4fc..46732848 100644 --- a/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb +++ b/spec/datadog/ci/test_visibility/serializers/test_v2_spec.rb @@ -74,4 +74,28 @@ end end end + + describe "#valid?" do + context "test_session_id" do + before do + produce_test_session_trace + end + + context "when test_session_id is not nil" do + it "returns true" do + expect(subject.valid?).to eq(true) + end + end + + context "when test_session_id is nil" do + before do + first_test_span.clear_tag("_test.session_id") + end + + it "returns false" do + expect(subject.valid?).to eq(false) + end + end + end + end end diff --git a/spec/support/tracer_helpers.rb b/spec/support/tracer_helpers.rb index 729232bf..a1ca5bd6 100644 --- a/spec/support/tracer_helpers.rb +++ b/spec/support/tracer_helpers.rb @@ -58,6 +58,10 @@ def produce_test_session_trace( skip_reason: nil, start_time: Time.now, duration_seconds: 2, with_http_span: false ) + allow(Process).to receive(:clock_gettime).and_return( + 0, duration_seconds, 2 * duration_seconds, 3 * duration_seconds + ) + test_session = Datadog::CI.start_test_session( service_name: service, tags: { @@ -90,7 +94,7 @@ def first_test_span end def first_other_span - spans.find { |span| span.type != "test" } + spans.find { |span| !Datadog::CI::Ext::AppTypes::CI_SPAN_TYPES.include?(span.type) } end # Returns spans and caches it (similar to +let(:spans)+). From d656b42e8d524ea8e73235fcebdd6d9b2a3609dc Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 24 Nov 2023 14:09:54 +0100 Subject: [PATCH 16/16] remove synchronization around methods that set multiple tags as mutext is not reentrant, keep only smallest operations synchronized --- lib/datadog/ci/concurrent_span.rb | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/lib/datadog/ci/concurrent_span.rb b/lib/datadog/ci/concurrent_span.rb index 19c52713..994e5038 100644 --- a/lib/datadog/ci/concurrent_span.rb +++ b/lib/datadog/ci/concurrent_span.rb @@ -15,27 +15,6 @@ def initialize(tracer_span) @mutex = Mutex.new end - # Sets the status of the span to "pass". This method is thread-safe. - # @return [void] - def passed! - synchronize { super } - end - - # Sets the status of the span to "fail". This method is thread-safe. - # @param [Exception] exception the exception that caused the test to fail. - # @return [void] - def failed!(exception: nil) - synchronize { super } - end - - # Sets the status of the span to "skip". This method is thread-safe. - # @param [Exception] exception the exception that caused the test to fail. - # @param [String] reason the reason why the test was skipped. - # @return [void] - def skipped!(exception: nil, reason: nil) - synchronize { super } - end - # Gets tag value by key. This method is thread-safe. # @param [String] key the key of the tag. # @return [String] the value of the tag.