Merge pull request #95 from DataDog/anmarchenko/source_code_integration

[CIVIS-3035] Source code tags integration
DataDog · Jan 5, 2024 · 867b86e · 867b86e
2 parents b117666 + 9394d72
commit 867b86e
Show file tree

Hide file tree

Showing 21 changed files with 207 additions and 36 deletions.
diff --git a/gemfiles/ruby_3.3_cucumber_3.gemfile.lock b/gemfiles/ruby_3.3_cucumber_3.gemfile.lock
@@ -45,9 +45,12 @@ GEM
     gherkin (5.1.0)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     msgpack (1.7.2)
@@ -131,6 +134,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   appraisal

diff --git a/gemfiles/ruby_3.3_cucumber_4.gemfile.lock b/gemfiles/ruby_3.3_cucumber_4.gemfile.lock
@@ -62,9 +62,12 @@ GEM
       concurrent-ruby (~> 1.0)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     middleware (0.1.0)
@@ -159,6 +162,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   activesupport (< 7.1)

diff --git a/gemfiles/ruby_3.3_cucumber_5.gemfile.lock b/gemfiles/ruby_3.3_cucumber_5.gemfile.lock
@@ -66,9 +66,12 @@ GEM
       concurrent-ruby (~> 1.0)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     middleware (0.1.0)
@@ -163,6 +166,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   activesupport (< 7.1)

diff --git a/gemfiles/ruby_3.3_cucumber_6.gemfile.lock b/gemfiles/ruby_3.3_cucumber_6.gemfile.lock
@@ -67,9 +67,12 @@ GEM
       concurrent-ruby (~> 1.0)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     middleware (0.1.0)
@@ -167,6 +170,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   activesupport (< 7.1)

diff --git a/gemfiles/ruby_3.3_cucumber_7.gemfile.lock b/gemfiles/ruby_3.3_cucumber_7.gemfile.lock
@@ -57,9 +57,12 @@ GEM
     ffi (1.16.3)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     mime-types (3.5.1)
@@ -147,6 +150,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   appraisal

diff --git a/gemfiles/ruby_3.3_cucumber_8.gemfile.lock b/gemfiles/ruby_3.3_cucumber_8.gemfile.lock
@@ -51,9 +51,12 @@ GEM
     ffi (1.16.3)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     mime-types (3.5.1)
@@ -141,6 +144,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   appraisal

diff --git a/gemfiles/ruby_3.3_minitest_5.gemfile.lock b/gemfiles/ruby_3.3_minitest_5.gemfile.lock
@@ -26,9 +26,12 @@ GEM
     ffi (1.16.3)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     minitest (5.20.0)
@@ -111,6 +114,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   appraisal

diff --git a/gemfiles/ruby_3.3_rspec_3.gemfile.lock b/gemfiles/ruby_3.3_rspec_3.gemfile.lock
@@ -26,9 +26,12 @@ GEM
     ffi (1.16.3)
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    libdatadog (5.0.0.1.0)
     libdatadog (5.0.0.1.0-aarch64-linux)
     libddwaf (1.14.0.0.0-aarch64-linux)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-arm64-darwin)
+      ffi (~> 1.0)
     lint_roller (1.1.0)
     method_source (1.0.0)
     msgpack (1.7.2)
@@ -110,6 +113,7 @@ GEM
 
 PLATFORMS
   aarch64-linux
+  arm64-darwin-23
 
 DEPENDENCIES
   appraisal

diff --git a/lib/datadog/ci/contrib/cucumber/formatter.rb b/lib/datadog/ci/contrib/cucumber/formatter.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../ext/test"
+require_relative "../../utils/git"
 require_relative "ext"
 
 module Datadog
@@ -61,7 +62,9 @@ def on_test_case_started(event)
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
+                CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(event.test_case.location.file),
+                CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s
               },
               service: configuration[:service_name]
             )

diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb
@@ -24,13 +24,17 @@ def before_setup
               CI.start_test_suite(test_suite_name)
             end
 
+            source_file, line_number = method(name).source_location
+
             CI.start_test(
               test_name,
               test_suite_name,
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s,
-                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
+                CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(source_file),
+                CI::Ext::Test::TAG_SOURCE_START => line_number.to_s
               },
               service: datadog_configuration[:service_name]
             )

diff --git a/lib/datadog/ci/contrib/rspec/example.rb b/lib/datadog/ci/contrib/rspec/example.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../../ext/test"
+require_relative "../../utils/git"
 require_relative "ext"
 
 module Datadog
@@ -30,7 +31,9 @@ def run(example_group_instance, reporter)
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE
+                  CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE,
+                  CI::Ext::Test::TAG_SOURCE_FILE => Utils::Git.relative_to_root(metadata[:file_path]),
+                  CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
                 },
                 service: configuration[:service_name]
               ) do |test_span|

diff --git a/lib/datadog/ci/ext/environment/providers/local_git.rb b/lib/datadog/ci/ext/environment/providers/local_git.rb
@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require "open3"
-
 require_relative "base"
-require_relative "../../git"
+require_relative "../../../utils/git"
 
 module Datadog
   module CI
@@ -13,7 +11,7 @@ module Providers
           # As a fallback we try to fetch git information from the local git repository
           class LocalGit < Base
             def git_repository_url
-              exec_git_command("git ls-remote --get-url")
+              Utils::Git.exec_git_command("git ls-remote --get-url")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git repository url: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -22,7 +20,7 @@ def git_repository_url
             end
 
             def git_commit_sha
-              exec_git_command("git rev-parse HEAD")
+              Utils::Git.exec_git_command("git rev-parse HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git commit SHA: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -31,7 +29,7 @@ def git_commit_sha
             end
 
             def git_branch
-              exec_git_command("git rev-parse --abbrev-ref HEAD")
+              Utils::Git.exec_git_command("git rev-parse --abbrev-ref HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git branch: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -40,7 +38,7 @@ def git_branch
             end
 
             def git_tag
-              exec_git_command("git tag --points-at HEAD")
+              Utils::Git.exec_git_command("git tag --points-at HEAD")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git tag: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -49,7 +47,7 @@ def git_tag
             end
 
             def git_commit_message
-              exec_git_command("git show -s --format=%s")
+              Utils::Git.exec_git_command("git show -s --format=%s")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git commit message: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -82,7 +80,7 @@ def git_commit_committer_date
             end
 
             def workspace_path
-              exec_git_command("git rev-parse --show-toplevel")
+              Utils::Git.exec_git_command("git rev-parse --show-toplevel")
             rescue => e
               Datadog.logger.debug(
                 "Unable to read git base directory: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
@@ -92,25 +90,6 @@ def workspace_path
 
             private
 
-            def exec_git_command(cmd)
-              out, status = Open3.capture2e(cmd)
-
-              raise "Failed to run git command #{cmd}: #{out}" unless status.success?
-
-              # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
-              # commit messages with UTF-8 characters like emojis
-              # We force output's encoding to be UTF-8 in this case
-              # This is safe to do as UTF-8 is compatible with US-ASCII
-              if Encoding.default_external == Encoding::US_ASCII
-                out = out.force_encoding(Encoding::UTF_8)
-              end
-              out.strip! # There's always a "\n" at the end of the command output
-
-              return nil if out.empty?
-
-              out
-            end
-
             def author
               return @author if defined?(@author)
 
@@ -127,7 +106,7 @@ def committer
 
             def set_git_commit_users
               # Get committer and author information in one command.
-              output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
+              output = Utils::Git.exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
               unless output
                 Datadog.logger.debug(
                   "Unable to read git commit users: git command output is nil"

diff --git a/lib/datadog/ci/ext/test.rb b/lib/datadog/ci/ext/test.rb
@@ -19,6 +19,8 @@ module Test
         TAG_TRAITS = "test.traits"
         TAG_TYPE = "test.type"
         TAG_COMMAND = "test.command"
+        TAG_SOURCE_FILE = "test.source.file"
+        TAG_SOURCE_START = "test.source.start"
 
         TEST_TYPE = "test"
 

diff --git a/lib/datadog/ci/utils/git.rb b/lib/datadog/ci/utils/git.rb
@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "open3"
+require "pathname"
+
 module Datadog
   module CI
     module Utils
@@ -16,6 +19,48 @@ def self.normalize_ref(ref)
         def self.is_git_tag?(ref)
           !ref.nil? && ref.include?("tags/")
         end
+
+        def self.root
+          return @@root if defined?(@@root)
+
+          @@root = exec_git_command("git rev-parse --show-toplevel")
+        rescue => e
+          Datadog.logger.debug(
+            "Unable to read git root: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+          )
+          @@root = nil
+        end
+
+        def self.relative_to_root(path)
+          return nil if path.nil?
+
+          git_root = root
+          return path if git_root.nil?
+
+          path = Pathname.new(File.expand_path(path))
+          git_root = Pathname.new(git_root)
+
+          path.relative_path_from(git_root).to_s
+        end
+
+        def self.exec_git_command(cmd)
+          out, status = Open3.capture2e(cmd)
+
+          raise "Failed to run git command #{cmd}: #{out}" unless status.success?
+
+          # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
+          # commit messages with UTF-8 characters like emojis
+          # We force output's encoding to be UTF-8 in this case
+          # This is safe to do as UTF-8 is compatible with US-ASCII
+          if Encoding.default_external == Encoding::US_ASCII
+            out = out.force_encoding(Encoding::UTF_8)
+          end
+          out.strip! # There's always a "\n" at the end of the command output
+
+          return nil if out.empty?
+
+          out
+        end
       end
     end
   end