Kbauer/add tf based bare metal test

.github/workflows/ci_nightly.yaml

@@ -56,7 +56,7 @@ jobs:
           cluster_name: ${{ env.TEST_CLUSTER_NAME }}
           wait: 60s
 
-      - name: Run local e2e tests
+      - name: Run slow local tests
         run: |
           IMAGE_TAG=${{ env.version }}-rc-amd64 \
           KIND_CLUSTER_NAME=${{ env.TEST_CLUSTER_NAME }} \
@@ -87,3 +87,10 @@ jobs:
           distribution: goreleaser
           version: '~> v2'
           args: --skip=announce,validate --clean --timeout 2h --config .goreleaser-nightly.yaml
+
+      - name: Run nightly tests
+        run: |
+          NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \
+          NR_ACCOUNT_ID=${{ secrets.OTELCOMM_NR_TEST_ACCOUNT_ID }} \
+          NR_API_BASE_URL=${{ secrets.NR_STAGING_API_BASE_URL }} \
+          make -f ./test/e2e/Makefile ci_test-nightly

test/charts/nr_backend/templates/daemonset.yaml

@@ -29,6 +29,10 @@ spec:
             - name: health
               containerPort: 13133
           env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
             - name: OTEL_EXPORTER_OTLP_ENDPOINT
               valueFrom:
                 secretKeyRef:
@@ -42,7 +46,7 @@ spec:
                   name: daemonset-secrets
                   key: nrIngestKey
             - name: OTEL_RESOURCE_ATTRIBUTES
-              value: "host.name={{ .Values.collector.hostname }}"
+              value: "host.name={{ .Values.collector.hostname }}-$(NODE_NAME)"
 ---
 apiVersion: v1
 kind: Secret

test/e2e/Makefile

@@ -35,14 +35,18 @@ ci_build-load-mocked-otlp-image: assert_cluster-exists
 
 .PHONY: ci_test-fast
 ci_test-fast: TEST_MODE=fastOnly
-ci_test-fast: ci_test
+ci_test-fast: ci_load-image ci_build-load-mocked-otlp-image ci_test
 
 .PHONY: ci_test-slow
 ci_test-slow: TEST_MODE=slowOnly
-ci_test-slow: ci_test
+ci_test-slow: ci_load-image ci_test
+
+.PHONY: ci_test-nightly
+ci_test-nightly: TEST_MODE=nightlyOnly
+ci_test-nightly: ci_test
 
 .PHONY: ci_test
-ci_test: ci_load-image ci_build-load-mocked-otlp-image
+ci_test:
 	cd ${THIS_MAKEFILE_DIR} && \
 	E2E_TEST__K8S_CONTEXT_NAME=${K8S_CONTEXT_NAME} \
 	E2E_TEST__IMAGE_TAG=${IMAGE_TAG} \

test/e2e/hostmetrics/hostmetrics_test.go

@@ -46,15 +46,11 @@ func setupTest(tb testing.TB) testEnv {
 	}}
 }
 
-func TestMain(m *testing.M) {
-	testId = testutil.NewTestId()
-	kubectlOptions = k8sutil.NewKubectlOptions(TestNamespace)
-	testChart = chart.MockedBackend
-	m.Run()
-}
-
 func TestStartupBehavior(t *testing.T) {
 	testutil.TagAsFastTest(t)
+	kubectlOptions = k8sutil.NewKubectlOptions(TestNamespace)
+	testChart = chart.MockedBackend
+	testId = testutil.NewTestId()
 	cleanup := helmutil.ApplyChart(t, kubectlOptions, testChart.AsChart(), "hostmetrics-startup", testId)
 	defer cleanup()
 

test/e2e/hostmetrics_nightly/hostmetrics_nightly_test.go

@@ -0,0 +1,52 @@
+package hostmetrics
+
+import (
+	"fmt"
+	"test/e2e/util/assert"
+	"test/e2e/util/nr"
+	"test/e2e/util/spec"
+	testutil "test/e2e/util/test"
+	"testing"
+	"time"
+)
+
+type systemUnderTest struct {
+	hostNamePattern string
+	excludedMetrics []string
+}
+
+var ec2Ubuntu22 = systemUnderTest{
+	hostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "ec2_ubuntu22_04"),
+}
+var ec2Ubuntu24 = systemUnderTest{
+	hostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "ec2_ubuntu24_04"),
+}
+var k8sNode = systemUnderTest{
+	hostNamePattern: testutil.NewNrQueryHostNamePattern("nightly", testutil.Wildcard, "k8s_node"),
+	excludedMetrics: []string{"system.paging.usage"},
+}
+
+func TestNightlyHostMetrics(t *testing.T) {
+	testutil.TagAsNightlyTest(t)
+
+	// space out requests to not run into 25 concurrent request limit
+	requestsPerSecond := 4.0
+	requestSpacing := time.Duration((1/requestsPerSecond)*1000) * time.Millisecond
+	client := nr.NewClient()
+
+	for _, sut := range []systemUnderTest{ec2Ubuntu22, ec2Ubuntu24, k8sNode} {
+		for i, testCase := range spec.GetOnHostTestCasesWithout(sut.excludedMetrics) {
+			t.Run(fmt.Sprintf("%s/%d/%s", sut.hostNamePattern, i, testCase.Name), func(t *testing.T) {
+				t.Parallel()
+				assertionFactory := assert.NewNrMetricAssertionFactory(
+					fmt.Sprintf("WHERE host.name like '%s'", sut.hostNamePattern),
+					"2 hour ago",
+				)
+				assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
+				// space out requests to avoid rate limiting
+				time.Sleep(time.Duration(i) * requestSpacing)
+				assertion.Execute(t, client)
+			})
+		}
+	}
+}

test/e2e/hostmetrics_slow/hostmetrics_slow_test.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"test/e2e/util/assert"
 	"test/e2e/util/chart"
-	envutil "test/e2e/util/env"
 	helmutil "test/e2e/util/helm"
 	k8sutil "test/e2e/util/k8s"
 	"test/e2e/util/nr"
@@ -22,10 +21,8 @@ const (
 )
 
 var (
-	kubectlOptions            *k8s.KubectlOptions
-	testChart                 chart.NrBackendChart
-	testId                    string
-	collectorReportedHostname string
+	kubectlOptions *k8s.KubectlOptions
+	testChart      chart.NrBackendChart
 )
 
 type testEnv struct {
@@ -42,22 +39,13 @@ func setupTest(tb testing.TB) testEnv {
 
 }
 
-func TestMain(m *testing.M) {
-	testId = testutil.NewTestId()
-	kubectlOptions = k8sutil.NewKubectlOptions(TestNamespace)
-	environmentName := "local"
-	if envutil.IsContinuousIntegration() {
-		environmentName = "ci"
-	}
-	collectorReportedHostname = fmt.Sprintf("nr-otel-collector-%s-%s", environmentName, testId)
-	testChart = chart.NewNrBackendChart(collectorReportedHostname)
-	m.Run()
-}
-
 func TestStartupBehavior(t *testing.T) {
 	testutil.TagAsSlowTest(t)
+	kubectlOptions = k8sutil.NewKubectlOptions(TestNamespace)
+	testId := testutil.NewTestId()
+	testChart = chart.NewNrBackendChart(testId)
 
-	t.Logf("host.name used for test: %s", collectorReportedHostname)
+	t.Logf("hostname used for test: %s", testChart.NrQueryHostNamePattern)
 	cleanup := helmutil.ApplyChart(t, kubectlOptions, testChart.AsChart(), "hostmetrics-startup", testId)
 	t.Cleanup(cleanup)
 	te := setupTest(t)
@@ -69,15 +57,15 @@ func TestStartupBehavior(t *testing.T) {
 	// space out requests to not run into 25 concurrent request limit
 	requestsPerSecond := 4.0
 	requestSpacing := time.Duration((1/requestsPerSecond)*1000) * time.Millisecond
+	client := nr.NewClient()
 
 	for i, testCase := range spec.GetOnHostTestCases() {
 		t.Run(fmt.Sprintf(testCase.Name), func(t *testing.T) {
 			t.Parallel()
 			assertionFactory := assert.NewNrMetricAssertionFactory(
-				fmt.Sprintf("WHERE host.name = '%s'", collectorReportedHostname),
+				fmt.Sprintf("WHERE host.name like '%s'", testChart.NrQueryHostNamePattern),
 				"5 minutes ago",
 			)
-			client := nr.NewClient()
 			assertion := assertionFactory.NewNrMetricAssertion(testCase.Metric, testCase.Assertions)
 			// space out requests to avoid rate limiting
 			time.Sleep(time.Duration(i) * requestSpacing)

test/e2e/util/chart/nr_backend.go

@@ -1,14 +1,35 @@
 package chart
 
-import envutil "test/e2e/util/env"
+import (
+	"fmt"
+	"log"
+	osuser "os/user"
+	envutil "test/e2e/util/env"
+	testutil "test/e2e/util/test"
+)
 
 type NrBackendChart struct {
-	collectorHostname string
+	collectorHostNamePrefix string
+	NrQueryHostNamePattern  string
 }
 
-func NewNrBackendChart(collectorHostname string) NrBackendChart {
+func NewNrBackendChart(testId string) NrBackendChart {
+	var environmentName string
+	if envutil.IsContinuousIntegration() {
+		environmentName = "ci"
+	} else {
+		user, err := osuser.Current()
+		if err != nil {
+			log.Panicf("Couldn't determine current user: %v", err)
+		}
+		environmentName = fmt.Sprintf("local_%s", user.Username)
+	}
+	hostNamePrefix := testutil.NewHostNamePrefix(environmentName, testId, "k8s-node")
+	hostNamePattern := testutil.NewNrQueryHostNamePattern(environmentName, testId, "k8s-node")
+
 	return NrBackendChart{
-		collectorHostname: collectorHostname,
+		collectorHostNamePrefix: hostNamePrefix,
+		NrQueryHostNamePattern:  hostNamePattern,
 	}
 }
 
@@ -28,6 +49,6 @@ func (m *NrBackendChart) RequiredChartValues() map[string]string {
 		"image.tag":            envutil.GetImageTag(),
 		"secrets.nrBackendUrl": envutil.GetNrBackendUrl(),
 		"secrets.nrIngestKey":  envutil.GetNrIngestKey(),
-		"collector.hostname":   m.collectorHostname,
+		"collector.hostname":   m.collectorHostNamePrefix,
 	}
 }

test/e2e/util/spec/on_host.go

@@ -96,7 +96,7 @@ var testCases = []TestCase{
 			Name: "system.cpu.load_average.15m",
 		},
 		Assertions: []assert.NrAssertion{
-			{AggregationFunction: "max", ComparisonOperator: ">", Threshold: 0},
+			{AggregationFunction: "max", ComparisonOperator: ">=", Threshold: 0},
 		}},
 	{
 		Name: "host receiver memory.usage cached",
@@ -303,3 +303,19 @@ var testCases = []TestCase{
 func GetOnHostTestCases() []TestCase {
 	return testCases
 }
+
+func GetOnHostTestCasesWithout(excludedMetric []string) []TestCase {
+	var filteredTestCases []TestCase
+	for _, testCase := range testCases {
+		included := true
+		for _, skippedMetric := range excludedMetric {
+			if testCase.Metric.Name == skippedMetric {
+				included = false
+			}
+		}
+		if included {
+			filteredTestCases = append(filteredTestCases, testCase)
+		}
+	}
+	return filteredTestCases
+}

test/e2e/util/test/hostname.go

@@ -0,0 +1,15 @@
+package test
+
+import "fmt"
+
+func NewHostNamePrefix(envName string, deployId string, hostType string) string {
+	// only a prefix as helm chart appends hostId
+	return fmt.Sprintf("%s-%s-%s", envName, deployId, hostType)
+}
+
+const Wildcard = "%"
+
+func NewNrQueryHostNamePattern(envName string, deployId string, hostType string) string {
+	hostId := Wildcard
+	return fmt.Sprintf("%s-%s-%s-%s", envName, deployId, hostType, hostId)
+}

test/e2e/util/test/test.go

@@ -8,24 +8,46 @@ import (
 )
 
 const (
-	slowOnly = "slowOnly"
-	fastOnly = "fastOnly"
+	fastOnly    = "fastOnly"
+	slowOnly    = "slowOnly"
+	nightlyOnly = "nightlyOnly"
 )
 
+var onlyModes = []string{fastOnly, slowOnly, nightlyOnly}
+
+func TagAsFastTest(t *testing.T) {
+	if isAnyModeOf(allOnlyModesExcept(fastOnly)) {
+		t.Skip("Skipping fast test: ", t.Name())
+	}
+}
+
 func TagAsSlowTest(t *testing.T) {
-	if isModeEnabled(fastOnly) {
+	if isAnyModeOf(allOnlyModesExcept(slowOnly)) {
 		t.Skip("Skipping slow test: ", t.Name())
 	}
 }
 
-func TagAsFastTest(t *testing.T) {
-	if isModeEnabled(slowOnly) {
-		t.Skip("Skipping fast test: ", t.Name())
+func TagAsNightlyTest(t *testing.T) {
+	if isAnyModeOf(allOnlyModesExcept(nightlyOnly)) {
+		t.Skip("Skipping nightly test: ", t.Name())
 	}
 }
 
-func isModeEnabled(mode string) bool {
-	return strings.Contains(envutil.GetTestMode(), mode)
+func isAnyModeOf(modes []string) bool {
+	result := false
+	for _, mode := range modes {
+		result = result || strings.Contains(envutil.GetTestMode(), mode)
+	}
+	return result
+}
+func allOnlyModesExcept(filterOut string) []string {
+	var result []string
+	for _, onlyMode := range onlyModes {
+		if onlyMode != filterOut {
+			result = append(result, onlyMode)
+		}
+	}
+	return result
 }
 
 func NewTestId() string {