Merge remote-tracking branch 'upstream/main' into dependabot/go_modul…

…es/github.com/elastic/go-elasticsearch/v8-8.11.0

.buildkite/scripts/steps/integration_tests.sh

@@ -11,7 +11,7 @@ MAGE_SUBTARGET="${3:-""}"
 # Override the agent package version using a string with format <major>.<minor>.<patch>
 # NOTE: use only after version bump when the new version is not yet available, for example:
 # OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="".
-OVERRIDE_AGENT_PACKAGE_VERSION=""
+OVERRIDE_AGENT_PACKAGE_VERSION="8.13.0"
 
 if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
   OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"

.github/workflows/opentelemetry.yml

@@ -1,15 +1,16 @@
 ---
+# Look up results at https://ela.st/oblt-ci-cd-stats
+# There will be one service per GitHub repository, including the org name, and one Transaction per Workflow.
 name: OpenTelemetry Export Trace
 
 on:
   workflow_run:
-    workflows:
-      - bump-golang
-      - golangci-lint
-      - Elastic Agent workflow using Snyk
-      - macos
+    workflows: [ "*" ]
     types: [completed]
 
+permissions:
+  contents: read
+
 jobs:
   otel-export-trace:
     runs-on: ubuntu-latest

.mergify.yml

@@ -293,3 +293,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.13 branch
+    conditions:
+      - merged
+      - label=backport-v8.13.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.13"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

changelog/fragments/1706213166-fqdn-ctx.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Set timeout of 1 minute for FQDN lookups
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/4147
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1707405322-newpodupdater.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Make more selective the Pod autodiscovery upon node and namespace update events
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description:  Update the PodUpdater fucntion with additonal checks before actually triggering Pod watcher restarts
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/4226
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/beats/issues/37338

changelog/fragments/1707857612-windows_stop_before_uninstall.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: On Windows make sure the service is stopped before uninstalling.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1707999059-Fixes-an-issue-where-the-Elastic-Agent-did-not-utilize-the-download-settings-when-downloading-the-artifact-signature-file..yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fixes an issue where the Elastic Agent did not utilize the download settings when downloading the artifact signature file.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/4237

deploy/kubernetes/elastic-agent-managed-kubernetes.yaml

@@ -30,7 +30,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: elastic-agent
-          image: docker.elastic.co/beats/elastic-agent:8.13.0
+          image: docker.elastic.co/beats/elastic-agent:8.14.0
           env:
             # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode
             - name: FLEET_ENROLL

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -698,13 +698,13 @@ spec:
       #      - -c
       #      - >-
       #        mkdir -p /etc/elastic-agent/inputs.d &&
-      #        wget -O - https://github.com/elastic/elastic-agent/archive/8.13.tar.gz | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-8.13/deploy/kubernetes/elastic-agent-standalone/templates.d"
+      #        wget -O - https://github.com/elastic/elastic-agent/archive/8.14.tar.gz | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-8.14/deploy/kubernetes/elastic-agent-standalone/templates.d"
       #    volumeMounts:
       #      - name: external-inputs
       #        mountPath: /etc/elastic-agent/inputs.d
       containers:
         - name: elastic-agent-standalone
-          image: docker.elastic.co/beats/elastic-agent:8.13.0
+          image: docker.elastic.co/beats/elastic-agent:8.14.0
           args: ["-c", "/etc/elastic-agent/agent.yml", "-e"]
           env:
             # The basic authentication username used to connect to Elasticsearch

dev-tools/packaging/package_test.go

@@ -13,10 +13,13 @@ import (
 	"bufio"
 	"bytes"
 	"compress/gzip"
+	"crypto/sha512"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
+	"hash"
 	"io"
 	"os"
 	"path"
@@ -174,6 +177,8 @@ func checkTar(t *testing.T, file string) {
 		containingDir := strings.TrimSuffix(path.Base(file), ".tar.gz")
 		checkManifestFileContents(t, filepath.Join(tempExtractionPath, containingDir))
 	})
+
+	checkSha512PackageHash(t, file)
 }
 
 func checkZip(t *testing.T, file string) {
@@ -197,6 +202,8 @@ func checkZip(t *testing.T, file string) {
 		containingDir := strings.TrimSuffix(path.Base(file), ".zip")
 		checkManifestFileContents(t, filepath.Join(tempExtractionPath, containingDir))
 	})
+
+	checkSha512PackageHash(t, file)
 }
 
 func checkManifestFileContents(t *testing.T, extractedPackageDir string) {
@@ -939,3 +946,68 @@ func readDockerManifest(r io.Reader) (*dockerManifest, error) {
 
 	return manifests[0], nil
 }
+
+func checkSha512PackageHash(t *testing.T, packageFile string) {
+	t.Run("check hash file", func(t *testing.T) {
+		expectedHashFile := packageFile + ".sha512"
+		require.FileExists(t, expectedHashFile, "hash file for package %q should exist with name %q", packageFile, expectedHashFile)
+
+		// calculate SHA512 hash for the file
+		hashFile, err := os.Open(expectedHashFile)
+		require.NoError(t, err, "hash file should be readable")
+
+		checksumsMap := readHashFile(t, hashFile)
+
+		packageBaseName := filepath.Base(packageFile)
+		require.Containsf(t, checksumsMap, packageBaseName, "checksum file should contain an entry for %q", packageBaseName)
+
+		// compare checksum entry with actual package hash
+		checksum := calculateChecksum(t, packageFile, sha512.New())
+
+		assert.Equalf(t, checksum, checksumsMap[packageBaseName], "checksum for file %q does not match", packageFile)
+	})
+}
+
+func calculateChecksum(t *testing.T, file string, hasher hash.Hash) string {
+
+	input, err := os.Open(file)
+	require.NoErrorf(t, err, "error opening input file %q", file)
+
+	defer func(input *os.File) {
+		errClose := input.Close()
+		assert.NoErrorf(t, errClose, "error closing input file %q", file)
+	}(input)
+
+	_, err = io.Copy(hasher, input)
+	require.NoError(t, err, "error reading file to calculate hash")
+
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+// readHashFile return a map of {filename, hash} reading a .sha512 file.
+// If any line has not exactly 2 tokens separated by white spaces, it will fail the test.
+// When it's done reading it will close the reader
+func readHashFile(t *testing.T, reader io.ReadCloser) map[string]string {
+
+	defer func(reader io.ReadCloser) {
+		err := reader.Close()
+		assert.NoError(t, err, "error closing hash file reader")
+	}(reader)
+
+	checksums := map[string]string{}
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+		parts := strings.Fields(line)
+		if len(parts) != 2 {
+			// Fail test because it's malformed.
+			assert.Failf(t, "malformed line %q in hash file", line)
+			continue
+		}
+		filename := strings.TrimLeft(parts[1], "*")
+		checksum := parts[0]
+		checksums[filename] = checksum
+	}
+
+	return checksums
+}

docs/component-specs.md

@@ -87,6 +87,7 @@ The variables that can be accessed by a condition are:
 
 - `runtime.os`: the operating system, either `"windows"`, `"darwin"`, `"linux"`, or `"container"`.
 - `runtime.arch`: the CPU architecture, either `"amd64"` or `"arm64"`.
+- `runtime.native_arch`: the machine CPU architecture, either `"amd64"` or `"arm64"`.
 - `runtime.platform`: a string combining the OS and architecture, e.g. `"windows/amd64"`, `"darwin/arm64"`.
 - `runtime.family`: OS family, e.g. `"debian"`, `"redhat"`, `"windows"`, `"darwin"`
 - `runtime.major`, `runtime.minor`: the operating system version. Note that these are strings not integers, so they must be converted in order to use numeric comparison. For example to check if the OS major version is at most 12, use `number(runtime.major) <= 12`.