Merge branch 'main' into dependabot/go_modules/github.com/elastic/go-…

…elasticsearch/v8-8.11.0

NOTICE.txt

@@ -1377,11 +1377,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.8.0
+Version: v0.8.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE.txt:
 
                                  Apache License
                            Version 2.0, January 2004

changelog/fragments/1700520158-dynamic-timeout-for-diags.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Use shorter timeouts for diagnostic requests unless CPU diagnostics are requested
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: Use different timeout values for diagnostics requests, depending on if CPU diagnostics are requested
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: diagnostics
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/3197

changelog/fragments/1700678892-Fixes-the-Elastic-Agent-ignoring-agent.download.proxy_url-on-policy-update.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fixes the Elastic Agent ignoring agent.download.proxy_url on policy update
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: Upgrader
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/3560

dev-tools/packaging/packages.yml

@@ -421,12 +421,12 @@ shared:
 
   - &docker_ubi_spec
     extra_vars:
-      image_name: '{{.BeatName}}-ubi9'
+      image_name: '{{.BeatName}}-ubi'
       from: 'docker.elastic.co/ubi9/ubi-minimal'
 
   - &docker_arm_ubi_spec
     extra_vars:
-      image_name: '{{.BeatName}}-ubi9'
+      image_name: '{{.BeatName}}-ubi'
       from: 'registry.access.redhat.com/ubi9/ubi-minimal:9.3'
 
   - &elastic_docker_spec

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/elastic/elastic-agent-autodiscover v0.6.5
 	github.com/elastic/elastic-agent-client/v7 v7.5.0
 	github.com/elastic/elastic-agent-libs v0.7.2
-	github.com/elastic/elastic-agent-system-metrics v0.8.0
+	github.com/elastic/elastic-agent-system-metrics v0.8.1
 	github.com/elastic/elastic-transport-go/v8 v8.3.0
 	github.com/elastic/go-elasticsearch/v8 v8.11.0
 	github.com/elastic/go-licenser v0.4.1

go.sum

@@ -783,8 +783,8 @@ github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe
 github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0=
 github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o=
 github.com/elastic/elastic-agent-libs v0.7.2/go.mod h1:pVBEElQJUO9mr4WStWNXuQGsJn54lcjAoYAHmsvBLBc=
-github.com/elastic/elastic-agent-system-metrics v0.8.0 h1:EsWbtd83JvnaqnL57bKS1E6GhOdemTRbxdFDcenR8zQ=
-github.com/elastic/elastic-agent-system-metrics v0.8.0/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
+github.com/elastic/elastic-agent-system-metrics v0.8.1 h1:eg6actuLeGJlIJFotHRdlAsz/3WhX2G8E0qI301IKBA=
+github.com/elastic/elastic-agent-system-metrics v0.8.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0/go.mod h1:uf9N86y+UACGybdEhZLpwZ93XHWVhsYZAA4c2T2v6YM=
 github.com/elastic/elastic-package v0.77.0/go.mod h1:Xeqx0OOVnKBfFoSHsHmKI74RxgRGiDhU6yXEu8BkJJM=
 github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo=

internal/pkg/agent/application/upgrade/step_download.go

@@ -257,7 +257,8 @@ func (u *Upgrader) downloadWithRetries(
 	}
 
 	opFailureNotificationFn := func(err error, retryAfter time.Duration) {
-		u.log.Warnf("%s; retrying (will be retry %d) in %s.", err.Error(), attempt, retryAfter)
+		u.log.Warnf("download attempt %d failed: %s; retrying in %s.",
+			attempt, err.Error(), retryAfter)
 	}
 
 	if err := backoff.RetryNotify(opFn, boCtx, opFailureNotificationFn); err != nil {

internal/pkg/agent/application/upgrade/step_download_test.go

@@ -139,7 +139,7 @@ func TestDownloadWithRetries(t *testing.T) {
 		logs := obs.TakeAll()
 		require.Len(t, logs, 3)
 		require.Equal(t, "download attempt 1", logs[0].Message)
-		require.Contains(t, logs[1].Message, "unable to create fetcher: failed to construct downloader; retrying (will be retry 1)")
+		require.Contains(t, logs[1].Message, "unable to create fetcher: failed to construct downloader")
 		require.Equal(t, "download attempt 2", logs[2].Message)
 	})
 
@@ -178,7 +178,7 @@ func TestDownloadWithRetries(t *testing.T) {
 		logs := obs.TakeAll()
 		require.Len(t, logs, 3)
 		require.Equal(t, "download attempt 1", logs[0].Message)
-		require.Contains(t, logs[1].Message, "unable to download package: download failed; retrying (will be retry 1)")
+		require.Contains(t, logs[1].Message, "unable to download package: download failed; retrying")
 		require.Equal(t, "download attempt 2", logs[2].Message)
 	})
 
@@ -207,7 +207,7 @@ func TestDownloadWithRetries(t *testing.T) {
 		require.GreaterOrEqual(t, len(logs), minNmExpectedAttempts*2)
 		for i := 0; i < minNmExpectedAttempts; i++ {
 			require.Equal(t, fmt.Sprintf("download attempt %d", i+1), logs[(2*i)].Message)
-			require.Contains(t, logs[(2*i+1)].Message, fmt.Sprintf("unable to download package: download failed; retrying (will be retry %d)", i+1))
+			require.Contains(t, logs[(2*i+1)].Message, "unable to download package: download failed; retrying")
 		}
 	})
 }

internal/pkg/agent/application/upgrade/upgrade.go

@@ -20,6 +20,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/reexec"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/errors"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
 	"github.com/elastic/elastic-agent/internal/pkg/config"
@@ -89,35 +90,39 @@ func (u *Upgrader) SetClient(c fleetclient.Sender) {
 
 // Reload reloads the artifact configuration for the upgrader.
 func (u *Upgrader) Reload(rawConfig *config.Config) error {
-	type reloadConfig struct {
-		// SourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/
-		SourceURI string `json:"agent.download.sourceURI" config:"agent.download.sourceURI"`
+	cfg, err := configuration.NewFromConfig(rawConfig)
+	if err != nil {
+		return fmt.Errorf("invalid config: %w", err)
+	}
 
-		// FleetSourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/ coming from fleet which uses
-		// different naming.
+	// the source URI coming from fleet which uses a different naming.
+	type fleetCfg struct {
+		// FleetSourceURI: source of the artifacts, e.g https://artifacts.elastic.co/downloads/
 		FleetSourceURI string `json:"agent.download.source_uri" config:"agent.download.source_uri"`
 	}
-	cfg := &reloadConfig{}
-	if err := rawConfig.Unpack(&cfg); err != nil {
+	fleetSourceURI := &fleetCfg{}
+	if err := rawConfig.Unpack(&fleetSourceURI); err != nil {
 		return errors.New(err, "failed to unpack config during reload")
 	}
 
-	var newSourceURI string
-	if cfg.FleetSourceURI != "" {
-		// fleet configuration takes precedence
-		newSourceURI = cfg.FleetSourceURI
-	} else if cfg.SourceURI != "" {
-		newSourceURI = cfg.SourceURI
+	// fleet configuration takes precedence
+	if fleetSourceURI.FleetSourceURI != "" {
+		cfg.Settings.DownloadConfig.SourceURI = fleetSourceURI.FleetSourceURI
 	}
 
-	if newSourceURI != "" {
-		u.log.Infof("Source URI changed from %q to %q", u.settings.SourceURI, newSourceURI)
-		u.settings.SourceURI = newSourceURI
+	if cfg.Settings.DownloadConfig.SourceURI != "" {
+		u.log.Infof("Source URI changed from %q to %q",
+			u.settings.SourceURI,
+			cfg.Settings.DownloadConfig.SourceURI)
 	} else {
 		// source uri unset, reset to default
-		u.log.Infof("Source URI reset from %q to %q", u.settings.SourceURI, artifact.DefaultSourceURI)
-		u.settings.SourceURI = artifact.DefaultSourceURI
+		u.log.Infof("Source URI reset from %q to %q",
+			u.settings.SourceURI,
+			artifact.DefaultSourceURI)
+		cfg.Settings.DownloadConfig.SourceURI = artifact.DefaultSourceURI
 	}
+
+	u.settings = cfg.Settings.DownloadConfig
 	return nil
 }
 

internal/pkg/agent/application/upgrade/upgrade_test.go

@@ -14,15 +14,17 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/elastic/elastic-agent/pkg/control/v2/client"
-	"github.com/elastic/elastic-agent/pkg/control/v2/client/mocks"
-	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
-
 	"github.com/gofrs/flock"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/errors"
+	"github.com/elastic/elastic-agent/internal/pkg/config"
 	"github.com/elastic/elastic-agent/internal/pkg/release"
+	"github.com/elastic/elastic-agent/pkg/control/v2/client"
+	"github.com/elastic/elastic-agent/pkg/control/v2/client/mocks"
+	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
 )
 
@@ -209,3 +211,67 @@ func TestIsInProgress(t *testing.T) {
 		})
 	}
 }
+
+func TestUpgraderReload(t *testing.T) {
+	defaultCfg := artifact.DefaultConfig()
+	tcs := []struct {
+		name      string
+		sourceURL string
+		proxyURL  string
+		cfg       string
+	}{
+		{
+			name:      "proxy_url is applied",
+			sourceURL: defaultCfg.SourceURI,
+			proxyURL:  "http://someBrokenURL/",
+			cfg: `
+agent.download:
+  proxy_url: http://someBrokenURL/
+`,
+		},
+		{
+			name:      "source_uri has precedence over sourceURI",
+			sourceURL: "https://this.sourceURI.co/downloads/beats/",
+			cfg: `
+agent.download:
+  source_uri: "https://this.sourceURI.co/downloads/beats/"
+  sourceURI: "https://NOT.sourceURI.co/downloads/beats/"
+`}, {
+			name:      "only sourceURI",
+			sourceURL: "https://this.sourceURI.co/downloads/beats/",
+			cfg: `
+agent.download:
+  sourceURI: "https://this.sourceURI.co/downloads/beats/"
+`}, {
+			name:      "only source_uri",
+			sourceURL: "https://this.sourceURI.co/downloads/beats/",
+			cfg: `
+agent.download:
+  source_uri: "https://this.sourceURI.co/downloads/beats/"
+`},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			log, _ := logger.NewTesting("")
+
+			u := Upgrader{
+				log:      log,
+				settings: artifact.DefaultConfig(),
+			}
+
+			cfg, err := config.NewConfigFrom(tc.cfg)
+			require.NoError(t, err, "failed to create new config")
+
+			err = u.Reload(cfg)
+			require.NoError(t, err, "error reloading config")
+
+			assert.Equal(t, tc.sourceURL, u.settings.SourceURI)
+			if tc.proxyURL != "" {
+				require.NotNilf(t, u.settings.Proxy.URL,
+					"ProxyURI should not be nil, want %s", tc.proxyURL)
+				assert.Equal(t, tc.proxyURL, u.settings.Proxy.URL.String())
+			}
+		})
+	}
+}

internal/pkg/composable/providers/host/host_test.go

@@ -30,7 +30,7 @@ func TestContextProvider(t *testing.T) {
 	require.NoError(t, err)
 
 	const checkInterval = 50 * time.Millisecond
-	const testTimeout = 500 * time.Millisecond
+	const testTimeout = 1 * time.Second
 	c, err := config.NewConfigFrom(map[string]interface{}{
 		"check_interval": checkInterval,
 	})

pkg/component/runtime/manager.go

@@ -43,8 +43,11 @@ const (
 	// maxCheckinMisses is the maximum number of check-in misses a component can miss before it is killed
 	// and restarted.
 	maxCheckinMisses = 3
-	// diagnosticTimeout is the maximum amount of time to wait for a diagnostic response from a unit.
-	diagnosticTimeout = time.Minute
+	// diagnosticTimeoutCPU is the maximum amount of time to wait for a diagnostic response from a unit while collecting CPU profiles
+	diagnosticTimeoutCPU = time.Minute
+
+	// diagnosticTimeout is the maximum amount of time to wait for a diagnostic response from a unit
+	diagnosticTimeout = time.Second * 20
 
 	// stopCheckRetryPeriod is a idle time between checks for component stopped state
 	stopCheckRetryPeriod = 200 * time.Millisecond
@@ -944,7 +947,15 @@ func (m *Manager) getListenAddr() string {
 // performDiagAction creates a diagnostic ActionRequest and executes it against the runtime that's mapped to the specified component.
 // if the specified actionLevel is ActionRequest_COMPONENT, the unit field is ignored.
 func (m *Manager) performDiagAction(ctx context.Context, comp component.Component, unit component.Unit, actionLevel proto.ActionRequest_Level, params client.DiagnosticParams) ([]*proto.ActionDiagnosticUnitResult, error) {
-	ctx, cancel := context.WithTimeout(ctx, diagnosticTimeout)
+	// if we're gathering CPU diagnostics, request a longer timeout; CPU diag collection requires the diagnostic hook to sit and gather a CPU profile.
+	finalDiagnosticTime := diagnosticTimeout
+	for _, tag := range params.AdditionalMetrics {
+		if tag == "CPU" {
+			finalDiagnosticTime = diagnosticTimeoutCPU
+			break
+		}
+	}
+	ctx, cancel := context.WithTimeout(ctx, finalDiagnosticTime)
 	defer cancel()
 
 	id, err := uuid.NewV4()
@@ -966,7 +977,7 @@ func (m *Manager) performDiagAction(ctx context.Context, comp component.Componen
 	}
 
 	if len(params.AdditionalMetrics) > 0 {
-		m.logger.Debugf("Performing diagnostic action with params: %v", params.AdditionalMetrics)
+		m.logger.Debugf("Performing diagnostic action with params: %v; will wait %s", params.AdditionalMetrics, finalDiagnosticTime)
 	}
 	marshalParams, err := json.Marshal(params)
 	if err != nil {
@@ -989,7 +1000,7 @@ func (m *Manager) performDiagAction(ctx context.Context, comp component.Componen
 	// the only way this can return an error is a context Done(), be sure to make that explicit.
 	if err != nil {
 		if errors.Is(context.DeadlineExceeded, err) {
-			return nil, fmt.Errorf("diagnostic action timed out, deadline is %s: %w", diagnosticTimeout, err)
+			return nil, fmt.Errorf("diagnostic action timed out, deadline is %s: %w", finalDiagnosticTime, err)
 		}
 		return nil, fmt.Errorf("error running performAction: %w", err)
 	}