Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup manifest packaging #6540

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Cleanup manifest packaging #6540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
481aa67
Fixed packaging with manifest when manifest has a different version t…
pchila Jan 17, 2025
e04bab5
Fixed packaging non-snapshot agent version using snapshot manifest
pchila Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions dev-tools/mage/settings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ func BeatQualifiedVersion() (string, error) {
return "", err
}
// version qualifier can intentionally be set to "" to override build time var
if !versionQualified || versionQualifier == "" {
if PackagingFromManifest || !versionQualified || versionQualifier == "" {
return version, nil
}
return version + "-" + versionQualifier, nil
Expand Down Expand Up @@ -648,11 +648,10 @@ func (s *BuildVariableSources) GetDocBranch() (string, error) {

func parseBeatVersion(data []byte) (string, error) {
matches := beatVersionRegex.FindSubmatch(data)
if len(matches) == 2 {
return string(matches[1]), nil
if len(matches) != 2 {
return "", errors.New("failed to parse beat version file")
}

return "", errors.New("failed to parse beat version file")
return string(matches[1]), nil
}

func parseGoVersion(data []byte) (string, error) {
Expand Down
8 changes: 4 additions & 4 deletions dev-tools/packaging/packages.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,16 +281,16 @@ shared:
'data/cloud_downloads/metricbeat.sh':
source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/metricbeat.sh'
mode: 0755
'data/cloud_downloads/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
'data/cloud_downloads/agentbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0755

# service build is based on previous cloud variant
- &agent_docker_service_spec
docker_variant: 'service'
files:
'data/service/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip':
source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip'
'data/service/connectors-{{ beat_version }}.zip':
source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}.zip'
mode: 0755
'data/{{.BeatName}}-{{ commit_short }}/components/connectors':
source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/connectors.sh'
Expand Down
64 changes: 37 additions & 27 deletions magefile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,22 +504,22 @@ func Package(ctx context.Context) error {

var err error
var manifestResponse *manifest.Build
var dependenciesVersion string
if devtools.PackagingFromManifest {
manifestResponse, _, err = downloadManifestAndSetVersion(ctx, devtools.ManifestURL)
var parsedDependenciesVersion *version.ParsedSemVer
manifestResponse, parsedDependenciesVersion, err = downloadManifestAndSetVersion(ctx, devtools.ManifestURL)
if err != nil {
return fmt.Errorf("failed downloading manifest: %w", err)
}
}

var dependenciesVersion string
if beatVersion, found := os.LookupEnv("BEAT_VERSION"); !found {
dependenciesVersion = bversion.GetDefaultVersion()
dependenciesVersion = parsedDependenciesVersion.Original()
} else {
dependenciesVersion = beatVersion
if beatVersion, found := os.LookupEnv("BEAT_VERSION"); !found {
dependenciesVersion = bversion.GetDefaultVersion() + devtools.SnapshotSuffix()
} else {
dependenciesVersion = beatVersion
}
}

packageAgent(ctx, platforms, dependenciesVersion, manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(CrossBuild), devtools.SelectedPackageTypes)
return nil
return packageAgent(ctx, platforms, dependenciesVersion, manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(CrossBuild), devtools.SelectedPackageTypes)
}

// DownloadManifest downloads the provided manifest file into the predefined folder and downloads all components in the manifest.
Expand Down Expand Up @@ -984,7 +984,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s
}

// download/copy all the necessary dependencies for packaging elastic-agent
archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, platformPackageSuffixes, packageTypes)
archivePath, dropPath := collectPackageDependencies(ctx, platforms, dependenciesVersion, platformPackageSuffixes, packageTypes)

// cleanup after build
defer os.RemoveAll(archivePath)
Expand Down Expand Up @@ -1018,7 +1018,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s
// NOTE: after the build is done the caller must:
// - delete archivePath and dropPath contents
// - unset AGENT_DROP_PATH environment variable
func collectPackageDependencies(platforms []string, packageVersion string, platformPackageSuffixes []string, packageTypes []mage.PackageType) (archivePath string, dropPath string) {
func collectPackageDependencies(ctx context.Context, platforms []string, packageVersion string, platformPackageSuffixes []string, packageTypes []mage.PackageType) (archivePath string, dropPath string) {
dropPath, found := os.LookupEnv(agentDropPath)

// try not to shadow too many variables
Expand All @@ -1038,38 +1038,48 @@ func collectPackageDependencies(platforms []string, packageVersion string, platf
}
archivePath = movePackagesToArchive(dropPath, platformPackageSuffixes, packageVersion)

if hasSnapshotEnv() {
packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion)
}
//if hasSnapshotEnv() {
// packageVersion = fmt.Sprintf("%s-SNAPSHOT", packageVersion)
//}

os.Setenv(agentDropPath, dropPath)

if devtools.ExternalBuild == true {

if devtools.ExternalBuild {
log.Printf("--- Detected EXTERNAL=%v", devtools.ExternalBuild)
// Only log fatal logs for logs produced. This is the global logger
// used by github.com/elastic/elastic-agent/dev-tools/mage/downloads which can only be configured globally like this.
//
// Using FatalLevel avoids filling the build log with scary looking errors when we attempt to
// download artifacts on unsupported platforms and choose to ignore the errors.
//
// Change this to InfoLevel to see exactly what the downloader is doing.
downloads.LogLevel.Set(downloads.FatalLevel)
downloads.LogLevel.Set(downloads.TraceLevel)

errGroup, ctx := errgroup.WithContext(context.Background())
errGroup, ctx := errgroup.WithContext(ctx)
completedDownloads := &atomic.Int32{}
for _, spec := range manifest.ExpectedBinaries {
if mg.Verbose() {
log.Printf("--- Checking spec %s...", spec.ProjectName)
}
for _, platform := range platforms {
if !spec.SupportsPlatform(platform) {
fmt.Printf("--- Binary %s does not support %s, download skipped\n", spec.BinaryName, platform)
log.Printf("--- Binary %s does not support %s, download skipped", spec.BinaryName, platform)
continue
}
if mg.Verbose() {
log.Printf("--- Iterating over pkgTypes %s...", packageTypes)
}
for _, pkgType := range packageTypes {
if !spec.SupportsPackageType(pkgcommon.PackageType(pkgType)) {
log.Printf("--- Package type %s is not supported for spec %s", pkgType, spec.ProjectName)
continue
}
targetPath := filepath.Join(archivePath, manifest.PlatformPackages[platform])
os.MkdirAll(targetPath, 0o755)
packageName := spec.GetPackageName(packageVersion, platform)
if mg.Verbose() {
log.Printf("--- Starting download for package %s into %s...", packageName, targetPath)
}
errGroup.Go(downloadBinary(ctx, spec.ProjectName, packageName, spec.BinaryName, platform, packageVersion, targetPath, completedDownloads))
}
}
Expand Down Expand Up @@ -1319,13 +1329,13 @@ func downloadManifestAndSetVersion(ctx context.Context, url string) (*manifest.B
return nil, nil, fmt.Errorf("parsing manifest version %s: %w", resp.Version, err)
}

// When getting the packageVersion from snapshot we should also update the env of SNAPSHOT=true which is
// something that we use as an implicit parameter to various functions
if parsedVersion.IsSnapshot() {
os.Setenv(snapshotEnv, "true")
mage.Snapshot = true
}
os.Setenv("BEAT_VERSION", parsedVersion.CoreVersion())
//// When getting the packageVersion from snapshot we should also update the env of SNAPSHOT=true which is
//// something that we use as an implicit parameter to various functions
//if parsedVersion.IsSnapshot() {
// os.Setenv(snapshotEnv, "true")
// mage.Snapshot = true
//}
os.Setenv("BEAT_VERSION", resp.Version)

return &resp, parsedVersion, nil
}
Expand Down
Loading