Improve and use tweaked/new APIs for creating new BYOC installations …

…(#4119) This uses a new, modified version of BYOC cloud management APIs and statuses. It also allows for true installation of AWS-based clouds. --------- Co-authored-by: Brandon Schurman <[email protected]>
earthly · May 17, 2024 · fbc66cd · fbc66cd
1 parent 5caed35
commit fbc66cd
Show file tree

Hide file tree

Showing 4 changed files with 159 additions and 46 deletions.
diff --git a/cloud/cloud_installation.go b/cloud/cloud_installation.go
@@ -8,32 +8,70 @@ import (
 )
 
 const (
-	CloudStatusConnected = "Connected"
-	CloudStatusActive    = "Active"
-	CloudStatusProblem   = "Problem"
+	CloudStatusGreen   = "Green"
+	CloudStatusYellow  = "Yellow"
+	CloudStatusRed     = "Red"
+	CloudStatusUnknown = "Unknown"
 )
 
 type Installation struct {
 	Name          string
 	Org           string
 	Status        string
+	StatusMessage string
 	NumSatellites int
 	IsDefault     bool
 }
 
-func (c *Client) ConfigureCloud(ctx context.Context, orgID, cloudName string, setDefault bool) (*Installation, error) {
+type CloudConfigurationOpt struct {
+	Name               string
+	SetDefault         bool
+	SshKeyName         string
+	ComputeRoleArn     string
+	AccountId          string
+	AllowedSubnetIds   []string
+	SecurityGroupId    string
+	Region             string
+	InstanceProfileArn string
+}
+
+func (c *Client) ConfigureCloud(ctx context.Context, orgID string, configuration *CloudConfigurationOpt) (*Installation, error) {
 	resp, err := c.compute.ConfigureCloud(c.withAuth(ctx), &pb.ConfigureCloudRequest{
-		OrgId:      orgID,
-		Name:       cloudName,
-		SetDefault: setDefault,
+		OrgId:              orgID,
+		Name:               configuration.Name,
+		SetDefault:         configuration.SetDefault,
+		SshKeyName:         configuration.SshKeyName,
+		ComputeRoleArn:     configuration.ComputeRoleArn,
+		AccountId:          configuration.AccountId,
+		AllowedSubnetIds:   configuration.AllowedSubnetIds,
+		SecurityGroupId:    configuration.SecurityGroupId,
+		Region:             configuration.Region,
+		InstanceProfileArn: configuration.InstanceProfileArn,
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "error from ConfigureCloud API")
 	}
 	return &Installation{
-		Name:   cloudName,
-		Org:    orgID,
-		Status: installationStatus(resp.Status),
+		Name:          configuration.Name,
+		Org:           orgID,
+		Status:        installationStatus(resp.Status),
+		StatusMessage: resp.Message,
+	}, nil
+}
+
+func (c *Client) UseCloud(ctx context.Context, orgID string, configuration *CloudConfigurationOpt) (*Installation, error) {
+	resp, err := c.compute.UseCloud(c.withAuth(ctx), &pb.UseCloudRequest{
+		OrgId: orgID,
+		Name:  configuration.Name,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "error from UseCloud API")
+	}
+	return &Installation{
+		Name:          configuration.Name,
+		Org:           orgID,
+		Status:        installationStatus(resp.Status),
+		StatusMessage: resp.Message,
 	}, nil
 }
 
@@ -50,6 +88,7 @@ func (c *Client) ListClouds(ctx context.Context, orgID string) ([]Installation,
 			Name:          i.CloudName,
 			Org:           orgID,
 			Status:        installationStatus(i.Status),
+			StatusMessage: i.StatusContext,
 			NumSatellites: int(i.NumSatellites),
 			IsDefault:     i.IsDefault,
 		})
@@ -69,14 +108,14 @@ func (c *Client) DeleteCloud(ctx context.Context, orgID, cloudName string) error
 }
 
 func installationStatus(status pb.CloudStatus) string {
-	internalStatus := "UNKNOWN"
 	switch status {
-	case pb.CloudStatus_CLOUD_STATUS_ACCOUNT_ACTIVE:
-		internalStatus = CloudStatusActive
-	case pb.CloudStatus_CLOUD_STATUS_ACCOUNT_CONNECTED:
-		internalStatus = CloudStatusConnected
-	case pb.CloudStatus_CLOUD_STATUS_PROBLEM:
-		internalStatus = CloudStatusProblem
+	case pb.CloudStatus_CLOUD_STATUS_GREEN:
+		return CloudStatusGreen
+	case pb.CloudStatus_CLOUD_STATUS_YELLOW:
+		return CloudStatusYellow
+	case pb.CloudStatus_CLOUD_STATUS_RED:
+		return CloudStatusRed
+	default:
+		return CloudStatusUnknown
 	}
-	return internalStatus
 }
diff --git a/cmd/earthly/subcmd/cloud_installation_cmds.go b/cmd/earthly/subcmd/cloud_installation_cmds.go
@@ -1,10 +1,15 @@
 package subcmd
 
 import (
+	"context"
 	"fmt"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/fatih/color"
 	"os"
 	"text/tabwriter"
 
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
@@ -98,16 +103,22 @@ func (c *CloudInstallation) install(cliCtx *cli.Context) error {
 		return err
 	}
 
+	installation, err := c.getInstallationDataFromCloudFormation(ctx, cloudName)
+	if err != nil {
+		return err
+	}
+
 	c.cli.Console().Printf("Configuring new Cloud Installation: %s. Please wait...", cloudName)
 
-	install, err := cloudClient.ConfigureCloud(ctx, orgID, cloudName, false)
+	install, err := cloudClient.ConfigureCloud(ctx, orgID, installation)
 	if err != nil {
 		return errors.Wrap(err, "failed installing cloud")
 	}
 
-	if install.Status == cloud.CloudStatusProblem {
-		c.cli.Console().Warnf("There is a problem with the cloud installation. Please contact Earthly team for support.")
-		return errors.New("cloud Installation failed validation")
+	if install.Status == cloud.CloudStatusRed || install.Status == cloud.CloudStatusYellow {
+		c.cli.Console().Warnf("There is a problem with the cloud installation.")
+		c.cli.Console().Warnf(install.StatusMessage)
+		return errors.New("cloud installation failed validation")
 	}
 
 	c.cli.Console().Printf("...Done\n")
@@ -145,17 +156,17 @@ func (c *CloudInstallation) use(cliCtx *cli.Context) error {
 
 	c.cli.Console().Printf("Validating Cloud Installation: %s. Please wait...", cloudName)
 
-	install, err := cloudClient.ConfigureCloud(ctx, orgID, cloudName, true)
+	install, err := cloudClient.UseCloud(ctx, orgID, &cloud.CloudConfigurationOpt{
+		Name:       cloudName,
+		SetDefault: true,
+	})
 	if err != nil {
 		return errors.Wrap(err, "could not select cloud")
 	}
 
-	if err != nil {
-		return errors.Wrap(err, "failed selecting cloud")
-	}
-
-	if install.Status == cloud.CloudStatusProblem {
-		c.cli.Console().Warnf("There is a problem with the cloud installation. Please contact Earthly team for support.")
+	if install.Status == cloud.CloudStatusRed || install.Status == cloud.CloudStatusYellow {
+		c.cli.Console().Warnf("There is a problem with the cloud installation.")
+		c.cli.Console().Warnf(install.StatusMessage)
 		return errors.New("cloud Installation failed validation")
 	}
 
@@ -231,18 +242,78 @@ func (c *CloudInstallation) printTable(installations []cloud.Installation) {
 		if i.IsDefault {
 			selected = "*"
 		}
-		var description string
+		var coloredStatus string
 		switch i.Status {
-		case cloud.CloudStatusActive:
-			description = "Ready to use"
-		case cloud.CloudStatusConnected:
-			description = "Reachable, but not yet validated"
-		case cloud.CloudStatusProblem:
-			description = "Please contact Earthly for support"
+		case cloud.CloudStatusGreen:
+			coloredStatus = color.GreenString(i.Status)
+		case cloud.CloudStatusYellow:
+			coloredStatus = color.YellowString(i.Status)
+		case cloud.CloudStatusRed:
+			coloredStatus = color.RedString(i.Status)
+		default:
+			coloredStatus = color.HiRedString(i.Status)
+		}
+		suffix := ""
+		if i.StatusMessage != "" {
+			suffix = fmt.Sprintf(": %s", i.StatusMessage)
 		}
-		fmt.Fprintf(t, "%s\t%s\t%d\t%s: %s\t\n", selected, i.Name, i.NumSatellites, i.Status, description)
+		fullStatus := fmt.Sprintf("%s%s", coloredStatus, suffix)
+		fmt.Fprintf(t, "%s\t%s\t%d\t%s\t\n", selected, i.Name, i.NumSatellites, fullStatus)
 	}
 	if err := t.Flush(); err != nil {
 		fmt.Printf("failed to print satellites: %s", err.Error())
 	}
 }
+
+func (c *CloudInstallation) getInstallationDataFromCloudFormation(ctx context.Context, stackName string) (*cloud.CloudConfigurationOpt, error) {
+	awsConfig, err := awsconfig.LoadDefaultConfig(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not load aws config")
+	}
+
+	client := cloudformation.NewFromConfig(awsConfig)
+
+	describeStacksOutput, err := client.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{
+		StackName: aws.String(stackName),
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("could not describe stack %s", stackName))
+	}
+
+	if len(describeStacksOutput.Stacks) != 1 {
+		return nil, fmt.Errorf("unexpected number of stacks(%v) found with name %q", len(describeStacksOutput.Stacks), stackName)
+	}
+
+	stack := describeStacksOutput.Stacks[0]
+	params := &cloud.CloudConfigurationOpt{}
+
+	for _, output := range stack.Outputs {
+		if output.OutputKey == nil {
+			return nil, fmt.Errorf("specified stack %s has nil output key", stackName)
+		}
+		if output.OutputValue == nil {
+			return nil, fmt.Errorf("specified stack %s has nil value for key %s", stackName, *output.OutputKey)
+		}
+
+		switch *output.OutputKey {
+		case "InstallationName":
+			params.Name = *output.OutputValue
+		case "SshKeyName":
+			params.SshKeyName = *output.OutputValue
+		case "ComputeRoleArn":
+			params.ComputeRoleArn = *output.OutputValue
+		case "AccountId":
+			params.AccountId = *output.OutputValue
+		case "AllowedSubnetIds":
+			params.AllowedSubnetIds = []string{*output.OutputValue}
+		case "SecurityGroupId":
+			params.SecurityGroupId = *output.OutputValue
+		case "Region":
+			params.Region = *output.OutputValue
+		case "InstanceProfileArn":
+			params.InstanceProfileArn = *output.OutputValue
+		}
+	}
+
+	return params, nil
+}
diff --git a/go.mod b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/alexcb/binarystream v0.0.0-20231130184431-f2f7a7543c6d
 	github.com/aws/aws-sdk-go-v2 v1.26.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.16
+	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0
 	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230227212328-9f4511cd144a
 	github.com/containerd/containerd v1.7.8
 	github.com/containerd/go-runc v1.1.0
@@ -18,7 +19,7 @@ require (
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-units v0.5.0
 	github.com/dustin/go-humanize v1.0.1
-	github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f
+	github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150
 	github.com/earthly/earthly/ast v0.0.0-00010101000000-000000000000
 	github.com/earthly/earthly/util/deltautil v0.0.0-20240507235053-335389ed3e2a
 	github.com/elastic/go-sysinfo v1.9.0
@@ -72,8 +73,8 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230219212500-1f9a474cc2dc // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.16 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.1 // indirect

diff --git a/go.sum b/go.sum
@@ -85,16 +85,18 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24/go.mod h1:neYVaeKr5eT7Bzw
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30/go.mod h1:LUBAO3zNXQjoONBKn/kR1y0Q4cj/D02Ts0uHYjcCQLM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24/go.mod h1:gAuCezX/gob6BSMbItsSlMb6WZGV7K2+fWOvk8xBSto=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 h1:hf+Vhp5WtTdcSdE+yEcUz8L73sAzN0R+0jQv+Z51/mI=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31/go.mod h1:5zUjguZfG5qjhG9/wqmuyHRyUftl2B5Cp6NNxNC6kRA=
+github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 h1:Ap5tOJfeAH1hO2UQc3X3uMlwP7uryFeZXMvZCXIlLSE=
+github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0/go.mod h1:/v2KYdCW4BaHKayenaWEXOOdxItIwEA3oU0XzuQY3F0=
 github.com/aws/aws-sdk-go-v2/service/ecr v1.18.2/go.mod h1:53xgmccefO+AwKsxVKuTh2vo/IDOkeMWNpmDuhZH1Vc=
 github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 h1:+sbyLjtAq0Xg9ZOQ2mBibklsGUyX6I2OfRTDsha9uU4=
 github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3/go.mod h1:/m9MiYl5Ds0cZqy/bbeSUWxKLwTarGugjXxSgiXNQFc=
@@ -192,8 +194,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec h1:vf6x0fPOWKakjH3n2N1O9Tg5j1HDIJsC3Kkgmuko2U0=
 github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec/go.mod h1:1/yAC8A0Tu94Bdmv07gaG1pFBp+CetVwO7oB3qvZXUc=
-github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f h1:8CXT0MQ7dQrtm/IwVIexffosImh4ht0WUiWQAt0UoeQ=
-github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI=
+github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150 h1:hHdgFB5BE+OAFee+CpassZP2GOkdRt5YyhsFmJJVtc8=
+github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI=
 github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65 h1:6oyWHoxHXwcTt4EqmMw6361scIV87uEAB1N42+VpIwk=
 github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI=
 github.com/elastic/go-sysinfo v1.9.0 h1:usICqY/Nw4Mpn9f4LdtpFrKxXroJDe81GaxxUlCckIo=