From fbc66cd1429fdf741b4133df6b606f99d0fd953c Mon Sep 17 00:00:00 2001 From: Corey Larson Date: Thu, 16 May 2024 20:29:59 -0600 Subject: [PATCH] Improve and use tweaked/new APIs for creating new BYOC installations (#4119) This uses a new, modified version of BYOC cloud management APIs and statuses. It also allows for true installation of AWS-based clouds. --------- Co-authored-by: Brandon Schurman --- cloud/cloud_installation.go | 75 +++++++++--- cmd/earthly/subcmd/cloud_installation_cmds.go | 109 +++++++++++++++--- go.mod | 7 +- go.sum | 14 ++- 4 files changed, 159 insertions(+), 46 deletions(-) diff --git a/cloud/cloud_installation.go b/cloud/cloud_installation.go index b1d0a6d6..1ecfe9d6 100644 --- a/cloud/cloud_installation.go +++ b/cloud/cloud_installation.go @@ -8,32 +8,70 @@ import ( ) const ( - CloudStatusConnected = "Connected" - CloudStatusActive = "Active" - CloudStatusProblem = "Problem" + CloudStatusGreen = "Green" + CloudStatusYellow = "Yellow" + CloudStatusRed = "Red" + CloudStatusUnknown = "Unknown" ) type Installation struct { Name string Org string Status string + StatusMessage string NumSatellites int IsDefault bool } -func (c *Client) ConfigureCloud(ctx context.Context, orgID, cloudName string, setDefault bool) (*Installation, error) { +type CloudConfigurationOpt struct { + Name string + SetDefault bool + SshKeyName string + ComputeRoleArn string + AccountId string + AllowedSubnetIds []string + SecurityGroupId string + Region string + InstanceProfileArn string +} + +func (c *Client) ConfigureCloud(ctx context.Context, orgID string, configuration *CloudConfigurationOpt) (*Installation, error) { resp, err := c.compute.ConfigureCloud(c.withAuth(ctx), &pb.ConfigureCloudRequest{ - OrgId: orgID, - Name: cloudName, - SetDefault: setDefault, + OrgId: orgID, + Name: configuration.Name, + SetDefault: configuration.SetDefault, + SshKeyName: configuration.SshKeyName, + ComputeRoleArn: configuration.ComputeRoleArn, + AccountId: configuration.AccountId, + AllowedSubnetIds: configuration.AllowedSubnetIds, + SecurityGroupId: configuration.SecurityGroupId, + Region: configuration.Region, + InstanceProfileArn: configuration.InstanceProfileArn, }) if err != nil { return nil, errors.Wrap(err, "error from ConfigureCloud API") } return &Installation{ - Name: cloudName, - Org: orgID, - Status: installationStatus(resp.Status), + Name: configuration.Name, + Org: orgID, + Status: installationStatus(resp.Status), + StatusMessage: resp.Message, + }, nil +} + +func (c *Client) UseCloud(ctx context.Context, orgID string, configuration *CloudConfigurationOpt) (*Installation, error) { + resp, err := c.compute.UseCloud(c.withAuth(ctx), &pb.UseCloudRequest{ + OrgId: orgID, + Name: configuration.Name, + }) + if err != nil { + return nil, errors.Wrap(err, "error from UseCloud API") + } + return &Installation{ + Name: configuration.Name, + Org: orgID, + Status: installationStatus(resp.Status), + StatusMessage: resp.Message, }, nil } @@ -50,6 +88,7 @@ func (c *Client) ListClouds(ctx context.Context, orgID string) ([]Installation, Name: i.CloudName, Org: orgID, Status: installationStatus(i.Status), + StatusMessage: i.StatusContext, NumSatellites: int(i.NumSatellites), IsDefault: i.IsDefault, }) @@ -69,14 +108,14 @@ func (c *Client) DeleteCloud(ctx context.Context, orgID, cloudName string) error } func installationStatus(status pb.CloudStatus) string { - internalStatus := "UNKNOWN" switch status { - case pb.CloudStatus_CLOUD_STATUS_ACCOUNT_ACTIVE: - internalStatus = CloudStatusActive - case pb.CloudStatus_CLOUD_STATUS_ACCOUNT_CONNECTED: - internalStatus = CloudStatusConnected - case pb.CloudStatus_CLOUD_STATUS_PROBLEM: - internalStatus = CloudStatusProblem + case pb.CloudStatus_CLOUD_STATUS_GREEN: + return CloudStatusGreen + case pb.CloudStatus_CLOUD_STATUS_YELLOW: + return CloudStatusYellow + case pb.CloudStatus_CLOUD_STATUS_RED: + return CloudStatusRed + default: + return CloudStatusUnknown } - return internalStatus } diff --git a/cmd/earthly/subcmd/cloud_installation_cmds.go b/cmd/earthly/subcmd/cloud_installation_cmds.go index 90e6643d..ef63df51 100644 --- a/cmd/earthly/subcmd/cloud_installation_cmds.go +++ b/cmd/earthly/subcmd/cloud_installation_cmds.go @@ -1,10 +1,15 @@ package subcmd import ( + "context" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/fatih/color" "os" "text/tabwriter" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/pkg/errors" "github.com/urfave/cli/v2" @@ -98,16 +103,22 @@ func (c *CloudInstallation) install(cliCtx *cli.Context) error { return err } + installation, err := c.getInstallationDataFromCloudFormation(ctx, cloudName) + if err != nil { + return err + } + c.cli.Console().Printf("Configuring new Cloud Installation: %s. Please wait...", cloudName) - install, err := cloudClient.ConfigureCloud(ctx, orgID, cloudName, false) + install, err := cloudClient.ConfigureCloud(ctx, orgID, installation) if err != nil { return errors.Wrap(err, "failed installing cloud") } - if install.Status == cloud.CloudStatusProblem { - c.cli.Console().Warnf("There is a problem with the cloud installation. Please contact Earthly team for support.") - return errors.New("cloud Installation failed validation") + if install.Status == cloud.CloudStatusRed || install.Status == cloud.CloudStatusYellow { + c.cli.Console().Warnf("There is a problem with the cloud installation.") + c.cli.Console().Warnf(install.StatusMessage) + return errors.New("cloud installation failed validation") } c.cli.Console().Printf("...Done\n") @@ -145,17 +156,17 @@ func (c *CloudInstallation) use(cliCtx *cli.Context) error { c.cli.Console().Printf("Validating Cloud Installation: %s. Please wait...", cloudName) - install, err := cloudClient.ConfigureCloud(ctx, orgID, cloudName, true) + install, err := cloudClient.UseCloud(ctx, orgID, &cloud.CloudConfigurationOpt{ + Name: cloudName, + SetDefault: true, + }) if err != nil { return errors.Wrap(err, "could not select cloud") } - if err != nil { - return errors.Wrap(err, "failed selecting cloud") - } - - if install.Status == cloud.CloudStatusProblem { - c.cli.Console().Warnf("There is a problem with the cloud installation. Please contact Earthly team for support.") + if install.Status == cloud.CloudStatusRed || install.Status == cloud.CloudStatusYellow { + c.cli.Console().Warnf("There is a problem with the cloud installation.") + c.cli.Console().Warnf(install.StatusMessage) return errors.New("cloud Installation failed validation") } @@ -231,18 +242,78 @@ func (c *CloudInstallation) printTable(installations []cloud.Installation) { if i.IsDefault { selected = "*" } - var description string + var coloredStatus string switch i.Status { - case cloud.CloudStatusActive: - description = "Ready to use" - case cloud.CloudStatusConnected: - description = "Reachable, but not yet validated" - case cloud.CloudStatusProblem: - description = "Please contact Earthly for support" + case cloud.CloudStatusGreen: + coloredStatus = color.GreenString(i.Status) + case cloud.CloudStatusYellow: + coloredStatus = color.YellowString(i.Status) + case cloud.CloudStatusRed: + coloredStatus = color.RedString(i.Status) + default: + coloredStatus = color.HiRedString(i.Status) + } + suffix := "" + if i.StatusMessage != "" { + suffix = fmt.Sprintf(": %s", i.StatusMessage) } - fmt.Fprintf(t, "%s\t%s\t%d\t%s: %s\t\n", selected, i.Name, i.NumSatellites, i.Status, description) + fullStatus := fmt.Sprintf("%s%s", coloredStatus, suffix) + fmt.Fprintf(t, "%s\t%s\t%d\t%s\t\n", selected, i.Name, i.NumSatellites, fullStatus) } if err := t.Flush(); err != nil { fmt.Printf("failed to print satellites: %s", err.Error()) } } + +func (c *CloudInstallation) getInstallationDataFromCloudFormation(ctx context.Context, stackName string) (*cloud.CloudConfigurationOpt, error) { + awsConfig, err := awsconfig.LoadDefaultConfig(ctx) + if err != nil { + return nil, errors.Wrap(err, "could not load aws config") + } + + client := cloudformation.NewFromConfig(awsConfig) + + describeStacksOutput, err := client.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{ + StackName: aws.String(stackName), + }) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("could not describe stack %s", stackName)) + } + + if len(describeStacksOutput.Stacks) != 1 { + return nil, fmt.Errorf("unexpected number of stacks(%v) found with name %q", len(describeStacksOutput.Stacks), stackName) + } + + stack := describeStacksOutput.Stacks[0] + params := &cloud.CloudConfigurationOpt{} + + for _, output := range stack.Outputs { + if output.OutputKey == nil { + return nil, fmt.Errorf("specified stack %s has nil output key", stackName) + } + if output.OutputValue == nil { + return nil, fmt.Errorf("specified stack %s has nil value for key %s", stackName, *output.OutputKey) + } + + switch *output.OutputKey { + case "InstallationName": + params.Name = *output.OutputValue + case "SshKeyName": + params.SshKeyName = *output.OutputValue + case "ComputeRoleArn": + params.ComputeRoleArn = *output.OutputValue + case "AccountId": + params.AccountId = *output.OutputValue + case "AllowedSubnetIds": + params.AllowedSubnetIds = []string{*output.OutputValue} + case "SecurityGroupId": + params.SecurityGroupId = *output.OutputValue + case "Region": + params.Region = *output.OutputValue + case "InstanceProfileArn": + params.InstanceProfileArn = *output.OutputValue + } + } + + return params, nil +} diff --git a/go.mod b/go.mod index 0402334c..dc7b477d 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/alexcb/binarystream v0.0.0-20231130184431-f2f7a7543c6d github.com/aws/aws-sdk-go-v2 v1.26.1 github.com/aws/aws-sdk-go-v2/config v1.18.16 + github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230227212328-9f4511cd144a github.com/containerd/containerd v1.7.8 github.com/containerd/go-runc v1.1.0 @@ -18,7 +19,7 @@ require ( github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.5.0 github.com/dustin/go-humanize v1.0.1 - github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f + github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150 github.com/earthly/earthly/ast v0.0.0-00010101000000-000000000000 github.com/earthly/earthly/util/deltautil v0.0.0-20240507235053-335389ed3e2a github.com/elastic/go-sysinfo v1.9.0 @@ -72,8 +73,8 @@ require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230219212500-1f9a474cc2dc // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.16 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 // indirect github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.1 // indirect diff --git a/go.sum b/go.sum index 3ce488a6..540c48a2 100644 --- a/go.sum +++ b/go.sum @@ -85,16 +85,18 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24/go.mod h1:neYVaeKr5eT7Bzw github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30/go.mod h1:LUBAO3zNXQjoONBKn/kR1y0Q4cj/D02Ts0uHYjcCQLM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24/go.mod h1:gAuCezX/gob6BSMbItsSlMb6WZGV7K2+fWOvk8xBSto= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 h1:hf+Vhp5WtTdcSdE+yEcUz8L73sAzN0R+0jQv+Z51/mI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31/go.mod h1:5zUjguZfG5qjhG9/wqmuyHRyUftl2B5Cp6NNxNC6kRA= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 h1:Ap5tOJfeAH1hO2UQc3X3uMlwP7uryFeZXMvZCXIlLSE= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0/go.mod h1:/v2KYdCW4BaHKayenaWEXOOdxItIwEA3oU0XzuQY3F0= github.com/aws/aws-sdk-go-v2/service/ecr v1.18.2/go.mod h1:53xgmccefO+AwKsxVKuTh2vo/IDOkeMWNpmDuhZH1Vc= github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 h1:+sbyLjtAq0Xg9ZOQ2mBibklsGUyX6I2OfRTDsha9uU4= github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3/go.mod h1:/m9MiYl5Ds0cZqy/bbeSUWxKLwTarGugjXxSgiXNQFc= @@ -192,8 +194,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec h1:vf6x0fPOWKakjH3n2N1O9Tg5j1HDIJsC3Kkgmuko2U0= github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec/go.mod h1:1/yAC8A0Tu94Bdmv07gaG1pFBp+CetVwO7oB3qvZXUc= -github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f h1:8CXT0MQ7dQrtm/IwVIexffosImh4ht0WUiWQAt0UoeQ= -github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI= +github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150 h1:hHdgFB5BE+OAFee+CpassZP2GOkdRt5YyhsFmJJVtc8= +github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI= github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65 h1:6oyWHoxHXwcTt4EqmMw6361scIV87uEAB1N42+VpIwk= github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI= github.com/elastic/go-sysinfo v1.9.0 h1:usICqY/Nw4Mpn9f4LdtpFrKxXroJDe81GaxxUlCckIo=