Merge pull request #76 from aws/feature/subdomain-user-operations

Subdomain User Operations

controllers/cloudstackaffinitygroup_controller.go

@@ -37,7 +37,6 @@ import (
 type CloudStackAGReconciliationRunner struct {
 	csCtrlrUtils.ReconciliationRunner
 	ReconciliationSubject *infrav1.CloudStackAffinityGroup
-	CSUser                cloud.Client
 }
 
 // CloudStackAGReconciler is the base reconciler to adapt to k8s.
@@ -65,7 +64,7 @@ func (reconciler *CloudStackAffinityGroupReconciler) Reconcile(ctx context.Conte
 func (r *CloudStackAGReconciliationRunner) Reconcile() (ctrl.Result, error) {
 	controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.AffinityGroupFinalizer)
 	affinityGroup := &cloud.AffinityGroup{Name: r.ReconciliationSubject.Spec.Name, Type: "host affinity"}
-	if err := r.CSClient.GetOrCreateAffinityGroup(affinityGroup); err != nil {
+	if err := r.CSUser.GetOrCreateAffinityGroup(affinityGroup); err != nil {
 		return ctrl.Result{}, err
 	}
 	r.ReconciliationSubject.Spec.ID = affinityGroup.ID
@@ -75,11 +74,11 @@ func (r *CloudStackAGReconciliationRunner) Reconcile() (ctrl.Result, error) {
 
 func (r *CloudStackAGReconciliationRunner) ReconcileDelete() (ctrl.Result, error) {
 	group := &cloud.AffinityGroup{Name: r.ReconciliationSubject.Name}
-	_ = r.CSClient.FetchAffinityGroup(group)
+	_ = r.CSUser.FetchAffinityGroup(group)
 	if group.ID == "" { // Affinity group not found, must have been deleted.
 		return ctrl.Result{}, nil
 	}
-	if err := r.CSClient.DeleteAffinityGroup(group); err != nil {
+	if err := r.CSUser.DeleteAffinityGroup(group); err != nil {
 		return ctrl.Result{}, err
 	}
 	controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.AffinityGroupFinalizer)

controllers/cloudstackcluster_controller.go

@@ -30,7 +30,6 @@ import (
 
 	infrav1 "github.com/aws/cluster-api-provider-cloudstack/api/v1beta1"
 	csCtrlrUtils "github.com/aws/cluster-api-provider-cloudstack/controllers/utils"
-	"github.com/aws/cluster-api-provider-cloudstack/pkg/cloud"
 	"github.com/pkg/errors"
 	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util"
@@ -53,7 +52,6 @@ type CloudStackClusterReconciliationRunner struct {
 	csCtrlrUtils.ReconciliationRunner
 	Zones                 *infrav1.CloudStackZoneList
 	ReconciliationSubject *infrav1.CloudStackCluster
-	CSUser                cloud.Client
 }
 
 // CloudStackClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster.
@@ -94,24 +92,14 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re
 		r.GetZones(r.Zones),
 		r.VerifyZoneCRDs,
 		r.SetFailureDomains,
-		r.ResolveClusterDetails)
+		r.SetReady)
 }
 
-// ResolveClusterDetails fetches cluster specific details like domain and account IDs.
-func (r *CloudStackClusterReconciliationRunner) ResolveClusterDetails() (ctrl.Result, error) {
-	// Ensure that CAPI won't prematurely delete this CloudStackCluster.
+// SetReady adds a finalizer and sets the cluster status to ready.
+func (r *CloudStackClusterReconciliationRunner) SetReady() (ctrl.Result, error) {
 	controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
-
-	// Create and or fetch cluster components.
-	err := r.CSClient.GetOrCreateCluster(r.ReconciliationSubject)
-	if err == nil {
-		r.Log.Info("Fetched cluster info successfully.")
-		r.Log.V(1).Info("Post fetch cluster status.", "clusterStatus", r.ReconciliationSubject.Status)
-
-		// Set cluster to ready to indicate readiness to CAPI.
-		r.ReconciliationSubject.Status.Ready = true
-	}
-	return ctrl.Result{}, err
+	r.ReconciliationSubject.Status.Ready = true
+	return ctrl.Result{}, nil
 }
 
 // CheckZoneDetails verifies the Zone CRDs found match against those requested.
@@ -152,9 +140,6 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result,
 		}
 		return r.RequeueWithMessage("Child Zones still present, requeueing.")
 	}
-	if err := r.CSClient.DisposeClusterResources(r.ReconciliationSubject); err != nil {
-		return ctrl.Result{}, err
-	}
 	controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
 	return ctrl.Result{}, nil
 }

controllers/cloudstackisolatednetwork_controller.go

@@ -44,7 +44,6 @@ type CloudStackIsoNetReconciliationRunner struct {
 	csCtrlrUtils.ReconciliationRunner
 	Zone                  *infrav1.CloudStackZone
 	ReconciliationSubject *infrav1.CloudStackIsolatedNetwork
-	CSUser                cloud.Client
 }
 
 // Initialize a new CloudStackIsoNet reconciliation runner with concrete types and initialized member fields.
@@ -80,11 +79,11 @@ func (r *CloudStackIsoNetReconciliationRunner) Reconcile() (retRes ctrl.Result,
 	if err != nil {
 		return r.ReturnWrappedError(retErr, "setting up CloudStackCluster patcher:")
 	}
-	if err := r.CSClient.GetOrCreateIsolatedNetwork(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
+	if err := r.CSUser.GetOrCreateIsolatedNetwork(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
 		return ctrl.Result{}, err
 	}
 	// Tag the created network.
-	if err := r.CSClient.AddClusterTag(cloud.ResourceTypeNetwork, r.ReconciliationSubject.Spec.ID, r.CSCluster); err != nil {
+	if err := r.CSUser.AddClusterTag(cloud.ResourceTypeNetwork, r.ReconciliationSubject.Spec.ID, r.CSCluster); err != nil {
 		return ctrl.Result{}, errors.Wrapf(err, "tagging network with id %s:", r.ReconciliationSubject.Spec.ID)
 	}
 	if err := csClusterPatcher.Patch(r.RequestCtx, r.CSCluster); err != nil {
@@ -97,7 +96,7 @@ func (r *CloudStackIsoNetReconciliationRunner) Reconcile() (retRes ctrl.Result,
 
 func (r *CloudStackIsoNetReconciliationRunner) ReconcileDelete() (retRes ctrl.Result, retErr error) {
 	r.Log.Info("Deleting IsolatedNetwork.")
-	if err := r.CSClient.DisposeIsoNetResources(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
+	if err := r.CSUser.DisposeIsoNetResources(r.Zone, r.ReconciliationSubject, r.CSCluster); err != nil {
 		if !strings.Contains(strings.ToLower(err.Error()), "no match found") {
 			return ctrl.Result{}, err
 		}

controllers/cloudstackmachine_controller.go

@@ -53,7 +53,6 @@ type CloudStackMachineReconciliationRunner struct {
 	ReconciliationSubject *infrav1.CloudStackMachine
 	CAPIMachine           *capiv1.Machine
 	StateChecker          *infrav1.CloudStackMachineStateChecker
-	CSUser                cloud.Client
 	Zones                 *infrav1.CloudStackZoneList
 	FailureDomain         *infrav1.CloudStackZone
 	IsoNet                *infrav1.CloudStackIsolatedNetwork
@@ -122,7 +121,7 @@ func (r *CloudStackMachineReconciliationRunner) ConsiderAffinity() (ctrl.Result,
 		return res, err
 	}
 	if !r.AffinityGroup.Status.Ready {
-		return r.RequeueWithMessage("Required afinity group not ready.")
+		return r.RequeueWithMessage("Required affinity group not ready.")
 	}
 	return ctrl.Result{}, nil
 }
@@ -191,7 +190,7 @@ func (r *CloudStackMachineReconciliationRunner) GetOrCreateVMInstance() (retRes
 		return ctrl.Result{}, errors.New("bootstrap secret data not yet set")
 	}
 
-	err := r.CSClient.GetOrCreateVMInstance(r.ReconciliationSubject, r.CAPIMachine, r.CSCluster, &machineZone, r.AffinityGroup, string(data))
+	err := r.CSUser.GetOrCreateVMInstance(r.ReconciliationSubject, r.CAPIMachine, r.CSCluster, &machineZone, r.AffinityGroup, string(data))
 
 	if err == nil && !controllerutil.ContainsFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer) { // Fetched or Created?
 		r.Log.Info("CloudStack instance Created", "instanceStatus", r.ReconciliationSubject.Status)
@@ -225,7 +224,7 @@ func (r *CloudStackMachineReconciliationRunner) AddToLBIfNeeded() (retRes ctrl.R
 		if r.IsoNet.Spec.Name == "" {
 			return r.RequeueWithMessage("Could not get required Isolated Network for VM, requeueing.")
 		}
-		err := r.CSClient.AssignVMToLoadBalancerRule(r.IsoNet, *r.ReconciliationSubject.Spec.InstanceID)
+		err := r.CSUser.AssignVMToLoadBalancerRule(r.IsoNet, *r.ReconciliationSubject.Spec.InstanceID)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
@@ -251,6 +250,9 @@ func (r *CloudStackMachineReconciliationRunner) GetOrCreateMachineStateChecker()
 
 func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.Result, reterr error) {
 	r.Log.Info("Deleting instance", "instance-id", r.ReconciliationSubject.Spec.InstanceID)
+	// Use CSClient instead of CSUser here to expunge as admin.
+	// The CloudStack-Go API does not return an error, but the VM won't delete with Expunge set if requested by
+	// non-domain admin user.
 	if err := r.CSClient.DestroyVMInstance(r.ReconciliationSubject); err != nil {
 		if err.Error() == "VM deletion in progress" {
 			r.Log.Info(err.Error())

controllers/cloudstackzone_controller.go

@@ -24,7 +24,6 @@ import (
 
 	infrav1 "github.com/aws/cluster-api-provider-cloudstack/api/v1beta1"
 	csCtrlrUtils "github.com/aws/cluster-api-provider-cloudstack/controllers/utils"
-	"github.com/aws/cluster-api-provider-cloudstack/pkg/cloud"
 	"github.com/pkg/errors"
 )
 
@@ -37,7 +36,6 @@ type CloudStackZoneReconciliationRunner struct {
 	csCtrlrUtils.ReconciliationRunner
 	Zones                 *infrav1.CloudStackZoneList
 	ReconciliationSubject *infrav1.CloudStackZone
-	CSUser                cloud.Client
 	IsoNet                *infrav1.CloudStackIsolatedNetwork
 }
 
@@ -83,10 +81,10 @@ func (r *CloudStackZoneReconciliationRunner) Reconcile() (retRes ctrl.Result, re
 
 	r.Log.V(1).Info("Reconciling CloudStackZone.", "zoneSpec", r.ReconciliationSubject.Spec)
 	// Start by purely data fetching information about the zone and specified network.
-	if err := r.CSClient.ResolveZone(r.ReconciliationSubject); err != nil {
+	if err := r.CSUser.ResolveZone(r.ReconciliationSubject); err != nil {
 		return ctrl.Result{}, errors.Wrap(err, "resolving CloudStack zone information:")
 	}
-	if err := r.CSClient.ResolveNetworkForZone(r.ReconciliationSubject); err != nil &&
+	if err := r.CSUser.ResolveNetworkForZone(r.ReconciliationSubject); err != nil &&
 		!csCtrlrUtils.ContainsNoMatchSubstring(err) {
 		return ctrl.Result{}, errors.Wrap(err, "resolving Cloudstack network information:")
 	}

controllers/utils/affinity_group.go

@@ -22,6 +22,7 @@ import (
 
 	infrav1 "github.com/aws/cluster-api-provider-cloudstack/api/v1beta1"
 	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -40,25 +41,30 @@ func (r *ReconciliationRunner) GetOrCreateAffinityGroup(name string, affinityTyp
 			return ctrl.Result{}, nil
 		} // Didn't find a group, so create instead.
 
+		// Set affinity group type.
 		if affinityType == infrav1.ProAffinity {
 			ag.Spec.Type = "host affinity"
 		} else if affinityType == infrav1.AntiAffinity {
 			ag.Spec.Type = "host anti-affinity"
 		} else {
 			return ctrl.Result{}, errors.Errorf("unrecognized affinity type %s", affinityType)
 		}
+
+		// Setup basic metadata.
+		ag.Name = name
 		ag.Spec.Name = name
+		ag.ObjectMeta = r.NewChildObjectMeta(lowerName)
 
+		// Replace owner reference with controller of CAPI and CloudStack machines.
 		for _, ref := range r.ReconciliationSubject.GetOwnerReferences() {
-			if ref.Kind == "EtcdadmCluster" || ref.Kind == "KubeadmControlPlane" || ref.Kind == "MachineSet" {
-				ag.OwnerReferences = append(ag.OwnerReferences, ref)
+			if strings.EqualFold(ref.Kind, "EtcdadmCluster") ||
+				strings.EqualFold(ref.Kind, "KubeadmControlPlane") ||
+				strings.EqualFold(ref.Kind, "MachineSet") {
+				ag.OwnerReferences = []metav1.OwnerReference{ref}
 				break
 			}
 		}
 
-		ag.Name = name
-		ag.Spec.Name = name
-		ag.ObjectMeta = r.NewChildObjectMeta(lowerName)
 		if err := r.K8sClient.Create(r.RequestCtx, ag); err != nil && !ContainsAlreadyExistsSubstring(err) {
 			return r.ReturnWrappedError(err, "creating affinity group CRD:")
 		}

controllers/utils/base_reconciler.go

@@ -68,6 +68,7 @@ type ReconciliationRunner struct {
 	returnEarly           bool          // A signal that the reconcile should return early.
 	ReconcileDelete       CloudStackReconcilerMethod
 	Reconcile             CloudStackReconcilerMethod
+	CSUser                cloud.Client
 }
 
 type ConcreteRunner interface {
@@ -219,6 +220,29 @@ func (r *ReconciliationRunner) CheckOwnedCRDsForReadiness(gvks ...schema.GroupVe
 	}
 }
 
+// SetCSUser sets the CSUser client to any user that can operate in specified domain and account if specified.
+func (r *ReconciliationRunner) SetCSUser() (ctrl.Result, error) {
+	r.CSUser = r.CSClient
+	if r.CSCluster.Spec.Account != "" {
+		user := &cloud.User{}
+		user.Account.Domain.Path = r.CSCluster.Spec.Domain
+		user.Account.Name = r.CSCluster.Spec.Account
+		if found, err := r.CSClient.GetUserWithKeys(user); err != nil {
+			return ctrl.Result{}, err
+		} else if !found {
+			return ctrl.Result{}, errors.Errorf("could not find sufficient user (with API keys) in domain/account %s/%s",
+				r.CSCluster.Spec.Domain, r.CSCluster.Spec.Account)
+		}
+		cfg := cloud.Config{APIKey: user.APIKey, SecretKey: user.SecretKey}
+		client, err := r.CSClient.NewClientFromSpec(cfg)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		r.CSUser = client
+	}
+	return ctrl.Result{}, nil
+}
+
 // RequeueIfCloudStackClusterNotReady requeues the reconciliation request if the CloudStackCluster is not ready.
 func (r *ReconciliationRunner) RequeueIfCloudStackClusterNotReady() (ctrl.Result, error) {
 	if !r.CSCluster.Status.Ready {
@@ -303,6 +327,8 @@ func (r *ReconciliationRunner) RunBaseReconciliationStages() (res ctrl.Result, r
 		r.GetReconciliationSubject,
 		r.GetCAPICluster,
 		r.GetCSCluster,
+		r.RequeueIfMissingBaseCRs,
+		r.SetCSUser,
 		r.IfDeletionTimestampIsZero(r.Reconcile),
 		r.Else(r.ReconcileDelete),
 	)

go.mod

@@ -3,7 +3,7 @@ module github.com/aws/cluster-api-provider-cloudstack
 go 1.16
 
 require (
-	github.com/apache/cloudstack-go/v2 v2.11.1-0.20211020121644-369057554f66
+	github.com/apache/cloudstack-go/v2 v2.13.0
 	github.com/go-logr/logr v0.4.0
 	github.com/golang/mock v1.6.0
 	github.com/hashicorp/go-multierror v1.1.1

go.sum

@@ -76,16 +76,15 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/apache/cloudstack-go/v2 v2.11.1-0.20211020121644-369057554f66 h1:oy3ZgpLnqxczrbO48a3ajp2sEmRELLWcFH7foIACrkU=
-github.com/apache/cloudstack-go/v2 v2.11.1-0.20211020121644-369057554f66/go.mod h1:2mPSVR6kkM3u1i9L3mmR+Du1f+QBwFKYOSI2XQEyhj4=
+github.com/apache/cloudstack-go/v2 v2.13.0 h1:t0uj7QxQpnzD/LSTP6a4w2NTuZXisxIM/mIDNkF44lc=
+github.com/apache/cloudstack-go/v2 v2.13.0/go.mod h1:aosD8Svfu5nhH5Sp4zcsVV1hT5UGt3mTgRXM8YqTKe0=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
-github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=

pkg/cloud/affinity_groups_test.go

@@ -83,9 +83,6 @@ var _ = Describe("AffinityGroup Unit Tests", func() {
 			Ω(client.GetOrCreateAffinityGroup(dummies.AffinityGroup)).Should(Succeed())
 		})
 		It("Associates an affinity group.", func() {
-			if err := client.GetOrCreateCluster(dummies.CSCluster); err != nil {
-				Skip("Could not flesh out Cluster." + err.Error())
-			}
 			if err := client.GetOrCreateVMInstance(
 				dummies.CSMachine1, dummies.CAPIMachine, dummies.CSCluster, dummies.CSZone1, dummies.CSAffinityGroup, "",
 			); err != nil {