diff --git a/charts/hub-net-controller-manager/templates/deployment.yaml b/charts/hub-net-controller-manager/templates/deployment.yaml index ad59ceb0..14bb53e4 100644 --- a/charts/hub-net-controller-manager/templates/deployment.yaml +++ b/charts/hub-net-controller-manager/templates/deployment.yaml @@ -28,6 +28,7 @@ spec: - --leader-election-namespace={{ .Values.leaderElectionNamespace }} - --v={{ .Values.logVerbosity }} - --add_dir_header + - --force-delete-wait-time={{ .Values.forceDeleteWaitTime }} ports: - name: metrics containerPort: 8080 diff --git a/charts/hub-net-controller-manager/templates/rbac.yaml b/charts/hub-net-controller-manager/templates/rbac.yaml index 00702fd4..be9449be 100644 --- a/charts/hub-net-controller-manager/templates/rbac.yaml +++ b/charts/hub-net-controller-manager/templates/rbac.yaml @@ -129,6 +129,14 @@ rules: - get - patch - update +- apiGroups: + - cluster.kubernetes-fleet.io + resources: + - memberclusters + verbs: + - get + - list + - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/hub-net-controller-manager/values.yaml b/charts/hub-net-controller-manager/values.yaml index be0b77e0..f6fa97dd 100644 --- a/charts/hub-net-controller-manager/values.yaml +++ b/charts/hub-net-controller-manager/values.yaml @@ -6,7 +6,7 @@ replicaCount: 1 image: repository: ghcr.io/azure/fleet-networking/hub-net-controller-manager - pullPolicy: IfNotPresent + pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: "v0.1.0" @@ -14,6 +14,7 @@ logVerbosity: 2 leaderElectionNamespace: fleet-system fleetSystemNamespace: fleet-system +forceDeleteWaitTime: 2m0s resources: limits: diff --git a/cmd/hub-net-controller-manager/main.go b/cmd/hub-net-controller-manager/main.go index 77fca2f1..23daa5c8 100644 --- a/cmd/hub-net-controller-manager/main.go +++ b/cmd/hub-net-controller-manager/main.go @@ -26,11 +26,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" //+kubebuilder:scaffold:imports + clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" "go.goms.io/fleet-networking/pkg/controllers/hub/endpointsliceexport" "go.goms.io/fleet-networking/pkg/controllers/hub/internalserviceexport" "go.goms.io/fleet-networking/pkg/controllers/hub/internalserviceimport" + "go.goms.io/fleet-networking/pkg/controllers/hub/membercluster" "go.goms.io/fleet-networking/pkg/controllers/hub/serviceimport" ) @@ -47,11 +49,14 @@ var ( internalServiceExportRetryInterval = flag.Duration("internalserviceexport-retry-interval", 2*time.Second, "The wait time for the internalserviceexport controller to requeue the request and to wait for the"+ "ServiceImport controller to resolve the service Spec") + + forceDeleteWaitTime = flag.Duration("force-delete-wait-time", 15*time.Minute, "The duration the fleet hub agent waits before trying to force delete a member cluster.") ) func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(fleetnetv1alpha1.AddToScheme(scheme)) + utilruntime.Must(clusterv1beta1.AddToScheme(scheme)) klog.InitFlags(nil) //+kubebuilder:scaffold:scheme } @@ -140,6 +145,16 @@ func main() { exitWithErrorFunc() } + klog.V(1).InfoS("Start to setup MemberCluster controller") + if err := (&membercluster.Reconciler{ + Client: mgr.GetClient(), + Recorder: mgr.GetEventRecorderFor(membercluster.ControllerName), + ForceDeleteWaitTime: *forceDeleteWaitTime, + }).SetupWithManager(mgr); err != nil { + klog.ErrorS(err, "Unable to create MemberCluster controller") + exitWithErrorFunc() + } + klog.V(1).InfoS("Starting ServiceExportImport controller manager") if err := mgr.Start(ctx); err != nil { klog.ErrorS(err, "Problem running manager") diff --git a/examples/getting-started/artifacts/hub-rbac.yaml b/examples/getting-started/artifacts/hub-rbac.yaml index ea22d163..2e0f163e 100644 --- a/examples/getting-started/artifacts/hub-rbac.yaml +++ b/examples/getting-started/artifacts/hub-rbac.yaml @@ -26,6 +26,7 @@ rules: - patch - apiGroups: - networking.fleet.azure.com + - cluster.kubernetes-fleet.io resources: ["*"] verbs: ["*"] --- @@ -70,6 +71,7 @@ rules: - patch - apiGroups: - networking.fleet.azure.com + - cluster.kubernetes-fleet.io resources: ["*"] verbs: ["*"] --- diff --git a/go.mod b/go.mod index 6f5f9ac7..8ffe32ac 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( require ( github.com/stretchr/testify v1.9.0 go.goms.io/fleet v0.10.5 + golang.org/x/sync v0.7.0 ) require ( diff --git a/go.sum b/go.sum index 859000df..3660d722 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/common/hubconfig/hubconfig.go b/pkg/common/hubconfig/hubconfig.go index 6edf424f..a32163b2 100644 --- a/pkg/common/hubconfig/hubconfig.go +++ b/pkg/common/hubconfig/hubconfig.go @@ -34,7 +34,7 @@ const ( // Naming pattern of member cluster namespace in hub cluster, should be the same as envValue as defined in // https://github.com/Azure/fleet/blob/main/pkg/utils/common.go - hubNamespaceNameFormat = "fleet-member-%s" + HubNamespaceNameFormat = "fleet-member-%s" ) // PrepareHubConfig return the config holding attributes for a Kubernetes client to request hub cluster. @@ -115,5 +115,5 @@ func FetchMemberClusterNamespace() (string, error) { klog.ErrorS(err, "Member cluster name cannot be empty") return "", err } - return fmt.Sprintf(hubNamespaceNameFormat, mcName), nil + return fmt.Sprintf(HubNamespaceNameFormat, mcName), nil } diff --git a/pkg/common/hubconfig/hubconfig_test.go b/pkg/common/hubconfig/hubconfig_test.go index bd80f151..4e80de74 100644 --- a/pkg/common/hubconfig/hubconfig_test.go +++ b/pkg/common/hubconfig/hubconfig_test.go @@ -161,7 +161,7 @@ func TestFetchMemberClusterNamespace(t *testing.T) { name: "environment variable is present", envKey: "MEMBER_CLUSTER_NAME", envValue: memberCluster, - want: fmt.Sprintf(hubNamespaceNameFormat, memberCluster), + want: fmt.Sprintf(HubNamespaceNameFormat, memberCluster), wantErr: false, }, { diff --git a/pkg/controllers/hub/membercluster/controller.go b/pkg/controllers/hub/membercluster/controller.go new file mode 100644 index 00000000..0e0546e3 --- /dev/null +++ b/pkg/controllers/hub/membercluster/controller.go @@ -0,0 +1,134 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +// Package membercluster features the MemberCluster controller for watching +// update/delete events to the MemberCluster object and removes finalizers +// on all fleet networking resources in the fleet member cluster namespace. +package membercluster + +import ( + "context" + "fmt" + "time" + + "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" + "go.goms.io/fleet/pkg/utils/controller" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + "go.goms.io/fleet-networking/pkg/common/hubconfig" +) + +const ( + ControllerName = "membercluster-controller" +) + +// Reconciler reconciles a MemberCluster object. +type Reconciler struct { + client.Client + Recorder record.EventRecorder + // the wait time in minutes before we need to force delete a member cluster. + ForceDeleteWaitTime time.Duration +} + +// Reconcile watches the deletion of the member cluster and removes finalizers on fleet networking resources in the +// member cluster namespace. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + mcObjRef := klog.KRef(req.Namespace, req.Name) + startTime := time.Now() + klog.V(2).InfoS("Reconciliation starts", "memberCluster", mcObjRef) + defer func() { + latency := time.Since(startTime).Milliseconds() + klog.V(2).InfoS("Reconciliation ends", "memberCluster", mcObjRef, "latency", latency) + }() + var mc clusterv1beta1.MemberCluster + if err := r.Client.Get(ctx, req.NamespacedName, &mc); err != nil { + if errors.IsNotFound(err) { + klog.V(4).InfoS("Ignoring NotFound memberCluster", "memberCluster", mcObjRef) + return ctrl.Result{}, nil + } + klog.ErrorS(err, "Failed to get memberCluster", "memberCluster", mcObjRef) + return ctrl.Result{}, err + } + if mc.DeletionTimestamp.IsZero() { + klog.ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("member cluster %s is not being deleted", + mc.Name)), "The member cluster should have deletionTimeStamp set to a non-zero/non-nil value") + return ctrl.Result{}, nil // no need to retry. + } + + // Handle deleting member cluster, removes finalizers on all the resources in the cluster namespace + // after member cluster force delete wait time. + if !mc.DeletionTimestamp.IsZero() && time.Since(mc.DeletionTimestamp.Time) >= r.ForceDeleteWaitTime { + klog.V(2).InfoS("The member cluster deletion is stuck removing the "+ + "finalizers from all the resources in member cluster namespace", "memberCluster", mcObjRef) + return r.removeFinalizer(ctx, mc) + } + // we need to only wait for force delete wait time, if the update/delete member cluster event takes + // longer to be reconciled we need to account for that time. + return ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime - time.Since(mc.DeletionTimestamp.Time)}, nil +} + +// removeFinalizer removes finalizers on the resources in the member cluster namespace. +// For EndpointSliceExport, InternalServiceImport & InternalServiceExport resources, the finalizers should be +// removed by other hub networking controllers when leaving. So this MemberCluster controller only handles +// EndpointSliceImports here. +func (r *Reconciler) removeFinalizer(ctx context.Context, mc clusterv1beta1.MemberCluster) (ctrl.Result, error) { + // Remove finalizer for EndpointSliceImport resources in the cluster namespace. + mcObjRef := klog.KRef(mc.Namespace, mc.Name) + mcNamespace := fmt.Sprintf(hubconfig.HubNamespaceNameFormat, mc.Name) + var endpointSliceImportList fleetnetv1alpha1.EndpointSliceImportList + if err := r.Client.List(ctx, &endpointSliceImportList, client.InNamespace(mcNamespace)); err != nil { + klog.ErrorS(err, "Failed to list endpointSliceImports", "memberCluster", mcObjRef) + return ctrl.Result{}, err + } + errs, ctx := errgroup.WithContext(ctx) + for i := range endpointSliceImportList.Items { + esi := &endpointSliceImportList.Items[i] + errs.Go(func() error { + esiObjRef := klog.KRef(esi.Namespace, esi.Name) + esi.SetFinalizers(nil) + if err := r.Client.Update(ctx, esi); err != nil { + klog.ErrorS(err, "Failed to remove finalizers for endpointSliceImport", + "memberCluster", mcObjRef, "endpointSliceImport", esiObjRef) + return err + } + klog.V(2).InfoS("Removed finalizers for endpointSliceImport", + "memberCluster", mcObjRef, "endpointSliceImport", esiObjRef) + return nil + }) + } + return ctrl.Result{}, errs.Wait() +} + +// SetupWithManager sets up the controller with the Manager. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + customPredicate := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + // Ignore creation events. + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // trigger reconcile on delete event just in case update event is missed. + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + // If new object is being deleted, trigger reconcile. + return !e.ObjectNew.GetDeletionTimestamp().IsZero() + }, + } + // Watch for changes to primary resource MemberCluster + return ctrl.NewControllerManagedBy(mgr). + For(&clusterv1beta1.MemberCluster{}). + WithEventFilter(customPredicate). + Complete(r) +} diff --git a/pkg/controllers/hub/membercluster/controller_test.go b/pkg/controllers/hub/membercluster/controller_test.go new file mode 100644 index 00000000..2133f2c1 --- /dev/null +++ b/pkg/controllers/hub/membercluster/controller_test.go @@ -0,0 +1,220 @@ +package membercluster + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" +) + +const ( + testMemberClusterName = "test-mc" + testEndpointSliceImport = "test-esi" + forceDeleteWaitTime = 15 * time.Minute +) + +var ( + errFake = errors.New("fake error") +) + +var deletionTimeStamp = time.Now() + +func TestReconcile(t *testing.T) { + testCases := []struct { + name string + memberClusterName string + memberCluster clusterv1beta1.MemberCluster + shouldGetErr bool + wantResult ctrl.Result + wantErr error + }{ + { + name: "memberCluster is not found", + memberClusterName: testMemberClusterName, + wantResult: ctrl.Result{}, + wantErr: nil, + }, + { + name: "failed to get memberCluster", + memberClusterName: testMemberClusterName, + shouldGetErr: true, + wantResult: ctrl.Result{}, + wantErr: errFake, + }, + { + name: "memberCluster deletionTimestamp is nil", + memberClusterName: testMemberClusterName, + memberCluster: clusterv1beta1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mc", + }, + }, + wantResult: ctrl.Result{}, + wantErr: nil, + }, + { + name: "time since memberCluster deletionTimestamp is less than force delete wait time", + memberClusterName: testMemberClusterName, + memberCluster: clusterv1beta1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mc", + DeletionTimestamp: &metav1.Time{Time: deletionTimeStamp}, + Finalizers: []string{"test-member-cluster-cleanup-finalizer"}, + }, + }, + wantResult: ctrl.Result{RequeueAfter: forceDeleteWaitTime - time.Since(deletionTimeStamp)}, + wantErr: nil, + }, + { + name: "time since memberCluster deletionTimestamp is greater than force delete wait time", + memberClusterName: testMemberClusterName, + memberCluster: clusterv1beta1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mc", + // To set deletionTimeStamp to some time 20 minutes before. + DeletionTimestamp: &metav1.Time{Time: deletionTimeStamp.Add(-20 * time.Minute)}, + Finalizers: []string{"test-member-cluster-cleanup-finalizer"}, + }, + }, + wantResult: ctrl.Result{}, + wantErr: nil, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + errorFakeClient := errorReturningFakeClient{ + Client: fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(&tc.memberCluster). + Build(), + shouldReadError: tc.shouldGetErr, + } + + r := Reconciler{ + Client: errorFakeClient, + ForceDeleteWaitTime: forceDeleteWaitTime, + } + + gotResult, gotErr := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: tc.memberClusterName}}) + if !errors.Is(gotErr, tc.wantErr) { + t.Errorf("Reconcile() error = %+v, want = %+v", gotErr, tc.wantErr) + } + // Want RequeueAfter is calculated when we expect it to be not zero. Got RequeueAfter from reconcile + // will always be different from Want RequeueAfter because it calculated when the testCase is built. + if got, want := gotResult.RequeueAfter == 0, tc.wantResult.RequeueAfter == 0; got != want { + t.Errorf("Reconcile() RequeueAfter is zero = %v, want %v", got, want) + } + }) + } +} + +func TestRemoveFinalizer(t *testing.T) { + testCases := []struct { + name string + memberCluster clusterv1beta1.MemberCluster + endPointSliceImport fleetnetv1alpha1.EndpointSliceImport + shouldListErr bool + shouldUpdateErr bool + wantResult ctrl.Result + wantErr error + }{ + // the happy path is handled as part of IT. + { + name: "failed to list endpointSliceImports", + memberCluster: clusterv1beta1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberClusterName, + }, + }, + shouldListErr: true, + wantResult: ctrl.Result{}, + wantErr: errFake, + }, + { + name: "failed to update endpointSliceImport", + memberCluster: clusterv1beta1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberClusterName, + }, + }, + endPointSliceImport: *buildEndpointSliceImport(testEndpointSliceImport), + shouldListErr: false, + shouldUpdateErr: true, + wantResult: ctrl.Result{}, + wantErr: errFake, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + errorFakeClient := errorReturningFakeClient{ + Client: fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithObjects(&tc.endPointSliceImport). + Build(), + shouldReadError: tc.shouldListErr, + shouldWriteError: tc.shouldUpdateErr, + } + r := Reconciler{ + Client: errorFakeClient, + } + gotResult, gotErr := r.removeFinalizer(context.Background(), tc.memberCluster) + if !errors.Is(gotErr, tc.wantErr) { + t.Errorf("removeFinalizer() error = %+v, want = %+v", gotErr, tc.wantErr) + } + if !cmp.Equal(gotResult, tc.wantResult) { + t.Errorf("removeFinalizer() result = %v, want %v", gotResult, tc.wantResult) + } + }) + } +} + +func testScheme(t *testing.T) *runtime.Scheme { + scheme := runtime.NewScheme() + if err := clusterv1beta1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add scheme: %v", err) + } + if err := fleetnetv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add scheme: %v", err) + } + return scheme +} + +type errorReturningFakeClient struct { + client.Client + shouldReadError bool + shouldWriteError bool +} + +func (fc errorReturningFakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if fc.shouldReadError { + return fmt.Errorf("get failed %w", errFake) + } + return fc.Client.Get(ctx, key, obj, opts...) +} + +func (fc errorReturningFakeClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if fc.shouldReadError { + return fmt.Errorf("list failed %w", errFake) + } + return fc.Client.List(ctx, list, opts...) +} + +func (fc errorReturningFakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if fc.shouldWriteError { + return fmt.Errorf("update failed %w", errFake) + } + return fc.Client.Update(ctx, obj, opts...) +} diff --git a/pkg/controllers/hub/membercluster/controlller_integration_test.go b/pkg/controllers/hub/membercluster/controlller_integration_test.go new file mode 100644 index 00000000..ac904c42 --- /dev/null +++ b/pkg/controllers/hub/membercluster/controlller_integration_test.go @@ -0,0 +1,187 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package membercluster + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + "go.goms.io/fleet-networking/pkg/common/hubconfig" +) + +const ( + memberClusterName = "member-1" + + eventuallyTimeout = time.Minute * 2 + eventuallyInterval = time.Second * 5 +) + +var ( + fleetMemberNS = fmt.Sprintf(hubconfig.HubNamespaceNameFormat, "member-1") +) + +var ( + endpointSliceImportNames = []string{"test-endpoint-slice-import-1", "test-endpoint-slice-import-2"} +) + +var _ = Describe("Test MemberCluster Controller", func() { + Context("Test MemberCluster controller, handle force delete", func() { + BeforeEach(func() { + mc := clusterv1beta1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberClusterName, + // finalizer is added to ensure MC is not deleted before the force delete wait time, + // ideally added and removed by fleet hub member cluster controller. + Finalizers: []string{"test-member-cluster-cleanup-finalizer"}, + }, + Spec: clusterv1beta1.MemberClusterSpec{ + Identity: rbacv1.Subject{ + Name: "test-subject", + Kind: "ServiceAccount", + Namespace: "fleet-system", + APIGroup: "", + }, + }, + } + Expect(hubClient.Create(ctx, &mc)).Should(Succeed()) + + // Create the fleet member namespace. + memberNS := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: fleetMemberNS, + }, + } + Expect(hubClient.Create(ctx, &memberNS)).Should(Succeed()) + + // Create the EndpointSliceImports. + for i := range endpointSliceImportNames { + esi := buildEndpointSliceImport(endpointSliceImportNames[i]) + Expect(hubClient.Create(ctx, esi)).Should(Succeed()) + } + }) + + It("should remove finalizer on EndpointSliceImport on fleet member namespace, after force delete wait time is crossed", func() { + // ensure EndpointSliceImports have a finalizer. + var esi fleetnetv1alpha1.EndpointSliceImport + for i := range endpointSliceImportNames { + Expect(hubClient.Get(ctx, types.NamespacedName{Name: endpointSliceImportNames[i], Namespace: fleetMemberNS}, &esi)).Should(Succeed()) + Expect(esi.GetFinalizers()).ShouldNot(BeEmpty()) + } + // delete member cluster to trigger member cluster controller reconcile. + var mc clusterv1beta1.MemberCluster + Expect(hubClient.Get(ctx, types.NamespacedName{Name: memberClusterName}, &mc)).Should(Succeed()) + Expect(hubClient.Delete(ctx, &mc)).Should(Succeed()) + // the force delete wait time is set to 1 minute for this IT. + Eventually(func() error { + var endpointSliceImportList fleetnetv1alpha1.EndpointSliceImportList + if err := hubClient.List(ctx, &endpointSliceImportList, client.InNamespace(fleetMemberNS)); err != nil { + return err + } + if len(endpointSliceImportList.Items) != len(endpointSliceImportNames) { + return fmt.Errorf("length of listed endpointSliceImports %d doesn't match length of endpointSliceImports created %d", + len(endpointSliceImportList.Items), len(endpointSliceImportNames)) + } + for i := range endpointSliceImportList.Items { + esi := &endpointSliceImportList.Items[i] + if len(esi.GetFinalizers()) != 0 { + return fmt.Errorf("finalizers on EndpointSliceImport %s/%s have not been removed", esi.Namespace, esi.Name) + } + } + return nil + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + }) + + AfterEach(func() { + // Delete the namespace, the namespace controller doesn't run in this IT + // hence it won't be removed. + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: fleetMemberNS, + }, + } + Expect(hubClient.Delete(ctx, &ns)).Should(Succeed()) + Expect(hubClient.Get(ctx, types.NamespacedName{Name: fleetMemberNS}, &ns)).Should(Succeed()) + Expect(ns.DeletionTimestamp != nil).Should(BeTrue()) + // Namespace controller doesn't run in IT, so we manually delete and ensure EndpointSliceImports are deleted. + for i := range endpointSliceImportNames { + esi := fleetnetv1alpha1.EndpointSliceImport{ + ObjectMeta: metav1.ObjectMeta{ + Name: endpointSliceImportNames[i], + Namespace: fleetMemberNS, + }, + } + Expect(hubClient.Delete(ctx, &esi)) + } + var esi fleetnetv1alpha1.EndpointSliceImport + Eventually(func() bool { + for i := range endpointSliceImportNames { + if !apierrors.IsNotFound(hubClient.Get(ctx, types.NamespacedName{Name: endpointSliceImportNames[i], Namespace: fleetMemberNS}, &esi)) { + return false + } + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(BeTrue()) + // Remove finalizer from MemberCluster. + var mc clusterv1beta1.MemberCluster + Eventually(func() error { + if err := hubClient.Get(ctx, types.NamespacedName{Name: memberClusterName}, &mc); err != nil { + return err + } + mc.SetFinalizers(nil) + return hubClient.Update(ctx, &mc) + }, eventuallyTimeout, eventuallyInterval).Should(Succeed()) + // Ensure MemberCluster is deleted. + Eventually(func() bool { + return apierrors.IsNotFound(hubClient.Get(ctx, types.NamespacedName{Name: memberClusterName}, &mc)) + }, eventuallyTimeout, eventuallyInterval).Should(BeTrue()) + }) + }) +}) + +func buildEndpointSliceImport(name string) *fleetnetv1alpha1.EndpointSliceImport { + return &fleetnetv1alpha1.EndpointSliceImport{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: fleetMemberNS, + Finalizers: []string{"networking.fleet.azure.com/endpointsliceimport-cleanup"}, + }, + Spec: fleetnetv1alpha1.EndpointSliceExportSpec{ + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: []fleetnetv1alpha1.Endpoint{ + { + Addresses: []string{"1.2.3.4"}, + }, + }, + EndpointSliceReference: fleetnetv1alpha1.ExportedObjectReference{ + ClusterID: memberClusterName, + Kind: "EndpointSlice", + Namespace: fleetMemberNS, + Name: "test-endpoint-slice", + ResourceVersion: "0", + Generation: 1, + UID: "00000000-0000-0000-0000-000000000000", + ExportedSince: metav1.NewTime(time.Now().Round(time.Second)), + }, + OwnerServiceReference: fleetnetv1alpha1.OwnerServiceReference{ + Namespace: "work", + Name: "test-service", + }, + }, + } +} diff --git a/pkg/controllers/hub/membercluster/suite_test.go b/pkg/controllers/hub/membercluster/suite_test.go new file mode 100644 index 00000000..e0804372 --- /dev/null +++ b/pkg/controllers/hub/membercluster/suite_test.go @@ -0,0 +1,106 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package membercluster + +import ( + "context" + "flag" + "go/build" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "k8s.io/klog/v2/textlogger" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" +) + +var ( + hubTestEnv *envtest.Environment + hubClient client.Client + ctx context.Context + cancel context.CancelFunc +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "MemberCluster Controller (Hub) Suite") +} + +var _ = BeforeSuite(func() { + By("Setup klog") + fs := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(fs) + Expect(fs.Parse([]string{"--v", "5", "-add_dir_header", "true"})).Should(Succeed()) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrap the test environment") + // Start the cluster. + hubTestEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "..", "config", "crd", "bases"), + filepath.Join(build.Default.GOPATH, "pkg", "mod", "go.goms.io", "fleet@v0.10.5", "config", "crd", "bases", "cluster.kubernetes-fleet.io_memberclusters.yaml"), + }, + ErrorIfCRDPathMissing: true, + } + hubCfg, err := hubTestEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(hubCfg).NotTo(BeNil()) + + // Add custom APIs to the runtime scheme. + Expect(fleetnetv1alpha1.AddToScheme(scheme.Scheme)).Should(Succeed()) + Expect(clusterv1beta1.AddToScheme(scheme.Scheme)).Should(Succeed()) + + // Start up the MemberCluster controller. + klog.InitFlags(flag.CommandLine) + flag.Parse() + hubCtrlMgr, err := ctrl.NewManager(hubCfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4))), + }) + Expect(err).NotTo(HaveOccurred()) + + // Set up the client. + // The client must be one with cache (i.e. configured by the controller manager) to make + // use of the cache indexes. + hubClient = hubCtrlMgr.GetClient() + Expect(hubClient).NotTo(BeNil()) + + err = (&Reconciler{ + Client: hubClient, + ForceDeleteWaitTime: 1 * time.Minute, + }).SetupWithManager(hubCtrlMgr) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = hubCtrlMgr.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to start manager for hub controllers") + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + cancel() + + By("tearing down the test environment") + Expect(hubTestEnv.Stop()).Should(Succeed()) +})