From 75a234d96ddcf465e7157c3aeef5a9e8c5844dba Mon Sep 17 00:00:00 2001 From: zach Date: Sun, 3 Jan 2021 22:16:54 -0500 Subject: [PATCH] control plane working --- Dockerfile | 4 +- Makefile | 6 +- PROJECT | 4 +- .../groupversion_info.go | 8 +- .../kthreesconfig_types.go | 3 +- .../zz_generated.deepcopy.go | 2 +- cluster.yaml | 29 +++ ...strap.cluster.x-k8s.io_kthreesconfigs.yaml | 81 ++++++++ .../bootstrap.k8s.io_kthreesconfigs.yaml | 8 +- config/crd/kustomization.yaml | 6 +- .../cainjection_in_kthreesconfigs.yaml | 2 +- .../patches/webhook_in_kthreesconfigs.yaml | 2 +- config/default/kustomization.yaml | 4 +- config/manager/kustomization.yaml | 2 +- config/manager/manager.yaml | 9 +- config/rbac/kthreesconfig_editor_role.yaml | 4 +- config/rbac/kthreesconfig_viewer_role.yaml | 4 +- config/rbac/role.yaml | 8 +- .../bootstrap_v1alpha1_kthreesconfig.yaml | 2 +- controllers/kthreesconfig_controller.go | 57 ++++-- controllers/suite_test.go | 4 +- go.mod | 1 + go.sum | 5 + hack/azure-ccm.yaml | 187 ++++++++++++++++++ hack/kind-setup.sh | 2 + machine.yaml | 52 +++++ main.go | 9 +- pkg/cloudinit/cloudinit.go | 3 + pkg/cloudinit/controlplane_init.go | 9 +- pkg/cloudinit/controlplane_init_test.go | 4 +- pkg/kubeconfig/kubeconfig.go | 173 ++++++++++++++++ pkg/secret/certificates.go | 16 +- pkg/secret/const.go | 3 + pkg/secret/secret.go | 71 +++++++ 34 files changed, 720 insertions(+), 64 deletions(-) rename api/{v1alpha1 => v1alpha3}/groupversion_info.go (80%) rename api/{v1alpha1 => v1alpha3}/kthreesconfig_types.go (98%) rename api/{v1alpha1 => v1alpha3}/zz_generated.deepcopy.go (99%) create mode 100644 cluster.yaml create mode 100644 config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml create mode 100644 hack/azure-ccm.yaml create mode 100644 hack/kind-setup.sh create mode 100644 machine.yaml create mode 100644 pkg/kubeconfig/kubeconfig.go create mode 100644 pkg/secret/secret.go diff --git a/Dockerfile b/Dockerfile index 4a98d398..a6508001 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -FROM gcr.io/distroless/static:latest +FROM gcr.io/distroless/static:nonroot WORKDIR / COPY bin/manager ./ -USER nobody +USER nonroot:nonroot ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 22ebaaf2..a18f51f6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Image URL to use all building/pushing image targets -IMG ?= ghcr.io/zawachte-msft/cluster-api-bootstrap-provider-k3s/controller:latest +IMG ?= ghcr.io/zawachte-msft/cluster-api-bootstrap-provider-k3s/controller:latest5 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" @@ -20,7 +20,7 @@ test: generate fmt vet manifests # Build manager binary manager: generate fmt vet - go build -o bin/manager main.go + CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o bin/manager main.go # Run against the configured Kubernetes cluster in ~/.kube/config run: generate fmt vet manifests @@ -60,7 +60,7 @@ generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." # Build the docker image -docker-build: #test +docker-build: manager docker build . -t ${IMG} # Push the docker image diff --git a/PROJECT b/PROJECT index 1e3a7410..e5bfa8fa 100644 --- a/PROJECT +++ b/PROJECT @@ -1,7 +1,7 @@ -domain: k8s.io +domain: cluster.x-k8s.io repo: github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s resources: - group: bootstrap kind: KThreesConfig - version: v1alpha1 + version: v1alpha3 version: "2" diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha3/groupversion_info.go similarity index 80% rename from api/v1alpha1/groupversion_info.go rename to api/v1alpha3/groupversion_info.go index 053f436a..9f028003 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha3/groupversion_info.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package v1alpha1 contains API Schema definitions for the bootstrap v1alpha1 API group +// Package v1alpha3 contains API Schema definitions for the bootstrap v1alpha3 API group // +kubebuilder:object:generate=true -// +groupName=bootstrap.k8s.io -package v1alpha1 +// +groupName=bootstrap.cluster.x-k8s.io +package v1alpha3 import ( "k8s.io/apimachinery/pkg/runtime/schema" @@ -26,7 +26,7 @@ import ( var ( // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "bootstrap.k8s.io", Version: "v1alpha1"} + GroupVersion = schema.GroupVersion{Group: "bootstrap.cluster.x-k8s.io", Version: "v1alpha3"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} diff --git a/api/v1alpha1/kthreesconfig_types.go b/api/v1alpha3/kthreesconfig_types.go similarity index 98% rename from api/v1alpha1/kthreesconfig_types.go rename to api/v1alpha3/kthreesconfig_types.go index 23ae8652..c24183ce 100644 --- a/api/v1alpha1/kthreesconfig_types.go +++ b/api/v1alpha3/kthreesconfig_types.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1alpha3 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -61,6 +61,7 @@ type KThreesConfigStatus struct { } // +kubebuilder:object:root=true +// +kubebuilder:subresource:status // KThreesConfig is the Schema for the kthreesconfigs API type KThreesConfig struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go similarity index 99% rename from api/v1alpha1/zz_generated.deepcopy.go rename to api/v1alpha3/zz_generated.deepcopy.go index 17582744..68c2e2e2 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -18,7 +18,7 @@ limitations under the License. // Code generated by controller-gen. DO NOT EDIT. -package v1alpha1 +package v1alpha3 import ( runtime "k8s.io/apimachinery/pkg/runtime" diff --git a/cluster.yaml b/cluster.yaml new file mode 100644 index 00000000..2b237073 --- /dev/null +++ b/cluster.yaml @@ -0,0 +1,29 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: calico + name: joe-test + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: joe-test +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: joe-test + namespace: default +spec: + location: eastus + networkSpec: + vnet: + name: joe-test-vnet + resourceGroup: joe-test + subscriptionID: bbda620e-ad6d-4440-8ba0-ba1297702503 diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml new file mode 100644 index 00000000..e0d65459 --- /dev/null +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml @@ -0,0 +1,81 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: kthreesconfigs.bootstrap.cluster.x-k8s.io +spec: + group: bootstrap.cluster.x-k8s.io + names: + kind: KThreesConfig + listKind: KThreesConfigList + plural: kthreesconfigs + singular: kthreesconfig + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: KThreesConfig is the Schema for the kthreesconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KThreesConfigSpec defines the desired state of KThreesConfig + properties: + foo: + description: Foo is an example field of KThreesConfig. Edit KThreesConfig_types.go + to remove/update + type: string + type: object + status: + description: KThreesConfigStatus defines the observed state of KThreesConfig + properties: + bootstrapData: + format: byte + type: string + dataSecretName: + description: DataSecretName is the name of the secret that stores the + bootstrap data script. + type: string + failureMessage: + description: FailureMessage will be set on non-retryable errors + type: string + failureReason: + description: FailureReason will be set on non-retryable errors + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed by + the controller. + format: int64 + type: integer + ready: + description: Ready indicates the BootstrapData field is ready to be + consumed + type: boolean + type: object + type: object + version: v1alpha3 + versions: + - name: v1alpha3 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/bootstrap.k8s.io_kthreesconfigs.yaml b/config/crd/bases/bootstrap.k8s.io_kthreesconfigs.yaml index a116ff7b..2988a1f0 100644 --- a/config/crd/bases/bootstrap.k8s.io_kthreesconfigs.yaml +++ b/config/crd/bases/bootstrap.k8s.io_kthreesconfigs.yaml @@ -6,9 +6,9 @@ metadata: annotations: controller-gen.kubebuilder.io/version: v0.2.5 creationTimestamp: null - name: kthreesconfigs.bootstrap.k8s.io + name: kthreesconfigs.bootstrap.cluster.x-k8s.io spec: - group: bootstrap.k8s.io + group: bootstrap.cluster.x-k8s.io names: kind: KThreesConfig listKind: KThreesConfigList @@ -66,9 +66,9 @@ spec: type: boolean type: object type: object - version: v1alpha1 + version: v1alpha3 versions: - - name: v1alpha1 + - name: v1alpha3 served: true storage: true status: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 75d38e57..52e89b84 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -1,8 +1,12 @@ +commonLabels: + cluster.x-k8s.io/v1alpha2: v1alpha2 + cluster.x-k8s.io/v1alpha3: v1alpha3 + # This kustomization.yaml is not intended to be run by itself, # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- bases/bootstrap.k8s.io_kthreesconfigs.yaml +- bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/crd/patches/cainjection_in_kthreesconfigs.yaml b/config/crd/patches/cainjection_in_kthreesconfigs.yaml index 2bc6db34..eaa093f6 100644 --- a/config/crd/patches/cainjection_in_kthreesconfigs.yaml +++ b/config/crd/patches/cainjection_in_kthreesconfigs.yaml @@ -5,4 +5,4 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kthreesconfigs.bootstrap.k8s.io + name: kthreesconfigs.bootstrap.cluster.x-k8s.io diff --git a/config/crd/patches/webhook_in_kthreesconfigs.yaml b/config/crd/patches/webhook_in_kthreesconfigs.yaml index f1e48a72..7ddd71bf 100644 --- a/config/crd/patches/webhook_in_kthreesconfigs.yaml +++ b/config/crd/patches/webhook_in_kthreesconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: kthreesconfigs.bootstrap.k8s.io + name: kthreesconfigs.bootstrap.cluster.x-k8s.io spec: conversion: strategy: Webhook diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 738d8c12..f7e95f97 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,12 +1,12 @@ # Adds namespace to all resources. -namespace: cluster-api-bootstrap-provider-k3s-system +namespace: capi-k3s-bootstrap-system # Value of this field is prepended to the # names of all resources, e.g. a deployment named # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. -namePrefix: cluster-api-bootstrap-provider-k3s- +namePrefix: capi-k3s-bootstrap- # Labels to add to all resources and selectors. #commonLabels: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index d2ba6097..ccd47cb7 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/zawachte-msft/cluster-api-bootstrap-provider-k3s/controller - newTag: latest + newTag: latest5 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b6c85a52..bc39ea11 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -22,6 +22,8 @@ spec: labels: control-plane: controller-manager spec: + imagePullSecrets: + - name: regcred containers: - command: - /manager @@ -29,11 +31,4 @@ spec: - --enable-leader-election image: controller:latest name: manager - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi terminationGracePeriodSeconds: 10 diff --git a/config/rbac/kthreesconfig_editor_role.yaml b/config/rbac/kthreesconfig_editor_role.yaml index 62047ead..b004d117 100644 --- a/config/rbac/kthreesconfig_editor_role.yaml +++ b/config/rbac/kthreesconfig_editor_role.yaml @@ -5,7 +5,7 @@ metadata: name: kthreesconfig-editor-role rules: - apiGroups: - - bootstrap.k8s.io + - bootstrap.cluster.x-k8s.io resources: - kthreesconfigs verbs: @@ -17,7 +17,7 @@ rules: - update - watch - apiGroups: - - bootstrap.k8s.io + - bootstrap.cluster.x-k8s.io resources: - kthreesconfigs/status verbs: diff --git a/config/rbac/kthreesconfig_viewer_role.yaml b/config/rbac/kthreesconfig_viewer_role.yaml index 789df9dc..ba853d91 100644 --- a/config/rbac/kthreesconfig_viewer_role.yaml +++ b/config/rbac/kthreesconfig_viewer_role.yaml @@ -5,7 +5,7 @@ metadata: name: kthreesconfig-viewer-role rules: - apiGroups: - - bootstrap.k8s.io + - bootstrap.cluster.x-k8s.io resources: - kthreesconfigs verbs: @@ -13,7 +13,7 @@ rules: - list - watch - apiGroups: - - bootstrap.k8s.io + - bootstrap.cluster.x-k8s.io resources: - kthreesconfigs/status verbs: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 46e522f4..fdb28373 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -21,7 +21,7 @@ rules: - update - watch - apiGroups: - - bootstrap.k8s.io + - bootstrap.cluster.x-k8s.io resources: - kthreesconfigs verbs: @@ -33,13 +33,17 @@ rules: - update - watch - apiGroups: - - bootstrap.k8s.io + - bootstrap.cluster.x-k8s.io resources: - kthreesconfigs/status verbs: + - create + - delete - get + - list - patch - update + - watch - apiGroups: - cluster.x-k8s.io resources: diff --git a/config/samples/bootstrap_v1alpha1_kthreesconfig.yaml b/config/samples/bootstrap_v1alpha1_kthreesconfig.yaml index 61392b75..27fb2e2a 100644 --- a/config/samples/bootstrap_v1alpha1_kthreesconfig.yaml +++ b/config/samples/bootstrap_v1alpha1_kthreesconfig.yaml @@ -1,4 +1,4 @@ -apiVersion: bootstrap.k8s.io/v1alpha1 +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: KThreesConfig metadata: name: kthreesconfig-sample diff --git a/controllers/kthreesconfig_controller.go b/controllers/kthreesconfig_controller.go index 33e76fbc..20cb20e9 100644 --- a/controllers/kthreesconfig_controller.go +++ b/controllers/kthreesconfig_controller.go @@ -28,10 +28,10 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" + "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/pkg/kubeconfig" "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/pkg/secret" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" bsutil "sigs.k8s.io/cluster-api/bootstrap/util" @@ -42,7 +42,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - bootstrapv1alpha1 "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/api/v1alpha1" + bootstrapv1alpha3 "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/api/v1alpha3" "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/pkg/cloudinit" ) @@ -62,13 +62,13 @@ type KThreesConfigReconciler struct { type Scope struct { logr.Logger - Config *bootstrapv1alpha1.KThreesConfig + Config *bootstrapv1alpha3.KThreesConfig ConfigOwner *bsutil.ConfigOwner Cluster *clusterv1.Cluster } -// +kubebuilder:rbac:groups=bootstrap.k8s.io,resources=kthreesconfigs,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=bootstrap.k8s.io,resources=kthreesconfigs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kthreesconfigs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kthreesconfigs/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status,verbs=get;list;watch // +kubebuilder:rbac:groups=exp.cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete @@ -78,9 +78,10 @@ func (r *KThreesConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re log := r.Log.WithValues("kthreesconfig", req.NamespacedName) // Lookup the kubeadm config - config := &bootstrapv1alpha1.KThreesConfig{} + config := &bootstrapv1alpha3.KThreesConfig{} if err := r.Client.Get(ctx, req.NamespacedName, config); err != nil { if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil } log.Error(err, "Failed to get config") @@ -100,6 +101,7 @@ func (r *KThreesConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re if configOwner == nil { return ctrl.Result{}, nil } + log = log.WithValues("kind", configOwner.GetKind(), "version", configOwner.GetResourceVersion(), "name", configOwner.GetName()) // Lookup the cluster the config owner is associated with @@ -192,6 +194,7 @@ func (r *KThreesConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re } } **/ + // In any other case just return as the config is already generated and need not be generated again. return ctrl.Result{}, nil } @@ -237,6 +240,7 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex // if it's NOT a control plane machine, requeue if !scope.ConfigOwner.IsControlPlaneMachine() { + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } @@ -248,6 +252,7 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex // acquire the init lock so that only the first machine configured // as control plane get processed here // if not the first, requeue + /** if !r.KThreesInitLock.Lock(ctx, scope.Cluster, machine) { scope.Info("A control plane is already being initialized, requeing until control plane is ready") return ctrl.Result{RequeueAfter: 30 * time.Second}, nil @@ -260,6 +265,7 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex } } }() + **/ scope.Info("Creating BootstrapData for the init control plane") @@ -268,7 +274,7 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex ctx, r.Client, util.ObjectKey(scope.Cluster), - *metav1.NewControllerRef(scope.Config, bootstrapv1alpha1.GroupVersion.WithKind("KThreesConfig")), + *metav1.NewControllerRef(scope.Config, bootstrapv1alpha3.GroupVersion.WithKind("KThreesConfig")), ) if err != nil { // conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) @@ -279,10 +285,11 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex cpinput := &cloudinit.ControlPlaneInput{ BaseUserData: cloudinit.BaseUserData{ PreK3sCommands: nil, - PostK3sCommands: nil, + PostK3sCommands: []string{cloudinit.SleepCommand, cloudinit.PatchCommand}, AdditionalFiles: []infrav1.File{}, }, - Certificates: certificates, + ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host, + Certificates: certificates, } cloudInitData, err := cloudinit.NewInitControlPlane(cpinput) @@ -295,12 +302,17 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex return ctrl.Result{}, err } - return ctrl.Result{}, nil + return r.reconcileKubeconfig(ctx, scope) } func (r *KThreesConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + + // if r.KubeadmInitLock == nil { + // r.KubeadmInitLock = locking.NewControlPlaneInitMutex(ctrl.Log.WithName("init-locker"), mgr.GetClient()) + // } + return ctrl.NewControllerManagedBy(mgr). - For(&bootstrapv1alpha1.KThreesConfig{}). + For(&bootstrapv1alpha3.KThreesConfig{}). Complete(r) } @@ -316,7 +328,7 @@ func (r *KThreesConfigReconciler) storeBootstrapData(ctx context.Context, scope }, OwnerReferences: []metav1.OwnerReference{ { - APIVersion: bootstrapv1alpha1.GroupVersion.String(), + APIVersion: bootstrapv1alpha3.GroupVersion.String(), Kind: "KThreesConfig", Name: scope.Config.Name, UID: scope.Config.UID, @@ -341,8 +353,29 @@ func (r *KThreesConfigReconciler) storeBootstrapData(ctx context.Context, scope return errors.Wrapf(err, "failed to update bootstrap data secret for KubeadmConfig %s/%s", scope.Config.Namespace, scope.Config.Name) } } + scope.Config.Status.DataSecretName = pointer.StringPtr(secret.Name) scope.Config.Status.Ready = true // conditions.MarkTrue(scope.Config, bootstrapv1.DataSecretAvailableCondition) return nil } + +func (r *KThreesConfigReconciler) reconcileKubeconfig(ctx context.Context, scope *Scope) (ctrl.Result, error) { + logger := r.Log.WithValues("cluster", scope.Cluster.Name, "namespace", scope.Cluster.Namespace) + + _, err := secret.Get(ctx, r.Client, util.ObjectKey(scope.Cluster), secret.Kubeconfig) + switch { + case apierrors.IsNotFound(errors.Cause(err)): + if err := kubeconfig.CreateSecret(ctx, r.Client, scope.Cluster); err != nil { + if err == kubeconfig.ErrDependentCertificateNotFound { + logger.Info("could not find secret for cluster, requeuing", "secret", secret.ClusterCA) + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + return ctrl.Result{}, err + } + case err != nil: + return ctrl.Result{}, errors.Wrapf(err, "failed to retrieve Kubeconfig Secret for Cluster %q in namespace %q", scope.Cluster.Name, scope.Cluster.Namespace) + } + + return ctrl.Result{}, nil +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 14ce0129..be3e74e8 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -30,7 +30,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - bootstrapv1alpha1 "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/api/v1alpha1" + bootstrapv1alpha3 "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/api/v1alpha3" // +kubebuilder:scaffold:imports ) @@ -62,7 +62,7 @@ var _ = BeforeSuite(func(done Done) { Expect(err).ToNot(HaveOccurred()) Expect(cfg).ToNot(BeNil()) - err = bootstrapv1alpha1.AddToScheme(scheme.Scheme) + err = bootstrapv1alpha3.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme diff --git a/go.mod b/go.mod index e132e7e4..ec85fdbc 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/onsi/ginkgo v1.12.1 github.com/onsi/gomega v1.10.1 github.com/pkg/errors v0.9.1 + github.com/prometheus/common v0.9.1 k8s.io/api v0.17.9 k8s.io/apimachinery v0.17.9 k8s.io/client-go v0.17.9 diff --git a/go.sum b/go.sum index c77e467b..4e4e1598 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -364,6 +366,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -562,6 +565,7 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= @@ -606,6 +610,7 @@ k8s.io/apimachinery v0.17.2 h1:hwDQQFbdRlpnnsR64Asdi55GyCaIP/3WQpMmbNBeWr4= k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.9 h1:knQxNgMu57Oxlm12J6DS375kmGMeuWV0VNzRRUBB2Yk= k8s.io/apimachinery v0.17.9/go.mod h1:Lg8zZ5iC/O8UjCqW6DNhcQG2m4TdjF9kwG3891OWbbA= +k8s.io/apimachinery v0.20.1 h1:LAhz8pKbgR8tUwn7boK+b2HZdt7MiTu2mkYtFMUjTRQ= k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= k8s.io/apiserver v0.17.9 h1:q50QEJ51xdHy2Gl1lo9yJexiyixxof/yDUFdWNnZxh0= k8s.io/apiserver v0.17.9/go.mod h1:Qaxd3EbeoPRBHVMtFyuKNAObqP6VAkzIMyWYz8KuE2k= diff --git a/hack/azure-ccm.yaml b/hack/azure-ccm.yaml new file mode 100644 index 00000000..5a9faba4 --- /dev/null +++ b/hack/azure-ccm.yaml @@ -0,0 +1,187 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + k8s-app: cloud-controller-manager +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager +--- +apiVersion: v1 +kind: Pod +metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + tier: control-plane + component: cloud-controller-manager +spec: + priorityClassName: system-node-critical + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: cloud-controller-manager + image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v0.5.0 + imagePullPolicy: IfNotPresent + command: ["cloud-controller-manager"] + args: + - --allocate-node-cidrs=false + - --cloud-config=/etc/kubernetes/azure.json + - --cloud-provider=azure + - --cluster-cidr=192.168.0.0/16 + - --configure-cloud-routes=true + - --controllers=* + - --leader-elect=true + - --route-reconciliation-period=10s + - --v=2 + - --profiling=false + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "4" + memory: 2Gi + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl \ No newline at end of file diff --git a/hack/kind-setup.sh b/hack/kind-setup.sh new file mode 100644 index 00000000..0902e8df --- /dev/null +++ b/hack/kind-setup.sh @@ -0,0 +1,2 @@ +export KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge +kind create cluster \ No newline at end of file diff --git a/machine.yaml b/machine.yaml new file mode 100644 index 00000000..82439a71 --- /dev/null +++ b/machine.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachine +metadata: + name: joe-test-control-plane-0 + namespace: default +spec: + location: eastus + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: "" + vmSize: Standard_D2s_v3 + image: + marketplace: + publisher: Canonical + sku: 18.04-LTS + offer: UbuntuServer + version: latest +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KThreesConfig +metadata: + name: joe-test-control-plane-0 + namespace: default +spec: + # Add fields here + foo: asdfasdsa +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Machine +metadata: + labels: + cluster.x-k8s.io/cluster-name: joe-test + cluster.x-k8s.io/control-plane: "" + name: joe-test-control-plane-0 + namespace: default +spec: + version: v1.18.2 + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KThreesConfig + name: joe-test-control-plane-0 + clusterName: joe-test + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachine + name: joe-test-control-plane-0 + \ No newline at end of file diff --git a/main.go b/main.go index 019909c0..d8783132 100644 --- a/main.go +++ b/main.go @@ -26,8 +26,10 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" - bootstrapv1alpha1 "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/api/v1alpha1" + bootstrapv1alpha3 "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/api/v1alpha3" "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/controllers" + clusterv1alpha3 "sigs.k8s.io/cluster-api/api/v1alpha3" + expv1alpha3 "sigs.k8s.io/cluster-api/exp/api/v1alpha3" // +kubebuilder:scaffold:imports ) @@ -38,8 +40,9 @@ var ( func init() { _ = clientgoscheme.AddToScheme(scheme) - - _ = bootstrapv1alpha1.AddToScheme(scheme) + _ = clusterv1alpha3.AddToScheme(scheme) + _ = expv1alpha3.AddToScheme(scheme) + _ = bootstrapv1alpha3.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } diff --git a/pkg/cloudinit/cloudinit.go b/pkg/cloudinit/cloudinit.go index dcdefcf0..bb49c571 100644 --- a/pkg/cloudinit/cloudinit.go +++ b/pkg/cloudinit/cloudinit.go @@ -69,6 +69,9 @@ write_files:{{ range . }} {{- end -}} {{- end -}} ` + + PatchCommand = `kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml patch node joe-test-control-plane-0 -p $'spec:\n providerID: azure:////subscriptions/bbda620e-ad6d-4440-8ba0-ba1297702503/resourceGroups/joe-test/providers/Microsoft.Compute/virtualMachines/joe-test-control-plane-0'` + SleepCommand = "sleep 30" ) // BaseUserData is shared across all the various types of files written to disk. diff --git a/pkg/cloudinit/controlplane_init.go b/pkg/cloudinit/controlplane_init.go index 193effda..390ff2de 100644 --- a/pkg/cloudinit/controlplane_init.go +++ b/pkg/cloudinit/controlplane_init.go @@ -17,6 +17,8 @@ limitations under the License. package cloudinit import ( + "fmt" + "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/pkg/secret" ) @@ -25,7 +27,7 @@ const ( {{template "files" .WriteFiles}} runcmd: {{- template "commands" .PreK3sCommands }} - - 'curl -sfL https://get.k3s.io | sh -s - --disable-cloud-controller' + - 'curl -sfL https://get.k3s.io | sh -s - --disable-cloud-controller --kube-apiserver-arg anonymous-auth=true --tls-san "%s"' {{- template "commands" .PostK3sCommands }} ` ) @@ -34,14 +36,17 @@ runcmd: type ControlPlaneInput struct { BaseUserData secret.Certificates + ControlPlaneEndpoint string } // NewInitControlPlane returns the user data string to be used on a controlplane instance. func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) { input.Header = cloudConfigHeader input.WriteFiles = input.Certificates.AsFiles() + + withEndpoint := fmt.Sprintf(controlPlaneCloudInit, input.ControlPlaneEndpoint) input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - userData, err := generate("InitControlplane", controlPlaneCloudInit, input) + userData, err := generate("InitControlplane", withEndpoint, input) if err != nil { return nil, err } diff --git a/pkg/cloudinit/controlplane_init_test.go b/pkg/cloudinit/controlplane_init_test.go index fa7c8194..d499fbf5 100644 --- a/pkg/cloudinit/controlplane_init_test.go +++ b/pkg/cloudinit/controlplane_init_test.go @@ -44,8 +44,8 @@ func TestControlPlaneInit(t *testing.T) { }, }, }, - - Certificates: secret.Certificates{}, + ControlPlaneEndpoint: "zach.local", + Certificates: secret.Certificates{}, } out, err := NewInitControlPlane(cpinput) diff --git a/pkg/kubeconfig/kubeconfig.go b/pkg/kubeconfig/kubeconfig.go new file mode 100644 index 00000000..d993e730 --- /dev/null +++ b/pkg/kubeconfig/kubeconfig.go @@ -0,0 +1,173 @@ +package kubeconfig + +import ( + "context" + "crypto" + "crypto/x509" + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/certs" + + "github.com/zawachte-msft/cluster-api-bootstrap-provider-k3s/pkg/secret" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + ErrDependentCertificateNotFound = errors.New("could not find secret ca") +) + +func generateKubeconfig(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string) ([]byte, error) { + clusterCA, err := secret.GetFromNamespacedName(ctx, c, clusterName, secret.ClusterCA) + if err != nil { + if apierrors.IsNotFound(errors.Cause(err)) { + return nil, ErrDependentCertificateNotFound + } + return nil, err + } + + clientClusterCA, err := secret.GetFromNamespacedName(ctx, c, clusterName, secret.ClientClusterCA) + if err != nil { + if apierrors.IsNotFound(errors.Cause(err)) { + return nil, ErrDependentCertificateNotFound + } + return nil, err + } + + clientCACert, err := certs.DecodeCertPEM(clientClusterCA.Data[secret.TLSCrtDataName]) + if err != nil { + return nil, errors.Wrap(err, "failed to decode CA Cert") + } else if clientCACert == nil { + return nil, errors.New("certificate not found in config") + } + + clientCAKey, err := certs.DecodePrivateKeyPEM(clientClusterCA.Data[secret.TLSKeyDataName]) + if err != nil { + return nil, errors.Wrap(err, "failed to decode private key") + } else if clientCAKey == nil { + return nil, errors.New("CA private key not found") + } + + serverCACert, err := certs.DecodeCertPEM(clusterCA.Data[secret.TLSCrtDataName]) + if err != nil { + return nil, errors.Wrap(err, "failed to decode CA Cert") + } else if serverCACert == nil { + return nil, errors.New("certificate not found in config") + } + + cfg, err := New(clusterName.Name, endpoint, clientCACert, clientCAKey, serverCACert) + if err != nil { + return nil, errors.Wrap(err, "failed to generate a kubeconfig") + } + + out, err := clientcmd.Write(*cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to serialize config to yaml") + } + return out, nil +} + +// New creates a new Kubeconfig using the cluster name and specified endpoint. +func New(clusterName, endpoint string, clientCACert *x509.Certificate, clientCAKey crypto.Signer, serverCACert *x509.Certificate) (*api.Config, error) { + + cfg := &certs.Config{ + CommonName: "kubernetes-admin", + Organization: []string{"system:masters"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + + clientKey, err := certs.NewPrivateKey() + if err != nil { + return nil, errors.Wrap(err, "unable to create private key") + } + + clientCert, err := cfg.NewSignedCert(clientKey, clientCACert, clientCAKey) + if err != nil { + return nil, errors.Wrap(err, "unable to sign certificate") + } + + userName := fmt.Sprintf("%s-admin", clusterName) + contextName := fmt.Sprintf("%s@%s", userName, clusterName) + + return &api.Config{ + Clusters: map[string]*api.Cluster{ + clusterName: { + Server: endpoint, + CertificateAuthorityData: certs.EncodeCertPEM(serverCACert), + }, + }, + Contexts: map[string]*api.Context{ + contextName: { + Cluster: clusterName, + AuthInfo: userName, + }, + }, + AuthInfos: map[string]*api.AuthInfo{ + userName: { + ClientKeyData: certs.EncodePrivateKeyPEM(clientKey), + ClientCertificateData: certs.EncodeCertPEM(clientCert), + }, + }, + CurrentContext: contextName, + }, nil +} + +// CreateSecret creates the Kubeconfig secret for the given cluster. +func CreateSecret(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) error { + name := util.ObjectKey(cluster) + return CreateSecretWithOwner(ctx, c, name, cluster.Spec.ControlPlaneEndpoint.String(), metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + }) +} + +// CreateSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. +func CreateSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { + server := fmt.Sprintf("https://%s", endpoint) + out, err := generateKubeconfig(ctx, c, clusterName, server) + if err != nil { + return err + } + + return c.Create(ctx, GenerateSecretWithOwner(clusterName, out, owner)) +} + +// GenerateSecret returns a Kubernetes secret for the given Cluster and kubeconfig data. +func GenerateSecret(cluster *clusterv1.Cluster, data []byte) *corev1.Secret { + name := util.ObjectKey(cluster) + return GenerateSecretWithOwner(name, data, metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "Cluster", + Name: cluster.Name, + UID: cluster.UID, + }) +} + +// GenerateSecretWithOwner returns a Kubernetes secret for the given Cluster name, namespace, kubeconfig data, and ownerReference. +func GenerateSecretWithOwner(clusterName client.ObjectKey, data []byte, owner metav1.OwnerReference) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name(clusterName.Name, secret.Kubeconfig), + Namespace: clusterName.Namespace, + Labels: map[string]string{ + clusterv1.ClusterLabelName: clusterName.Name, + }, + OwnerReferences: []metav1.OwnerReference{ + owner, + }, + }, + Data: map[string][]byte{ + secret.KubeconfigDataName: data, + }, + } +} diff --git a/pkg/secret/certificates.go b/pkg/secret/certificates.go index e52f50eb..0de50801 100644 --- a/pkg/secret/certificates.go +++ b/pkg/secret/certificates.go @@ -24,7 +24,6 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/hex" - "fmt" "math/big" "path/filepath" "strings" @@ -71,6 +70,11 @@ func NewCertificatesForInitialControlPlane() Certificates { CertFile: filepath.Join(certificatesDir, "server-ca.crt"), KeyFile: filepath.Join(certificatesDir, "server-ca.key"), }, + &Certificate{ + Purpose: ClientClusterCA, + CertFile: filepath.Join(certificatesDir, "client-ca.crt"), + KeyFile: filepath.Join(certificatesDir, "client-ca.key"), + }, } return certificates @@ -87,11 +91,6 @@ func (c Certificates) GetByPurpose(purpose Purpose) *Certificate { return nil } -// Name returns the name of the secret for a cluster. -func Name(cluster string, suffix Purpose) string { - return fmt.Sprintf("%s-%s", cluster, suffix) -} - // Lookup looks up each certificate from secrets and populates the certificate with the secret data. func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey) error { // Look up each certificate as a secret and populate the certificate/key @@ -282,6 +281,8 @@ func (c *Certificate) Generate() error { // AsFiles converts a slice of certificates into bootstrap files. func (c Certificates) AsFiles() []bootstrapv1.File { clusterCA := c.GetByPurpose(ClusterCA) + clientClusterCA := c.GetByPurpose(ClientClusterCA) + etcdCA := c.GetByPurpose(EtcdCA) frontProxyCA := c.GetByPurpose(FrontProxyCA) serviceAccountKey := c.GetByPurpose(ServiceAccount) @@ -290,6 +291,9 @@ func (c Certificates) AsFiles() []bootstrapv1.File { if clusterCA != nil { certFiles = append(certFiles, clusterCA.AsFiles()...) } + if clientClusterCA != nil { + certFiles = append(certFiles, clientClusterCA.AsFiles()...) + } if etcdCA != nil { certFiles = append(certFiles, etcdCA.AsFiles()...) } diff --git a/pkg/secret/const.go b/pkg/secret/const.go index 98cbf6aa..30fc6797 100644 --- a/pkg/secret/const.go +++ b/pkg/secret/const.go @@ -35,6 +35,9 @@ const ( // ClusterCA is the secret name suffix for APIServer CA. ClusterCA = Purpose("ca") + // ClientClusterCA is the secret name suffix for APIServer CA. + ClientClusterCA = Purpose("cca") + // EtcdCA is the secret name suffix for the Etcd CA EtcdCA Purpose = "etcd" diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go new file mode 100644 index 00000000..5af9fd9f --- /dev/null +++ b/pkg/secret/secret.go @@ -0,0 +1,71 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secret + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Get retrieves the specified Secret (if any) from the given +// cluster name and namespace. +func Get(ctx context.Context, c client.Reader, cluster client.ObjectKey, purpose Purpose) (*corev1.Secret, error) { + return GetFromNamespacedName(ctx, c, cluster, purpose) +} + +// GetFromNamespacedName retrieves the specified Secret (if any) from the given +// cluster name and namespace. +func GetFromNamespacedName(ctx context.Context, c client.Reader, clusterName client.ObjectKey, purpose Purpose) (*corev1.Secret, error) { + secret := &corev1.Secret{} + secretKey := client.ObjectKey{ + Namespace: clusterName.Namespace, + Name: Name(clusterName.Name, purpose), + } + + if err := c.Get(ctx, secretKey, secret); err != nil { + return nil, err + } + + return secret, nil +} + +// Name returns the name of the secret for a cluster. +func Name(cluster string, suffix Purpose) string { + return fmt.Sprintf("%s-%s", cluster, suffix) +} + +// ParseSecretName return the cluster name and the suffix Purpose in name is a valid cluster secrets, +// otherwise it return error. +func ParseSecretName(name string) (string, Purpose, error) { + separatorPos := strings.LastIndex(name, "-") + if separatorPos == -1 { + return "", "", errors.Errorf("%q is not a valid cluster secret name. The purpose suffix is missing", name) + } + clusterName := name[:separatorPos] + purposeSuffix := Purpose(name[separatorPos+1:]) + for _, purpose := range allSecretPurposes { + if purpose == purposeSuffix { + return clusterName, purposeSuffix, nil + } + } + return "", "", errors.Errorf("%q is not a valid cluster secret name. Invalid purpose suffix", name) +}