update to latest capi; minor clean up; fix docker build (k3s-io#12)

gridscale · Oct 6, 2022 · 1138354 · 1138354
1 parent bf7f69e
commit 1138354
Show file tree

Hide file tree

Showing 29 changed files with 869 additions and 370 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,55 @@
-FROM gcr.io/distroless/static:nonroot
+# Build the manager binary
+# Run this with docker build --build-arg builder_image=<golang:x.y.z>
+ARG builder_image
+
+# Build architecture
+ARG ARCH
+
+# Ignore Hadolint rule "Always tag the version of an image explicitly."
+# It's an invalid finding since the image is explicitly set in the Makefile.
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# hadolint ignore=DL3006
+FROM ${builder_image} as builder
+WORKDIR /workspace
+
+# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+# Run this with docker build --build-arg package=./controlplane/kubeadm or --build-arg package=./bootstrap/kubeadm
+ENV GOPROXY=$goproxy
+
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+# Copy the sources
+COPY ./ ./
+
+# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go build ./bootstrap/main.go
+
+# Build
+ARG package=.
+ARG ARCH
+ARG ldflags
+
+# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+    go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
+    -o manager ${package}
+
+# Production image
+FROM gcr.io/distroless/static:nonroot-${ARCH}
 WORKDIR /
-COPY bin/manager ./
-USER nonroot:nonroot
+COPY --from=builder /workspace/manager .
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
+USER 65532
 ENTRYPOINT ["/manager"]
diff --git a/Makefile b/Makefile
@@ -1,9 +1,29 @@
+GO_VERSION ?= 1.19.0
+GO_CONTAINER_IMAGE ?= docker.io/library/golang:$(GO_VERSION)
+
+ARCH ?= $(shell go env GOARCH)
+
+# Use GOPROXY environment variable if set
+GOPROXY := $(shell go env GOPROXY)
+ifeq ($(GOPROXY),)
+GOPROXY := https://proxy.golang.org
+endif
+export GOPROXY
+
+# Active module mode, as we use go modules to manage dependencies
+export GO111MODULE=on
+
+GO_INSTALL := ./hack/go_install.sh
+
+BIN_DIR := bin
+TOOLS_BIN_DIR := $(abspath $(BIN_DIR))
+
 
 # Image URL to use all building/pushing image targets
-BOOTSTRAP_IMG ?= ghcr.io/zawachte/cluster-api-k3s/bootstrap-controller:v0.1.3
+BOOTSTRAP_IMG ?= ghcr.io/zawachte/cluster-api-k3s/bootstrap-controller:v0.1.4
 
 # Image URL to use all building/pushing image targets
-CONTROLPLANE_IMG ?= ghcr.io/zawachte/cluster-api-k3s/controlplane-controller:v0.1.3
+CONTROLPLANE_IMG ?= ghcr.io/zawachte/cluster-api-k3s/controlplane-controller:v0.1.4
 
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
@@ -16,10 +36,14 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+CONTROLLER_GEN_BIN = controller-gen
+CONTROLLER_GEN_PKG = "sigs.k8s.io/controller-tools/cmd/controller-gen"
+CONTROLLER_GEN_VER = "v0.8.0"
+CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER))
+
 .PHONY: controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
+	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONTROLLER_GEN_PKG) $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 .PHONY: kustomize
@@ -82,7 +106,7 @@ generate-bootstrap: controller-gen
 
 # Build the docker image
 docker-build-bootstrap: manager-bootstrap
-	docker build . -t ${BOOTSTRAP_IMG}
+	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}
 
 # Push the docker image
 docker-push-bootstrap:
@@ -130,22 +154,8 @@ generate-controlplane: controller-gen
 
 # Build the docker image
 docker-build-controlplane: manager-controlplane
-	docker build . -t ${CONTROLPLANE_IMG}
+	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}
 
 # Push the docker image
 docker-push-controlplane:
 	docker push ${CONTROLPLANE_IMG}
-
-# go-get-tool will 'go get' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
-@[ -f $(1) ] || { \
-set -e ;\
-TMP_DIR=$$(mktemp -d) ;\
-cd $$TMP_DIR ;\
-go mod init tmp ;\
-echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
-rm -rf $$TMP_DIR ;\
-}
-endef
diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: kthreesconfigs.bootstrap.cluster.x-k8s.io
 spec:

diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: kthreesconfigtemplates.bootstrap.cluster.x-k8s.io
 spec:

diff --git a/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: kthreescontrolplanes.controlplane.cluster.x-k8s.io
 spec:
@@ -17,8 +16,8 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - description: This denotes whether or not the control plane has the uploaded
-        kubeadm-config configmap
+    - description: This denotes whether or not the control plane has completed the
+        k3s server initialization
       jsonPath: .status.initialized
       name: Initialized
       type: boolean
@@ -302,7 +301,6 @@ spec:
                 type: string
             required:
             - infrastructureTemplate
-            - kthreesConfigSpec
             - version
             type: object
           status:
@@ -363,8 +361,8 @@ spec:
                   for programmatic interpretation.
                 type: string
               initialized:
-                description: Initialized denotes whether or not the control plane
-                  has the uploaded kubeadm-config configmap.
+                description: Initialized denotes whether or not the k3s server is
+                  up.
                 type: boolean
               observedGeneration:
                 description: ObservedGeneration is the latest generation observed

diff --git a/bootstrap/config/manager/kustomization.yaml b/bootstrap/config/manager/kustomization.yaml
@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/zawachte/cluster-api-k3s/bootstrap-controller
-  newTag: v0.1.3
+  newTag: v0.1.4
diff --git a/bootstrap/config/rbac/auth_proxy_client_clusterrole.yaml b/bootstrap/config/rbac/auth_proxy_client_clusterrole.yaml
@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: metrics-reader

diff --git a/bootstrap/config/rbac/role.yaml b/bootstrap/config/rbac/role.yaml
@@ -1,4 +1,3 @@
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/controlplane/api/v1beta1/condition_consts.go b/controlplane/api/v1beta1/condition_consts.go
@@ -37,12 +37,12 @@ const (
 )
 
 const (
-	// AvailableCondition documents that the first control plane instance has completed the kubeadm init operation
+	// AvailableCondition documents that the first control plane instance has completed the server init operation
 	// and so the control plane is available and an API server instance is ready for processing requests.
 	AvailableCondition clusterv1.ConditionType = "Available"
 
 	// WaitingForKthreesServerReason (Severity=Info) documents a KThreesControlPlane object waiting for the first
-	// control plane instance to complete the kubeadm init operation.
+	// control plane instance to complete the k3s server operation.
 	WaitingForKthreesServerReason = "WaitingForKthreesServer"
 )
 
@@ -68,9 +68,7 @@ const (
 )
 
 const (
-	// ControlPlaneComponentsHealthyCondition reports the overall status of control plane components
-	// implemented as static pods generated by kubeadm including kube-api-server, kube-controller manager,
-	// kube-scheduler and etcd if managed.
+	// ControlPlaneComponentsHealthyCondition reports the overall status of the k3s server.
 	ControlPlaneComponentsHealthyCondition clusterv1.ConditionType = "ControlPlaneComponentsHealthy"
 
 	// ControlPlaneComponentsUnhealthyReason (Severity=Error) documents a control plane component not healthy.

diff --git a/controlplane/api/v1beta1/kthreescontrolplane_types.go b/controlplane/api/v1beta1/kthreescontrolplane_types.go
@@ -101,8 +101,7 @@ type KThreesControlPlaneStatus struct {
 	// +optional
 	UnavailableReplicas int32 `json:"unavailableReplicas,omitempty"`
 
-	// Initialized denotes whether or not the control plane has the
-	// uploaded kubeadm-config configmap.
+	// Initialized denotes whether or not the k3s server is initialized.
 	// +optional
 	Initialized bool `json:"initialized"`
 
@@ -134,7 +133,7 @@ type KThreesControlPlaneStatus struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector
-// +kubebuilder:printcolumn:name="Initialized",type=boolean,JSONPath=".status.initialized",description="This denotes whether or not the control plane has the uploaded kubeadm-config configmap"
+// +kubebuilder:printcolumn:name="Initialized",type=boolean,JSONPath=".status.initialized",description="This denotes whether or not the control plane has completed the k3s server initialization"
 // +kubebuilder:printcolumn:name="API Server Available",type=boolean,JSONPath=".status.ready",description="KThreesControlPlane API Server is ready to receive requests"
 // +kubebuilder:printcolumn:name="Version",type=string,JSONPath=".spec.version",description="Kubernetes version associated with this control plane"
 // +kubebuilder:printcolumn:name="Replicas",type=integer,JSONPath=".status.replicas",description="Total number of non-terminated machines targeted by this control plane"

diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: kthreesconfigs.bootstrap.cluster.x-k8s.io
 spec:

diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: kthreesconfigtemplates.bootstrap.cluster.x-k8s.io
 spec:

diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml
@@ -1,10 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.8.0
   creationTimestamp: null
   name: kthreescontrolplanes.controlplane.cluster.x-k8s.io
 spec:
@@ -17,8 +16,8 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - description: This denotes whether or not the control plane has the uploaded
-        kubeadm-config configmap
+    - description: This denotes whether or not the control plane has completed the
+        k3s server initialization
       jsonPath: .status.initialized
       name: Initialized
       type: boolean
@@ -302,7 +301,6 @@ spec:
                 type: string
             required:
             - infrastructureTemplate
-            - kthreesConfigSpec
             - version
             type: object
           status:
@@ -363,8 +361,8 @@ spec:
                   for programmatic interpretation.
                 type: string
               initialized:
-                description: Initialized denotes whether or not the control plane
-                  has the uploaded kubeadm-config configmap.
+                description: Initialized denotes whether or not the k3s server is
+                  up.
                 type: boolean
               observedGeneration:
                 description: ObservedGeneration is the latest generation observed

diff --git a/controlplane/config/manager/kustomization.yaml b/controlplane/config/manager/kustomization.yaml
@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/zawachte/cluster-api-k3s/controlplane-controller
-  newTag: v0.1.3
+  newTag: v0.1.4
diff --git a/controlplane/config/rbac/auth_proxy_client_clusterrole.yaml b/controlplane/config/rbac/auth_proxy_client_clusterrole.yaml
@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: metrics-reader

diff --git a/controlplane/controllers/kthreescontrolplane_controller.go b/controlplane/controllers/kthreescontrolplane_controller.go
@@ -514,12 +514,6 @@ func (r *KThreesControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 		return ctrl.Result{Requeue: true}, nil
 	}
 
-	// Ensure kubeadm role bindings for v1.18+
-
-	if err := workloadCluster.AllowBootstrapTokensToGetNodes(ctx); err != nil {
-		return ctrl.Result{}, errors.Wrap(err, "failed to set role and role binding for kubeadm")
-	}
-
 	// Update kube-proxy daemonset.
 	if err := workloadCluster.UpdateKubeProxyImageInfo(ctx, kcp); err != nil {
 		logger.Error(err, "failed to update kube-proxy daemonset")
@@ -661,12 +655,6 @@ func (r *KThreesControlPlaneReconciler) upgradeControlPlane(
 		return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kcp.Spec.Version)
 	}
 
-	// Ensure kubeadm cluster role  & bindings for v1.18+
-	// as per https://github.com/kubernetes/kubernetes/commit/b117a928a6c3f650931bdac02a41fca6680548c4
-	//	if err := workloadCluster.AllowBootstrapTokensToGetNodes(ctx); err != nil {
-	//		return ctrl.Result{}, errors.Wrap(err, "failed to set role and role binding for kubeadm")
-	//	}
-
 
 	if kcp.Spec.KThreesConfigSpec.ClusterConfiguration != nil {
 		imageRepository := kcp.Spec.KThreesConfigSpec.ClusterConfiguration.ImageRepository