From b139b0a2b657faa04a1bd16c869c35603974f0f8 Mon Sep 17 00:00:00 2001 From: Andrei Ilas Date: Thu, 24 Oct 2024 08:05:39 +0300 Subject: [PATCH] add cluster addon support --- docs/src/SUMMARY.md | 1 + docs/src/guide/cluster_addons.md | 19 +++++ .../guide/extensions_cluster_autoscaler.md | 4 + .../vars-cluster-addons.auto.tfvars | 21 +++++ module-cluster-addons.tf | 28 +++++++ modules/cluster-addons/addons.tf | 83 +++++++++++++++++++ modules/cluster-addons/delete_addons.tf | 49 +++++++++++ modules/cluster-addons/outputs.tf | 6 ++ modules/cluster-addons/variables.tf | 17 ++++ modules/cluster-addons/versions.tf | 13 +++ variables-cluster-addons.tf | 14 ++++ 11 files changed, 255 insertions(+) create mode 100644 docs/src/guide/cluster_addons.md create mode 100644 examples/cluster-addons/vars-cluster-addons.auto.tfvars create mode 100644 module-cluster-addons.tf create mode 100644 modules/cluster-addons/addons.tf create mode 100644 modules/cluster-addons/delete_addons.tf create mode 100644 modules/cluster-addons/outputs.tf create mode 100644 modules/cluster-addons/variables.tf create mode 100644 modules/cluster-addons/versions.tf create mode 100644 variables-cluster-addons.tf diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 33777cb6..4d2be362 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -12,6 +12,7 @@ - [Subnets](./guide/network_subnets.md) - [Network Security Groups](./guide/network_nsgs.md) - [Cluster](./guide/cluster.md) + - [Cluster Add-ons](./guide/cluster_addons.md) - [Workers](./guide/workers.md) - [Mode](./guide/workers_mode.md) - [Node Pool](./guide/workers_mode_nodepool.md) diff --git a/docs/src/guide/cluster_addons.md b/docs/src/guide/cluster_addons.md new file mode 100644 index 00000000..3d093827 --- /dev/null +++ b/docs/src/guide/cluster_addons.md @@ -0,0 +1,19 @@ +# Cluster Add-ons + +With this module to manage both essential and optional add-ons on enhanced OKE clusters. + +This module provides the option to remove [Essential addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-essential-addons) and to manage, both essential & [optional addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-optional-addons). + +Cluster add-on removal (using the `cluster_addons_to_remove` variable) requires the creation of the operator host. + +**Note**: For the cluster autoscaler you should choose **only one** of the options: +- the stand-alone cluster-autoscaler deployment, using the [extension module](./extensions_cluster_autoscaler.md) +- the cluster-autoscaler add-on + +## Example usage +```javascript +{{#include ../../../examples/cluster-addons/vars-cluster-addons.auto.tfvars:4:}} +``` + +## Reference +* [OKE Cluster Add-ons Documentation](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons.htm) diff --git a/docs/src/guide/extensions_cluster_autoscaler.md b/docs/src/guide/extensions_cluster_autoscaler.md index 21c563c5..ec43af21 100644 --- a/docs/src/guide/extensions_cluster_autoscaler.md +++ b/docs/src/guide/extensions_cluster_autoscaler.md @@ -1,5 +1,9 @@ # Extensions: Standalone Cluster Autoscaler +**Note**: For the cluster autoscaler you should choose **only one** of the options: +- the stand-alone cluster-autoscaler deployment, using this extension +- the [cluster-autoscaler add-on](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-configurationarguments_ClusterAutoscaler), using the [addons](./cluster_addons.md). + Deployed using the [cluster-autoscaler Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler) with configuration from the `worker_pools` variable. The module is using the `oke.oraclecloud.com/cluster_autoscaler` nodepool label to facilitate the understanding of how the Kubernetes cluster auto-scaler will interact with the node: diff --git a/examples/cluster-addons/vars-cluster-addons.auto.tfvars b/examples/cluster-addons/vars-cluster-addons.auto.tfvars new file mode 100644 index 00000000..39a41092 --- /dev/null +++ b/examples/cluster-addons/vars-cluster-addons.auto.tfvars @@ -0,0 +1,21 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +cluster_addons = { + "CertManager" = { + remove_addon_resources_on_delete = true + # The list of supported configurations for the cluster addons is here: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-configurationarguments_CertificateManager + configurations = [ + { + key = "numOfReplicas" + value = "1" + } + ] + } +} + +cluster_addons_to_remove = { + Flannel = { + remove_k8s_resources = true + } +} \ No newline at end of file diff --git a/module-cluster-addons.tf b/module-cluster-addons.tf new file mode 100644 index 00000000..a3a1bf4f --- /dev/null +++ b/module-cluster-addons.tf @@ -0,0 +1,28 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +module "cluster-addons" { + count = local.cluster_enabled && lower(var.cluster_type) == "enhanced" ? 1 : 0 + source = "./modules/cluster-addons" + + operator_enabled = local.operator_enabled + + cluster_addons = var.cluster_addons + cluster_addons_to_remove = var.cluster_addons_to_remove + + cluster_id = coalesce(var.cluster_id, one(module.cluster[*].cluster_id)) + kubernetes_version = var.kubernetes_version + + # Bastion/operator connection + ssh_private_key = sensitive(local.ssh_private_key) + bastion_host = local.bastion_public_ip + bastion_user = var.bastion_user + operator_host = local.operator_private_ip + operator_user = var.operator_user +} + + +# output "supported_addons" { +# description = "Supported cluster addons" +# value = var.output_detail ? try(one(module.cluster-addons[*].supported_addons), null) : null +# } \ No newline at end of file diff --git a/modules/cluster-addons/addons.tf b/modules/cluster-addons/addons.tf new file mode 100644 index 00000000..e72f74b5 --- /dev/null +++ b/modules/cluster-addons/addons.tf @@ -0,0 +1,83 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +data "oci_containerengine_addon_options" "k8s_addon_options" { + kubernetes_version = var.kubernetes_version +} + +locals { + supported_addons = [for entry in data.oci_containerengine_addon_options.k8s_addon_options.addon_options : entry.name] + primary_addons = ["CertManager"] + addons_defaults = { + remove_addon_resources_on_delete = true + configurations = [] + version = null + } + addons_with_defaults = { for addon_name, addon_value in var.cluster_addons : + addon_name => merge(local.addons_defaults, addon_value) + } +} + +resource "oci_containerengine_addon" "primary_addon" { + for_each = { for k, v in local.addons_with_defaults : k => v if contains(local.primary_addons, k) } + + addon_name = each.key + cluster_id = var.cluster_id + + remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true) + + dynamic "configurations" { + for_each = lookup(each.value, "configurations", []) + iterator = config + + content { + key = tostring(lookup(config.value, "key")) + value = tostring(lookup(config.value, "value")) + } + } + + version = lookup(each.value, "version", null) + + lifecycle { + + precondition { + condition = contains(local.supported_addons, each.key) + error_message = <<-EOT + The addon ${each.key} is not supported. + The list of supported addons is: ${join(", ", local.supported_addons)}. + EOT + } + } +} + +resource "oci_containerengine_addon" "secondary_addon" { + for_each = { for k, v in local.addons_with_defaults : k => v if !contains(local.primary_addons, k) } + depends_on = [oci_containerengine_addon.primary_addon] + addon_name = each.key + cluster_id = var.cluster_id + + remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true) + + dynamic "configurations" { + for_each = lookup(each.value, "configurations", []) + iterator = config + + content { + key = tostring(lookup(config.value, "key")) + value = tostring(lookup(config.value, "value")) + } + } + + version = lookup(each.value, "version", null) + + lifecycle { + + precondition { + condition = contains(local.supported_addons, each.key) + error_message = <<-EOT + The addon ${each.key} is not supported. + The list of supported addons is: ${join(", ", local.supported_addons)}. + EOT + } + } +} \ No newline at end of file diff --git a/modules/cluster-addons/delete_addons.tf b/modules/cluster-addons/delete_addons.tf new file mode 100644 index 00000000..eacd3ea5 --- /dev/null +++ b/modules/cluster-addons/delete_addons.tf @@ -0,0 +1,49 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +locals { + remove_addon_command = "oci ce cluster disable-addon --addon-name %s --cluster-id %s --is-remove-existing-add-on %t --force" + remove_addons_defaults = { + custom_commands = [] + remove_k8s_resources = true + } + remove_addons_with_defaults = { for addon_name, addon_value in var.cluster_addons_to_remove : + addon_name => merge(local.remove_addons_defaults, addon_value) + } +} + +resource "null_resource" "remove_addons" { + for_each = var.operator_enabled ? local.remove_addons_with_defaults : {} + depends_on = [oci_containerengine_addon.primary_addon, oci_containerengine_addon.secondary_addon] + + connection { + bastion_host = var.bastion_host + bastion_user = var.bastion_user + bastion_private_key = var.ssh_private_key + host = var.operator_host + user = var.operator_user + private_key = var.ssh_private_key + timeout = "40m" + type = "ssh" + } + + provisioner "remote-exec" { + inline = concat( + [ + "echo 'Removing ${each.key} addon'", + format(local.remove_addon_command, each.key, var.cluster_id, lookup(each.value, "remove_k8s_resources")) + ], + lookup(each.value, "custom_commands") + ) + } + + lifecycle { + precondition { + condition = contains(local.supported_addons, each.key) + error_message = <<-EOT + The addon ${each.key} is not supported. + The list of supported addons is: ${join(", ", local.supported_addons)}. + EOT + } + } +} \ No newline at end of file diff --git a/modules/cluster-addons/outputs.tf b/modules/cluster-addons/outputs.tf new file mode 100644 index 00000000..04d22cdb --- /dev/null +++ b/modules/cluster-addons/outputs.tf @@ -0,0 +1,6 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +output "supported_addons" { + value = data.oci_containerengine_addon_options.k8s_addon_options.addon_options +} diff --git a/modules/cluster-addons/variables.tf b/modules/cluster-addons/variables.tf new file mode 100644 index 00000000..3fc6984c --- /dev/null +++ b/modules/cluster-addons/variables.tf @@ -0,0 +1,17 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +# General variables +variable "cluster_id" { type = string } +variable "cluster_addons" { type = any } +variable "cluster_addons_to_remove" { type = any } +variable "kubernetes_version" { type = string } + +# Variables required to access the operator host +variable "bastion_host" { type = string } +variable "bastion_user" { type = string } +variable "operator_enabled" { type = bool } +variable "operator_host" { type = string } +variable "operator_user" { type = string } +variable "ssh_private_key" { type = string } + diff --git a/modules/cluster-addons/versions.tf b/modules/cluster-addons/versions.tf new file mode 100644 index 00000000..838c7697 --- /dev/null +++ b/modules/cluster-addons/versions.tf @@ -0,0 +1,13 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +terraform { + required_version = ">= 1.2.0" + + required_providers { + oci = { + source = "oracle/oci" + version = ">= 4.119.0" + } + } +} diff --git a/variables-cluster-addons.tf b/variables-cluster-addons.tf new file mode 100644 index 00000000..50e16cef --- /dev/null +++ b/variables-cluster-addons.tf @@ -0,0 +1,14 @@ +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +variable "cluster_addons" { + description = "Map with cluster addons not created by Terraform that should be removed. This operation is performed using oci-cli and requires the operator host to be deployed." + type = any + default = {} +} + +variable "cluster_addons_to_remove" { + description = "Map with cluster addons that should be enabled. See ClusterAddon documentation for the supported configuration of each addon." + type = any + default = {} +} \ No newline at end of file