diff --git a/CHANGELOG.md b/CHANGELOG.md index c070bc124c..017b69bba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Extending the adopted spec, each change should have a link to its corresponding pull request appended. ## [Unreleased] + +### Changed + +* All Beta functionality removed from non-beta clusters, some properties like node_pool taints available only in beta cluster now [#228] +* **Breaking**: Enabled metadata-concealment by default [#248] + ### Added -* Enabled metadata-concealment by default [#248] +* Added support for resource usage export config [#238] +* Added `sandbox_enabled` variable to use GKE Sandbox [#241] * Added `grant_registry_access` variable to grant Container Registry access to created SA [#236] * Support for Intranode Visbiility (IV) and Veritical Pod Autoscaling (VPA) beta features [#216] * Support for Workload Identity beta feature [#234] * Support for Google Groups based RBAC beta feature [#217] +* Support for disabling node pool autoscaling by setting `autoscaling` to `false` within the node pool variable. [#250] ## [v4.1.0] 2019-07-24 @@ -171,6 +179,10 @@ Extending the adopted spec, each change should have a link to its corresponding [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 [#248]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/248 +[#228]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/228 +[#238]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/238 +[#241]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/241 +[#250]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/250 [#236]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/236 [#217]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/217 [#234]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/234 diff --git a/README.md b/README.md index f38023e600..923d3f7a09 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | diff --git a/autogen/README.md b/autogen/README.md index 5b44ad2f14..73a6314289 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -28,7 +28,7 @@ There are multiple examples included in the [examples](./examples/) folder but s ```hcl module "gke" { - source = "terraform-google-modules/kubernetes-engine/google{% if private_cluster %}//modules/private-cluster{% endif %}" + source = "terraform-google-modules/kubernetes-engine/google{{ module_path }}" project_id = "" name = "gke-test-1" region = "us-central1" @@ -157,7 +157,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -{% if private_cluster or beta_cluster %} +{% if beta_cluster %} - [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 {% else %} - [Terraform Provider for GCP][terraform-provider-google] v2.9 @@ -339,7 +339,7 @@ command. {% else %} [upgrading-to-v3.0]: docs/upgrading_to_v3.0.md {% endif %} -{% if private_cluster or beta_cluster %} +{% if beta_cluster %} [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta {% else %} [terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google diff --git a/autogen/auth.tf b/autogen/auth.tf index 21275cd41e..a23689bb7b 100644 --- a/autogen/auth.tf +++ b/autogen/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - {% if private_cluster or beta_cluster %} + {% if beta_cluster %} provider = google-beta {% else %} provider = google diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 664de67855..4e5fd74d55 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -20,7 +20,7 @@ Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { - {% if private_cluster or beta_cluster %} + {% if beta_cluster %} provider = google-beta {% else %} provider = google @@ -67,6 +67,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "resource_usage_export_config" { + for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : [] + content { + enable_network_egress_metering = true + bigquery_destination { + dataset_id = resource_usage_export_config.value + } + } + } {% endif %} dynamic "master_authorized_networks_config" { for_each = var.master_authorized_networks_config @@ -134,7 +143,7 @@ resource "google_container_cluster" "primary" { } lifecycle { - ignore_changes = [node_pool] + ignore_changes = [node_pool, initial_node_count] } timeouts { @@ -158,6 +167,14 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } {% endif %} } } @@ -203,7 +220,11 @@ resource "google_container_cluster" "primary" { Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { + {% if beta_cluster %} provider = google-beta + {% else %} + provider = google + {% endif %} count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id @@ -223,9 +244,14 @@ resource "google_container_node_pool" "pools" { max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) {% endif %} - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + + dynamic "autoscaling" { + for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] + content { + min_node_count = lookup(autoscaling.value, "min_count", 1) + max_node_count = lookup(autoscaling.value, "max_count", 100) + } } management { @@ -259,6 +285,7 @@ resource "google_container_node_pool" "pools" { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) + {% if beta_cluster %} dynamic "taint" { for_each = concat( var.node_pools_taints["all"], @@ -270,6 +297,7 @@ resource "google_container_node_pool" "pools" { value = taint.value.value } } + {% endif %} tags = concat( ["gke-${var.name}"], ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], diff --git a/autogen/main.tf b/autogen/main.tf index 9d6476b916..d9ad888d01 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -20,7 +20,7 @@ Get available zones in region *****************************************/ data "google_compute_zones" "available" { - {% if private_cluster or beta_cluster %} + {% if beta_cluster %} provider = google-beta {% else %} provider = google @@ -75,6 +75,8 @@ locals { security_group = var.authenticator_security_group }] + cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] + {% endif %} cluster_output_name = google_container_cluster.primary.name @@ -102,10 +104,10 @@ locals { {% if beta_cluster %} # BETA features - cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled - cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false # /BETA features {% endif %} @@ -135,7 +137,7 @@ locals { cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled {% if beta_cluster %} # BETA features - cluster_istio_enabled = ! local.cluster_output_istio_enabled + cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled diff --git a/autogen/networks.tf b/autogen/networks.tf index 19a9af5307..88df19bc3b 100644 --- a/autogen/networks.tf +++ b/autogen/networks.tf @@ -17,7 +17,7 @@ {{ autogeneration_note }} data "google_compute_network" "gke_network" { - {% if private_cluster or beta_cluster %} + {% if beta_cluster %} provider = google-beta {% else %} provider = google @@ -28,7 +28,7 @@ data "google_compute_network" "gke_network" { } data "google_compute_subnetwork" "gke_subnetwork" { - {% if private_cluster or beta_cluster %} + {% if beta_cluster %} provider = google-beta {% else %} provider = google diff --git a/autogen/variables.tf b/autogen/variables.tf index f88ce66782..0fedacb2af 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -178,6 +178,7 @@ variable "node_pools_metadata" { } } +{% if beta_cluster %} variable "node_pools_taints" { type = map(list(object({key=string,value=string,effect=string}))) description = "Map of lists containing node taints by node-pool name" @@ -188,6 +189,7 @@ variable "node_pools_taints" { } } +{% endif %} variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" @@ -366,19 +368,31 @@ variable "pod_security_policy_config" { }] } +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + variable "node_metadata" { description = "Specifies how node metadata is exposed to the workload running on the node" default = "SECURE" type = string } +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + variable "enable_intranode_visibility" { type = bool description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" default = false } - variable "enable_vertical_pod_autoscaling" { +variable "enable_vertical_pod_autoscaling" { type = bool description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" default = false diff --git a/cluster.tf b/cluster.tf index 499e689a4b..ffdb27b0fc 100644 --- a/cluster.tf +++ b/cluster.tf @@ -99,7 +99,7 @@ resource "google_container_cluster" "primary" { } lifecycle { - ignore_changes = [node_pool] + ignore_changes = [node_pool, initial_node_count] } timeouts { @@ -125,7 +125,7 @@ resource "google_container_cluster" "primary" { Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = google-beta + provider = google count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id @@ -142,9 +142,14 @@ resource "google_container_node_pool" "pools" { lookup(var.node_pools[count.index], "min_count", 1), ) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + + dynamic "autoscaling" { + for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] + content { + min_node_count = lookup(autoscaling.value, "min_count", 1) + max_node_count = lookup(autoscaling.value, "max_count", 100) + } } management { @@ -178,17 +183,6 @@ resource "google_container_node_pool" "pools" { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } tags = concat( ["gke-${var.name}"], ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index 10bce771ef..3e15164913 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - provider "kubernetes" { load_config_file = false host = "https://${module.gke.endpoint}" diff --git a/examples/disable_client_cert/main.tf b/examples/disable_client_cert/main.tf index c64f09fd67..221eb994ce 100644 --- a/examples/disable_client_cert/main.tf +++ b/examples/disable_client_cert/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - module "gke" { source = "../../" diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 2fb447fbb3..6662bb84ac 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -18,18 +18,13 @@ locals { cluster_type = "node-pool" } -provider "google" { - version = "~> 2.12.0" - region = var.region -} - provider "google-beta" { version = "~> 2.12.0" region = var.region } module "gke" { - source = "../../" + source = "../../modules/beta-public-cluster/" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" regional = false diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf index a0221ca360..766239ead8 100644 --- a/examples/shared_vpc/main.tf +++ b/examples/shared_vpc/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - module "gke" { source = "../../" project_id = var.project_id diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index bd42f43d1a..4662435fbd 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - module "gke" { source = "../../" project_id = var.project_id diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index 9eaf2b6117..fc95090ede 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -18,12 +18,6 @@ locals { cluster_type = "simple-regional-beta" } -provider "google" { - version = "~> 2.12.0" - credentials = file(var.credentials_path) - region = var.region -} - provider "google-beta" { version = "~> 2.12.0" credentials = file(var.credentials_path) diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index b79c21c770..89568e86ee 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -18,7 +18,7 @@ locals { cluster_type = "simple-regional-private" } -provider "google-beta" { +provider "google" { version = "~> 2.12.0" region = var.region } diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf index 7e04d7e4fe..edd90f7a0d 100644 --- a/examples/simple_zonal/main.tf +++ b/examples/simple_zonal/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - module "gke" { source = "../../" project_id = var.project_id diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf index 2192787516..428218d76a 100644 --- a/examples/simple_zonal_private/main.tf +++ b/examples/simple_zonal_private/main.tf @@ -18,7 +18,7 @@ locals { cluster_type = "simple-regional-private" } -provider "google-beta" { +provider "google" { version = "~> 2.12.0" region = var.region } diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf index 4227aac952..37264f781b 100644 --- a/examples/stub_domains/main.tf +++ b/examples/stub_domains/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - module "gke" { source = "../../" project_id = var.project_id diff --git a/examples/stub_domains_private/main.tf b/examples/stub_domains_private/main.tf index 046f9838c0..65bc48247d 100644 --- a/examples/stub_domains_private/main.tf +++ b/examples/stub_domains_private/main.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -provider "google-beta" { +provider "google" { version = "~> 2.12.0" region = var.region } diff --git a/examples/stub_domains_upstream_nameservers/main.tf b/examples/stub_domains_upstream_nameservers/main.tf index 42f3967d5a..0da83b95c6 100644 --- a/examples/stub_domains_upstream_nameservers/main.tf +++ b/examples/stub_domains_upstream_nameservers/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - module "gke" { source = "../../" project_id = var.project_id diff --git a/examples/upstream_nameservers/main.tf b/examples/upstream_nameservers/main.tf index 8a997e8c7a..ecded7c29c 100644 --- a/examples/upstream_nameservers/main.tf +++ b/examples/upstream_nameservers/main.tf @@ -23,11 +23,6 @@ provider "google" { region = var.region } -provider "google-beta" { - version = "~> 2.12.0" - region = var.region -} - module "gke" { source = "../../" project_id = var.project_id diff --git a/helpers/generate_modules/generate_modules.py b/helpers/generate_modules/generate_modules.py index f6beb84832..c235e7ad65 100755 --- a/helpers/generate_modules/generate_modules.py +++ b/helpers/generate_modules/generate_modules.py @@ -46,13 +46,16 @@ def template_options(self, base): 'private_cluster': False, }), Module("./modules/private-cluster", { + 'module_path': '//modules/private-cluster', 'private_cluster': True }), Module("./modules/beta-private-cluster", { + 'module_path': '//modules/beta-private-cluster', 'private_cluster': True, 'beta_cluster': True, }), Module("./modules/beta-public-cluster", { + 'module_path': '//modules/beta-public-cluster', 'private_cluster': False, 'beta_cluster': True, }), diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 7c750d3bbe..1175553890 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -25,7 +25,7 @@ There are multiple examples included in the [examples](./examples/) folder but s ```hcl module "gke" { - source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" project_id = "" name = "gke-test-1" region = "us-central1" @@ -191,6 +191,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | region | The region to host the cluster in (required) | string | n/a | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index d887d332ed..c481c69a35 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -62,6 +62,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "resource_usage_export_config" { + for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : [] + content { + enable_network_egress_metering = true + bigquery_destination { + dataset_id = resource_usage_export_config.value + } + } + } dynamic "master_authorized_networks_config" { for_each = var.master_authorized_networks_config content { @@ -126,7 +135,7 @@ resource "google_container_cluster" "primary" { } lifecycle { - ignore_changes = [node_pool] + ignore_changes = [node_pool, initial_node_count] } timeouts { @@ -149,6 +158,14 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } } @@ -207,9 +224,14 @@ resource "google_container_node_pool" "pools" { ) max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + + dynamic "autoscaling" { + for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] + content { + min_node_count = lookup(autoscaling.value, "min_count", 1) + max_node_count = lookup(autoscaling.value, "max_count", 100) + } } management { diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 43fea3b6dc..fc38644871 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -70,6 +70,8 @@ locals { security_group = var.authenticator_security_group }] + cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] + cluster_output_name = google_container_cluster.primary.name cluster_output_location = google_container_cluster.primary.location @@ -91,10 +93,10 @@ locals { cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features - cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled - cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false # /BETA features @@ -122,7 +124,7 @@ locals { cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features - cluster_istio_enabled = ! local.cluster_output_istio_enabled + cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 82eb9906d0..9a869a830f 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -363,12 +363,24 @@ variable "pod_security_policy_config" { }] } +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + variable "node_metadata" { description = "Specifies how node metadata is exposed to the workload running on the node" default = "SECURE" type = string } +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + variable "enable_intranode_visibility" { type = bool description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7c257c619f..1f78c95082 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -23,7 +23,7 @@ There are multiple examples included in the [examples](./examples/) folder but s ```hcl module "gke" { - source = "terraform-google-modules/kubernetes-engine/google" + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster" project_id = "" name = "gke-test-1" region = "us-central1" @@ -182,6 +182,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | region | The region to host the cluster in (required) | string | n/a | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index c26d85ff50..a264e932b9 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -62,6 +62,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "resource_usage_export_config" { + for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : [] + content { + enable_network_egress_metering = true + bigquery_destination { + dataset_id = resource_usage_export_config.value + } + } + } dynamic "master_authorized_networks_config" { for_each = var.master_authorized_networks_config content { @@ -126,7 +135,7 @@ resource "google_container_cluster" "primary" { } lifecycle { - ignore_changes = [node_pool] + ignore_changes = [node_pool, initial_node_count] } timeouts { @@ -149,6 +158,14 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } } @@ -202,9 +219,14 @@ resource "google_container_node_pool" "pools" { ) max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + + dynamic "autoscaling" { + for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] + content { + min_node_count = lookup(autoscaling.value, "min_count", 1) + max_node_count = lookup(autoscaling.value, "max_count", 100) + } } management { diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index db5138e99f..dea58d4de5 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -70,6 +70,8 @@ locals { security_group = var.authenticator_security_group }] + cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : [] + cluster_output_name = google_container_cluster.primary.name cluster_output_location = google_container_cluster.primary.location @@ -91,10 +93,10 @@ locals { cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features - cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled - cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility - cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false # /BETA features @@ -122,7 +124,7 @@ locals { cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features - cluster_istio_enabled = ! local.cluster_output_istio_enabled + cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index acfecfb515..0ae2b75661 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -339,12 +339,24 @@ variable "pod_security_policy_config" { }] } +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + variable "node_metadata" { description = "Specifies how node metadata is exposed to the workload running on the node" default = "SECURE" type = string } +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + variable "enable_intranode_visibility" { type = bool description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index d3044b770d..c29d58ee93 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -171,7 +171,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | @@ -226,7 +225,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins - [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 +- [Terraform Provider for GCP][terraform-provider-google] v2.9 ### Configure a Service Account In order to execute this module you must have a Service Account with the @@ -396,6 +395,6 @@ command. [upgrading-to-v2.0]: ../../docs/upgrading_to_v2.0.md [upgrading-to-v3.0]: ../../docs/upgrading_to_v3.0.md -[terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta +[terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google [3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 [terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/modules/private-cluster/auth.tf b/modules/private-cluster/auth.tf index c177eee5a7..48e7cc6a5f 100644 --- a/modules/private-cluster/auth.tf +++ b/modules/private-cluster/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = google-beta + provider = google } /****************************************** diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 6ea5394b02..412e8295ed 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -20,7 +20,7 @@ Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = google-beta + provider = google name = var.name description = var.description @@ -99,7 +99,7 @@ resource "google_container_cluster" "primary" { } lifecycle { - ignore_changes = [node_pool] + ignore_changes = [node_pool, initial_node_count] } timeouts { @@ -130,7 +130,7 @@ resource "google_container_cluster" "primary" { Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = google-beta + provider = google count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id @@ -147,9 +147,14 @@ resource "google_container_node_pool" "pools" { lookup(var.node_pools[count.index], "min_count", 1), ) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + + dynamic "autoscaling" { + for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] + content { + min_node_count = lookup(autoscaling.value, "min_count", 1) + max_node_count = lookup(autoscaling.value, "max_count", 100) + } } management { @@ -183,17 +188,6 @@ resource "google_container_node_pool" "pools" { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } tags = concat( ["gke-${var.name}"], ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 9c8dc629eb..bfe746401c 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -20,7 +20,7 @@ Get available zones in region *****************************************/ data "google_compute_zones" "available" { - provider = google-beta + provider = google project = var.project_id region = var.region diff --git a/modules/private-cluster/networks.tf b/modules/private-cluster/networks.tf index 14ea500e03..a382073dc0 100644 --- a/modules/private-cluster/networks.tf +++ b/modules/private-cluster/networks.tf @@ -17,14 +17,14 @@ // This file was automatically generated from a template in ./autogen data "google_compute_network" "gke_network" { - provider = google-beta + provider = google name = var.network project = local.network_project_id } data "google_compute_subnetwork" "gke_subnetwork" { - provider = google-beta + provider = google name = var.subnetwork region = var.region diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 407787f143..8008e08975 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -178,16 +178,6 @@ variable "node_pools_metadata" { } } -variable "node_pools_taints" { - type = map(list(object({ key = string, value = string, effect = string }))) - description = "Map of lists containing node taints by node-pool name" - - default = { - all = [] - default-node-pool = [] - } -} - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/test/fixtures/simple_regional_private/network.tf b/test/fixtures/simple_regional_private/network.tf index c50c2d12d1..f34f629069 100644 --- a/test/fixtures/simple_regional_private/network.tf +++ b/test/fixtures/simple_regional_private/network.tf @@ -20,10 +20,6 @@ resource "random_string" "suffix" { upper = false } -provider "google-beta" { - project = var.project_id -} - resource "google_compute_network" "main" { project = var.project_id name = "cft-gke-test-${random_string.suffix.result}" diff --git a/test/fixtures/simple_zonal_private/network.tf b/test/fixtures/simple_zonal_private/network.tf index c50c2d12d1..76d33f6bfc 100644 --- a/test/fixtures/simple_zonal_private/network.tf +++ b/test/fixtures/simple_zonal_private/network.tf @@ -20,9 +20,6 @@ resource "random_string" "suffix" { upper = false } -provider "google-beta" { - project = var.project_id -} resource "google_compute_network" "main" { project = var.project_id diff --git a/variables.tf b/variables.tf index d8c339b50e..460bdeaeff 100644 --- a/variables.tf +++ b/variables.tf @@ -178,16 +178,6 @@ variable "node_pools_metadata" { } } -variable "node_pools_taints" { - type = map(list(object({ key = string, value = string, effect = string }))) - description = "Map of lists containing node taints by node-pool name" - - default = { - all = [] - default-node-pool = [] - } -} - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name"