Merge branch 'master' into master

CHANGELOG.md

@@ -7,12 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Extending the adopted spec, each change should have a link to its corresponding pull request appended.
 
 ## [Unreleased]
+
+### Changed
+
+* All Beta functionality removed from non-beta clusters, some properties like node_pool taints available only in beta cluster now [#228]
+* **Breaking**: Enabled metadata-concealment by default [#248]
+
 ### Added
-* Enabled metadata-concealment by default [#248]
+* Added support for resource usage export config [#238]
+* Added `sandbox_enabled` variable to use GKE Sandbox [#241]
 * Added `grant_registry_access` variable to grant Container Registry access to created SA [#236]
 * Support for Intranode Visbiility (IV) and Veritical Pod Autoscaling (VPA) beta features [#216]
 * Support for Workload Identity beta feature [#234]
 * Support for Google Groups based RBAC beta feature [#217]
+* Support for disabling node pool autoscaling by setting `autoscaling` to `false` within the node pool variable. [#250]
 
 ## [v4.1.0] 2019-07-24
 
@@ -171,6 +179,10 @@ Extending the adopted spec, each change should have a link to its corresponding
 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0
 
 [#248]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/248
+[#228]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/228
+[#238]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/238
+[#241]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/241
+[#250]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/250
 [#236]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/236
 [#217]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/217
 [#234]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/234

README.md

@@ -162,7 +162,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o
 | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `<map>` | no |
 | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `<map>` | no |
 | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `<map>` | no |
-| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `<map>` | no |
 | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no |
 | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `<list>` | no |
 | project\_id | The project ID to host the cluster in (required) | string | n/a | yes |

autogen/README.md

@@ -28,7 +28,7 @@ There are multiple examples included in the [examples](./examples/) folder but s
 
 ```hcl
 module "gke" {
-  source                     = "terraform-google-modules/kubernetes-engine/google{% if private_cluster %}//modules/private-cluster{% endif %}"
+  source                     = "terraform-google-modules/kubernetes-engine/google{{ module_path }}"
   project_id                 = "<PROJECT ID>"
   name                       = "gke-test-1"
   region                     = "us-central1"
@@ -157,7 +157,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog
 - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x
 #### Terraform and Plugins
 - [Terraform](https://www.terraform.io/downloads.html) 0.12
-{% if private_cluster or beta_cluster %}
+{% if beta_cluster %}
 - [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9
 {% else %}
 - [Terraform Provider for GCP][terraform-provider-google] v2.9
@@ -339,7 +339,7 @@ command.
 {% else %}
 [upgrading-to-v3.0]: docs/upgrading_to_v3.0.md
 {% endif %}
-{% if private_cluster or beta_cluster %}
+{% if beta_cluster %}
 [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta
 {% else %}
 [terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google

autogen/auth.tf

@@ -20,7 +20,7 @@
   Retrieve authentication token
  *****************************************/
 data "google_client_config" "default" {
-  {% if private_cluster or beta_cluster %}
+  {% if  beta_cluster %}
   provider = google-beta
   {% else %}
   provider = google

autogen/cluster.tf

@@ -20,7 +20,7 @@
   Create Container Cluster
  *****************************************/
 resource "google_container_cluster" "primary" {
-  {% if private_cluster or beta_cluster %}
+  {% if beta_cluster %}
   provider = google-beta
   {% else %}
   provider = google
@@ -67,6 +67,15 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "resource_usage_export_config" {
+    for_each = var.resource_usage_export_dataset_id != "" ? [var.resource_usage_export_dataset_id] : []
+    content {
+      enable_network_egress_metering = true
+      bigquery_destination {
+        dataset_id = resource_usage_export_config.value
+      }
+    }
+  }
 {% endif %}
   dynamic "master_authorized_networks_config" {
     for_each = var.master_authorized_networks_config
@@ -134,7 +143,7 @@ resource "google_container_cluster" "primary" {
   }
 
   lifecycle {
-    ignore_changes = [node_pool]
+    ignore_changes = [node_pool, initial_node_count]
   }
 
   timeouts {
@@ -158,6 +167,14 @@ resource "google_container_cluster" "primary" {
           node_metadata = workload_metadata_config.value.node_metadata
         }
       }
+
+      dynamic "sandbox_config" {
+        for_each = local.cluster_sandbox_enabled
+
+        content {
+          sandbox_type = sandbox_config.value
+        }
+      }
       {% endif %}
     }
   }
@@ -203,7 +220,11 @@ resource "google_container_cluster" "primary" {
   Create Container Cluster node pools
  *****************************************/
 resource "google_container_node_pool" "pools" {
+  {% if beta_cluster %}
   provider = google-beta
+  {% else %}
+  provider = google
+  {% endif %}
   count    = length(var.node_pools)
   name     = var.node_pools[count.index]["name"]
   project  = var.project_id
@@ -223,9 +244,14 @@ resource "google_container_node_pool" "pools" {
   max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null)
   {% endif %}
 
-  autoscaling {
-    min_node_count = lookup(var.node_pools[count.index], "min_count", 1)
-    max_node_count = lookup(var.node_pools[count.index], "max_count", 100)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+
+  dynamic "autoscaling" {
+    for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
+    content {
+      min_node_count = lookup(autoscaling.value, "min_count", 1)
+      max_node_count = lookup(autoscaling.value, "max_count", 100)
+    }
   }
 
   management {
@@ -259,6 +285,7 @@ resource "google_container_node_pool" "pools" {
         "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints
       },
     )
+    {% if beta_cluster %}
     dynamic "taint" {
       for_each = concat(
         var.node_pools_taints["all"],
@@ -270,6 +297,7 @@ resource "google_container_node_pool" "pools" {
         value  = taint.value.value
       }
     }
+    {% endif %}
     tags = concat(
       ["gke-${var.name}"],
       ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],

autogen/main.tf

@@ -20,7 +20,7 @@
   Get available zones in region
  *****************************************/
 data "google_compute_zones" "available" {
-  {% if private_cluster or beta_cluster %}
+  {% if beta_cluster %}
   provider = google-beta
   {% else %}
   provider = google
@@ -75,6 +75,8 @@ locals {
     security_group = var.authenticator_security_group
   }]
 
+  cluster_sandbox_enabled = var.sandbox_enabled ? ["gvisor"] : []
+
 {% endif %}
 
   cluster_output_name           = google_container_cluster.primary.name
@@ -102,10 +104,10 @@ locals {
 
 {% if beta_cluster %}
   # BETA features
-  cluster_output_istio_enabled                    = google_container_cluster.primary.addons_config.0.istio_config.0.disabled
-  cluster_output_pod_security_policy_enabled      = google_container_cluster.primary.pod_security_policy_config.0.enabled
+  cluster_output_istio_disabled                   = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false
+  cluster_output_pod_security_policy_enabled      = google_container_cluster.primary.pod_security_policy_config != null && length(google_container_cluster.primary.pod_security_policy_config) == 1 ? google_container_cluster.primary.pod_security_policy_config.0.enabled : false
   cluster_output_intranode_visbility_enabled      = google_container_cluster.primary.enable_intranode_visibility
-  cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled
+  cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling != null && length(google_container_cluster.primary.vertical_pod_autoscaling) == 1 ? google_container_cluster.primary.vertical_pod_autoscaling.0.enabled : false
 
   # /BETA features
   {% endif %}
@@ -135,7 +137,7 @@ locals {
   cluster_kubernetes_dashboard_enabled       = ! local.cluster_output_kubernetes_dashboard_enabled
 {% if beta_cluster %}
   # BETA features
-  cluster_istio_enabled                    = ! local.cluster_output_istio_enabled
+  cluster_istio_enabled                    = ! local.cluster_output_istio_disabled
   cluster_cloudrun_enabled                 = var.cloudrun
   cluster_pod_security_policy_enabled      = local.cluster_output_pod_security_policy_enabled
   cluster_intranode_visibility_enabled     = local.cluster_output_intranode_visbility_enabled

autogen/networks.tf

@@ -17,7 +17,7 @@
 {{ autogeneration_note }}
 
 data "google_compute_network" "gke_network" {
-  {% if private_cluster or beta_cluster %}
+  {% if beta_cluster %}
   provider = google-beta
   {% else %}
   provider = google
@@ -28,7 +28,7 @@ data "google_compute_network" "gke_network" {
 }
 
 data "google_compute_subnetwork" "gke_subnetwork" {
-  {% if private_cluster or beta_cluster %}
+  {% if beta_cluster %}
   provider = google-beta
   {% else %}
   provider = google

autogen/variables.tf

@@ -178,6 +178,7 @@ variable "node_pools_metadata" {
   }
 }
 
+{% if beta_cluster %}
 variable "node_pools_taints" {
   type        = map(list(object({key=string,value=string,effect=string})))
   description = "Map of lists containing node taints by node-pool name"
@@ -188,6 +189,7 @@ variable "node_pools_taints" {
   }
 }
 
+{% endif %}
 variable "node_pools_tags" {
   type        = map(list(string))
   description = "Map of lists containing node network tags by node-pool name"
@@ -366,19 +368,31 @@ variable "pod_security_policy_config" {
   }]
 }
 
+variable "resource_usage_export_dataset_id" {
+  type        = string
+  description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic."
+  default     = ""
+}
+
 variable "node_metadata" {
   description = "Specifies how node metadata is exposed to the workload running on the node"
   default     = "SECURE"
   type        = string
 }
 
+variable "sandbox_enabled" {
+  type        = bool
+  description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)."
+  default     = false
+}
+
 variable "enable_intranode_visibility" {
   type        = bool
   description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network"
   default     = false
 }
 
- variable "enable_vertical_pod_autoscaling" {
+variable "enable_vertical_pod_autoscaling" {
   type        = bool
   description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it"
   default     = false

cluster.tf

@@ -99,7 +99,7 @@ resource "google_container_cluster" "primary" {
   }
 
   lifecycle {
-    ignore_changes = [node_pool]
+    ignore_changes = [node_pool, initial_node_count]
   }
 
   timeouts {
@@ -125,7 +125,7 @@ resource "google_container_cluster" "primary" {
   Create Container Cluster node pools
  *****************************************/
 resource "google_container_node_pool" "pools" {
-  provider = google-beta
+  provider = google
   count    = length(var.node_pools)
   name     = var.node_pools[count.index]["name"]
   project  = var.project_id
@@ -142,9 +142,14 @@ resource "google_container_node_pool" "pools" {
     lookup(var.node_pools[count.index], "min_count", 1),
   )
 
-  autoscaling {
-    min_node_count = lookup(var.node_pools[count.index], "min_count", 1)
-    max_node_count = lookup(var.node_pools[count.index], "max_count", 100)
+  node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1)
+
+  dynamic "autoscaling" {
+    for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : []
+    content {
+      min_node_count = lookup(autoscaling.value, "min_count", 1)
+      max_node_count = lookup(autoscaling.value, "max_count", 100)
+    }
   }
 
   management {
@@ -178,17 +183,6 @@ resource "google_container_node_pool" "pools" {
         "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints
       },
     )
-    dynamic "taint" {
-      for_each = concat(
-        var.node_pools_taints["all"],
-        var.node_pools_taints[var.node_pools[count.index]["name"]],
-      )
-      content {
-        effect = taint.value.effect
-        key    = taint.value.key
-        value  = taint.value.value
-      }
-    }
     tags = concat(
       ["gke-${var.name}"],
       ["gke-${var.name}-${var.node_pools[count.index]["name"]}"],

examples/deploy_service/main.tf

@@ -23,11 +23,6 @@ provider "google" {
   region  = var.region
 }
 
-provider "google-beta" {
-  version = "~> 2.12.0"
-  region  = var.region
-}
-
 provider "kubernetes" {
   load_config_file       = false
   host                   = "https://${module.gke.endpoint}"

examples/disable_client_cert/main.tf

@@ -23,11 +23,6 @@ provider "google" {
   region  = var.region
 }
 
-provider "google-beta" {
-  version = "~> 2.12.0"
-  region  = var.region
-}
-
 module "gke" {
   source = "../../"
 

examples/node_pool/main.tf

@@ -18,18 +18,13 @@ locals {
   cluster_type = "node-pool"
 }
 
-provider "google" {
-  version = "~> 2.12.0"
-  region  = var.region
-}
-
 provider "google-beta" {
   version = "~> 2.12.0"
   region  = var.region
 }
 
 module "gke" {
-  source                            = "../../"
+  source                            = "../../modules/beta-public-cluster/"
   project_id                        = var.project_id
   name                              = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
   regional                          = false

examples/shared_vpc/main.tf

@@ -23,11 +23,6 @@ provider "google" {
   region  = var.region
 }
 
-provider "google-beta" {
-  version = "~> 2.12.0"
-  region  = var.region
-}
-
 module "gke" {
   source                 = "../../"
   project_id             = var.project_id