From c2a1401bcb01a3c08a18a685f5bcfb80cdac059a Mon Sep 17 00:00:00 2001 From: Bohdan Yurov Date: Thu, 13 Jun 2019 20:14:06 +0300 Subject: [PATCH] Fixes #158: Add support for Terraform v0.12 https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/158 https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/75 Add support for TF 0.12. - autogen - root and private_cluster modules - tests (including support for validation) - examples --- CHANGELOG.md | 11 + Gemfile | 2 +- Makefile | 2 +- README.md | 78 +---- auth.tf | 6 +- autogen/README.md | 6 +- autogen/auth.tf | 10 +- autogen/cluster_regional.tf | 281 +++++++++++----- autogen/cluster_zonal.tf | 270 +++++++++++----- autogen/dns.tf | 58 +++- autogen/main.tf | 299 +++++++++++------ autogen/masq.tf | 14 +- autogen/networks.tf | 24 +- autogen/outputs.tf | 44 +-- autogen/sa.tf | 37 ++- autogen/variables.tf | 65 +++- autogen/versions.tf | 19 ++ cluster_regional.tf | 213 ++++++++---- cluster_zonal.tf | 209 ++++++++---- dns.tf | 58 +++- examples/deploy_service/main.tf | 35 +- examples/deploy_service/outputs.tf | 9 +- examples/disable_client_cert/main.tf | 24 +- examples/disable_client_cert/outputs.tf | 9 +- examples/disable_client_cert/variables.tf | 1 + examples/node_pool/main.tf | 53 ++- examples/node_pool/outputs.tf | 9 +- examples/node_pool/variables.tf | 3 +- examples/shared_vpc/main.tf | 23 +- examples/shared_vpc/outputs.tf | 9 +- examples/shared_vpc/variables.tf | 1 + examples/simple_regional/main.tf | 21 +- examples/simple_regional/outputs.tf | 9 +- examples/simple_regional/variables.tf | 1 + examples/simple_regional_beta/main.tf | 34 +- examples/simple_regional_beta/outputs.tf | 8 +- examples/simple_regional_beta/test_outputs.tf | 22 +- examples/simple_regional_beta/versions.tf | 19 ++ examples/simple_regional_private/main.tf | 41 +-- examples/simple_regional_private/outputs.tf | 9 +- examples/simple_regional_private/variables.tf | 1 + examples/simple_regional_private_beta/main.tf | 50 +-- .../simple_regional_private_beta/outputs.tf | 9 +- .../test_outputs.tf | 23 +- .../simple_regional_private_beta/variables.tf | 1 + .../simple_regional_private_beta/versions.tf | 19 ++ examples/simple_zonal/main.tf | 21 +- examples/simple_zonal/outputs.tf | 9 +- examples/simple_zonal/variables.tf | 3 +- examples/simple_zonal_private/main.tf | 43 +-- examples/simple_zonal_private/outputs.tf | 9 +- examples/simple_zonal_private/variables.tf | 3 +- examples/stub_domains/main.tf | 24 +- examples/stub_domains/outputs.tf | 9 +- examples/stub_domains/variables.tf | 1 + examples/stub_domains_private/main.tf | 52 +-- examples/stub_domains_private/outputs.tf | 9 +- examples/stub_domains_private/test_outputs.tf | 21 +- examples/stub_domains_private/variables.tf | 1 + .../stub_domains_upstream_nameservers/main.tf | 24 +- .../outputs.tf | 9 +- .../test_outputs.tf | 21 +- .../variables.tf | 1 + .../versions.tf | 19 ++ examples/upstream_nameservers/main.tf | 23 +- examples/upstream_nameservers/outputs.tf | 9 +- examples/upstream_nameservers/test_outputs.tf | 21 +- examples/upstream_nameservers/variables.tf | 1 + examples/upstream_nameservers/versions.tf | 19 ++ examples/workload_metadata_config/main.tf | 43 +-- examples/workload_metadata_config/outputs.tf | 9 +- .../workload_metadata_config/variables.tf | 3 +- examples/workload_metadata_config/versions.tf | 19 ++ main.tf | 264 ++++++++++----- masq.tf | 14 +- modules/beta-private-cluster/README.md | 91 +----- modules/beta-private-cluster/auth.tf | 6 +- .../beta-private-cluster/cluster_regional.tf | 263 ++++++++++----- modules/beta-private-cluster/cluster_zonal.tf | 256 ++++++++++----- modules/beta-private-cluster/dns.tf | 58 +++- modules/beta-private-cluster/main.tf | 303 +++++++++++------- modules/beta-private-cluster/masq.tf | 14 +- modules/beta-private-cluster/networks.tf | 16 +- modules/beta-private-cluster/outputs.tf | 44 +-- modules/beta-private-cluster/sa.tf | 37 ++- modules/beta-private-cluster/variables.tf | 121 ++++--- modules/beta-private-cluster/versions.tf | 19 ++ modules/beta-public-cluster/README.md | 87 +---- modules/beta-public-cluster/auth.tf | 6 +- .../beta-public-cluster/cluster_regional.tf | 258 +++++++++++---- modules/beta-public-cluster/cluster_zonal.tf | 251 +++++++++++---- modules/beta-public-cluster/dns.tf | 58 +++- modules/beta-public-cluster/main.tf | 298 ++++++++++------- modules/beta-public-cluster/masq.tf | 14 +- modules/beta-public-cluster/networks.tf | 16 +- modules/beta-public-cluster/outputs.tf | 44 +-- modules/beta-public-cluster/sa.tf | 37 ++- modules/beta-public-cluster/variables.tf | 109 ++++--- modules/beta-public-cluster/versions.tf | 19 ++ modules/private-cluster/README.md | 82 +---- modules/private-cluster/auth.tf | 6 +- modules/private-cluster/cluster_regional.tf | 218 +++++++++---- modules/private-cluster/cluster_zonal.tf | 214 +++++++++---- modules/private-cluster/dns.tf | 58 +++- modules/private-cluster/main.tf | 273 ++++++++++------ modules/private-cluster/masq.tf | 14 +- modules/private-cluster/networks.tf | 16 +- modules/private-cluster/outputs.tf | 38 +-- modules/private-cluster/sa.tf | 37 ++- modules/private-cluster/variables.tf | 119 ++++--- modules/private-cluster/versions.tf | 19 ++ networks.tf | 16 +- outputs.tf | 38 +-- sa.tf | 37 ++- test/fixtures/all_examples/test_outputs.tf | 20 +- test/fixtures/deploy_service/example.tf | 15 +- test/fixtures/deploy_service/network.tf | 9 +- test/fixtures/disable_client_cert/example.tf | 19 +- test/fixtures/disable_client_cert/network.tf | 10 +- test/fixtures/node_pool/example.tf | 17 +- test/fixtures/node_pool/network.tf | 9 +- test/fixtures/shared/outputs.tf | 29 +- test/fixtures/shared/variables.tf | 3 +- test/fixtures/shared_vpc/example.tf | 17 +- test/fixtures/shared_vpc/network.tf | 9 +- test/fixtures/simple_regional/example.tf | 15 +- test/fixtures/simple_regional/network.tf | 9 +- .../simple_regional_private/example.tf | 15 +- .../simple_regional_private/network.tf | 13 +- test/fixtures/simple_zonal/example.tf | 15 +- test/fixtures/simple_zonal/network.tf | 9 +- test/fixtures/simple_zonal_private/example.tf | 17 +- test/fixtures/simple_zonal_private/network.tf | 13 +- test/fixtures/stub_domains/example.tf | 15 +- test/fixtures/stub_domains/network.tf | 9 +- test/fixtures/stub_domains_private/main.tf | 25 +- .../example.tf | 15 +- .../network.tf | 7 +- .../versions.tf | 4 + test/fixtures/upstream_nameservers/example.tf | 15 +- test/fixtures/upstream_nameservers/network.tf | 7 +- .../fixtures/upstream_nameservers/versions.tf | 4 + .../workload_metadata_config/example.tf | 16 +- .../workload_metadata_config/network.tf | 10 +- .../workload_metadata_config/versions.tf | 19 ++ variables.tf | 107 ++++--- versions.tf | 19 ++ 147 files changed, 4385 insertions(+), 2613 deletions(-) create mode 100644 autogen/versions.tf create mode 100644 examples/simple_regional_beta/versions.tf create mode 100644 examples/simple_regional_private_beta/versions.tf create mode 100644 examples/stub_domains_upstream_nameservers/versions.tf create mode 100644 examples/upstream_nameservers/versions.tf create mode 100644 examples/workload_metadata_config/versions.tf create mode 100644 modules/beta-private-cluster/versions.tf create mode 100644 modules/beta-public-cluster/versions.tf create mode 100644 modules/private-cluster/versions.tf create mode 100644 test/fixtures/stub_domains_upstream_nameservers/versions.tf create mode 100644 test/fixtures/upstream_nameservers/versions.tf create mode 100644 test/fixtures/workload_metadata_config/versions.tf create mode 100644 versions.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1269c5f0..f0948c4ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Extending the adopted spec, each change should have a link to its corresponding ## [Unreleased] +<<<<<<< HEAD ## [v3.0.0] - 2019-07-08 ### Added @@ -15,6 +16,16 @@ Extending the adopted spec, each change should have a link to its corresponding * Add configuration flag for enable BinAuthZ Admission controller [#160] [#188] * Add configuration flag for `pod_security_policy_config` [#163] [#188] * Support for a guest accelerator in node pool configuration. [#197] +======= +## [2.0.0] 2019-06-ZZ + +### Changed + +* Supported version of Terraform is 0.12. [#58] +* Add configuration flag for enable BinAuthZ Admission controller [#160] +* Add configuration flag for `pod_security_policy_config` [#163] +* Support for a guest accelerator in node pool configuration. [#157] +>>>>>>> Fixes #158: Add support for Terraform v0.12 * Support to scale the default node cluster. [#149] * Support for configuring the network policy provider. [#159] * Support for database encryption. [#165] diff --git a/Gemfile b/Gemfile index 2fffe26f1f..a54d14ec29 100644 --- a/Gemfile +++ b/Gemfile @@ -15,7 +15,7 @@ ruby "~> 2.5" source 'https://rubygems.org/' do - gem "kitchen-terraform", "~> 4.0" + gem "kitchen-terraform", "~> 4.9" gem "kubeclient", "~> 4.0" gem "rest-client", "~> 2.0" end diff --git a/Makefile b/Makefile index ccf08a8d6d..27ecbaf396 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ SHELL := /usr/bin/env bash # Docker build config variables CREDENTIALS_PATH ?= /cft/workdir/credentials.json DOCKER_ORG := gcr.io/cloud-foundation-cicd -DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 1.3.0 +DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 2.0.0 DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} DOCKER_TAG_KITCHEN_TERRAFORM ?= ${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} DOCKER_IMAGE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform_terraform-google-kubernetes-engine diff --git a/README.md b/README.md index f175e8d507..ec8cb68ce8 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ module "gke" { all = {} default-node-pool = { - default-node-pool = "true" + default-node-pool = true } } @@ -74,7 +74,7 @@ module "gke" { default-node-pool = [ { key = "default-node-pool" - value = "true" + value = true effect = "PREFER_NO_SCHEDULE" }, ] @@ -114,78 +114,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. [^]: (autogen_docs_start) - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | string | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements @@ -203,7 +131,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [Terraform](https://www.terraform.io/downloads.html) 0.12 - [Terraform Provider for GCP][terraform-provider-google] v2.9 ### Configure a Service Account diff --git a/auth.tf b/auth.tf index 5ad4160145..48e7cc6a5f 100644 --- a/auth.tf +++ b/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "google" + provider = google } /****************************************** @@ -29,6 +29,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/autogen/README.md b/autogen/README.md index e43b896ede..e66c920a78 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -70,7 +70,7 @@ module "gke" { all = {} default-node-pool = { - default-node-pool = "true" + default-node-pool = true } } @@ -88,7 +88,7 @@ module "gke" { default-node-pool = [ { key = "default-node-pool" - value = "true" + value = true effect = "PREFER_NO_SCHEDULE" }, ] @@ -145,7 +145,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [Terraform](https://www.terraform.io/downloads.html) 0.12 {% if private_cluster or beta_cluster %} - [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 {% else %} diff --git a/autogen/auth.tf b/autogen/auth.tf index a16136fccf..21275cd41e 100644 --- a/autogen/auth.tf +++ b/autogen/auth.tf @@ -20,7 +20,11 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "{% if private_cluster or beta_cluster %}google-beta{% else %}google{% endif %}" + {% if private_cluster or beta_cluster %} + provider = google-beta + {% else %} + provider = google + {% endif %} } /****************************************** @@ -29,6 +33,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index 6dcd2e01ed..c4354f92ca 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -20,78 +20,120 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "{% if private_cluster or beta_cluster %}google-beta{% else %}google{% endif %}" - count = "${var.regional ? 1 : 0}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" - - region = "${var.region}" - node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" - - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_regional}" - - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" - - {% if beta_cluster %} - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" + {% if private_cluster or beta_cluster %} + provider = google-beta + {% else %} + provider = google {% endif %} - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + + count = var.regional ? 1 : 0 + name = var.name + description = var.description + project = var.project_id + + region = var.region + + node_locations = coalescelist( + compact(var.zones), + sort(random_shuffle.available_zones.result), + ) + + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link + + dynamic "network_policy" { + for_each = local.cluster_network_policy + + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_regional + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + +{% if beta_cluster %} + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + +{% endif %} + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } {% if beta_cluster %} istio_config { - disabled = "${var.istio ? 0 : 1}" + disabled = ! var.istio } - cloudrun_config = "${local.cluster_cloudrun_config["${var.cloudrun ? "enabled" : "disabled"}"]}" + dynamic "cloudrun_config" { + for_each = local.cluster_cloudrun_config + + content { + disabled = cloudrun_config.value.disabled + } + } {% endif %} } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -102,28 +144,42 @@ resource "google_container_cluster" "primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) {% if beta_cluster %} - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } {% endif %} } } -{% if private_cluster %} +{% if private_cluster %} private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } {% endif %} - remove_default_node_pool = "${var.remove_default_node_pool}" + remove_default_node_pool = var.remove_default_node_pool {% if beta_cluster %} - database_encryption = ["${var.database_encryption}"] + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } {% endif %} } @@ -131,55 +187,115 @@ resource "google_container_cluster" "primary" { Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "google-beta" - count = "${var.regional ? length(var.node_pools) : 0}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - region = "${var.region}" - cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? length(var.node_pools) : 0 + name = var.node_pools[count.index]["name"] + project = var.project_id + region = var.region + cluster = google_container_cluster.primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_regional, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } {% if beta_cluster %} - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } {% endif %} } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -190,16 +306,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index 24ed5671e6..9f3d6c4273 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -20,79 +20,115 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "{% if private_cluster or beta_cluster %}google-beta{% else %}google{% endif %}" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + {% if private_cluster or beta_cluster %} + provider = google-beta + {% else %} + provider = google + {% endif %} - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + dynamic "network_policy" { + for_each = local.cluster_network_policy - {% if beta_cluster %} - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } - {% endif %} - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + +{% if beta_cluster %} + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + +{% endif %} + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } {% if beta_cluster %} istio_config { - disabled = "${var.istio ? 0 : 1}" + disabled = ! var.istio } - cloudrun_config = "${local.cluster_cloudrun_config["${var.cloudrun ? "enabled" : "disabled"}"]}" + dynamic "cloudrun_config" { + for_each = local.cluster_cloudrun_config + + content { + disabled = cloudrun_config.value.disabled + } + } {% endif %} } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -103,28 +139,42 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) {% if beta_cluster %} - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } {% endif %} } } -{% if private_cluster %} +{% if private_cluster %} private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } {% endif %} - remove_default_node_pool = "${var.remove_default_node_pool}" + remove_default_node_pool = var.remove_default_node_pool {% if beta_cluster %} - database_encryption = ["${var.database_encryption}"] + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } {% endif %} } @@ -132,55 +182,116 @@ resource "google_container_cluster" "zonal_primary" { Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } {% if beta_cluster %} - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } {% endif %} } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -191,16 +302,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/autogen/dns.tf b/autogen/dns.tf index 24a3f34844..65c8d99d65 100644 --- a/autogen/dns.tf +++ b/autogen/dns.tf @@ -20,73 +20,94 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config && ! local.upstream_nameservers_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + upstream_nameservers_config = length(var.upstream_nameservers) > 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + + cluster_type = var.regional ? "regional" : "zonal" + + cluster_network_policy = var.network_policy ? [{ + enabled = true + provider = var.network_policy_provider + }] : [{ + enabled = false + provider = null + }] - cluster_cloudrun_config = { - enabled = [{disabled = "false"}] - disabled = [] - } {% if beta_cluster %} + cluster_cloudrun_config = var.cloudrun ? [{disabled = false}] : [] - cluster_node_metadata_config = { - specified = [{node_metadata = "${var.node_metadata}"}] - unspecified = [] - } + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] {% endif %} cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } {% if private_cluster %} cluster_type_output_endpoint = { - regional = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.primary.*.endpoint, list("")), 0) - }" - - zonal = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0) - }" + regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) + + zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) } {% else %} cluster_type_output_endpoint = { - regional = "${element(concat(google_container_cluster.primary.*.endpoint, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.endpoint, [""]), + 0, + ) } {% endif %} cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } {% if beta_cluster %} # BETA features cluster_type_output_istio_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) } cluster_type_output_pod_security_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) } # /BETA features {% endif %} cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + cluster_network_policy_enabled = !local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = !local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = !local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = !local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] {% if beta_cluster %} # BETA features - cluster_istio_enabled = "${local.cluster_type_output_istio_enabled[local.cluster_type] ? false : true}" - cluster_cloudrun_enabled = "${var.cloudrun}" - cluster_pod_security_policy_enabled = "${local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] ? true : false}" + cluster_istio_enabled = !local.cluster_type_output_istio_enabled[local.cluster_type] + cluster_cloudrun_enabled = var.cloudrun + cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] # /BETA features {% endif %} } @@ -207,9 +296,9 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -217,7 +306,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/autogen/masq.tf b/autogen/masq.tf index 83aa443a3e..afd34e148d 100644 --- a/autogen/masq.tf +++ b/autogen/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.configure_ip_masq ? 1 : 0}" + count = var.configure_ip_masq ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = < 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -159,16 +239,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 466b81634d..13cdb99ace 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -20,66 +20,86 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + dynamic "network_policy" { + for_each = local.cluster_network_policy - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -90,65 +110,121 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } - remove_default_node_pool = "${var.remove_default_node_pool}" + + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -159,16 +235,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/dns.tf b/dns.tf index 91b41efac4..7138473ded 100644 --- a/dns.tf +++ b/dns.tf @@ -20,73 +20,94 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config && ! local.upstream_nameservers_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + upstream_nameservers_config = length(var.upstream_nameservers) > 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id - cluster_cloudrun_config = { - enabled = [{ - disabled = "false" - }] + cluster_type = var.regional ? "regional" : "zonal" - disabled = [] - } + cluster_network_policy = var.network_policy ? [{ + enabled = true + provider = var.network_policy_provider + }] : [{ + enabled = false + provider = null + }] cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } cluster_type_output_endpoint = { - regional = "${element(concat(google_container_cluster.primary.*.endpoint, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.endpoint, [""]), + 0, + ) } cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } + cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" - + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -180,7 +266,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/masq.tf b/masq.tf index 3006578627..1e9dc7791d 100644 --- a/masq.tf +++ b/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.configure_ip_masq ? 1 : 0}" + count = var.configure_ip_masq ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = <` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | string | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | string | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | string | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"UNSPECIFIED"` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements @@ -223,7 +138,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [Terraform](https://www.terraform.io/downloads.html) 0.12 - [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 ### Configure a Service Account diff --git a/modules/beta-private-cluster/auth.tf b/modules/beta-private-cluster/auth.tf index 0bbafaf4a2..c177eee5a7 100644 --- a/modules/beta-private-cluster/auth.tf +++ b/modules/beta-private-cluster/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "google-beta" + provider = google-beta } /****************************************** @@ -29,6 +29,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/modules/beta-private-cluster/cluster_regional.tf b/modules/beta-private-cluster/cluster_regional.tf index 4142486488..d26d9d9df9 100644 --- a/modules/beta-private-cluster/cluster_regional.tf +++ b/modules/beta-private-cluster/cluster_regional.tf @@ -20,74 +20,112 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google-beta" - count = "${var.regional ? 1 : 0}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta - region = "${var.region}" - node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 1 : 0 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_regional}" + region = var.region - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + node_locations = coalescelist( + compact(var.zones), + sort(random_shuffle.available_zones.result), + ) - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link + + dynamic "network_policy" { + for_each = local.cluster_network_policy + + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_regional + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } istio_config { - disabled = "${var.istio ? 0 : 1}" + disabled = ! var.istio } - cloudrun_config = "${local.cluster_cloudrun_config["${var.cloudrun ? "enabled" : "disabled"}"]}" + dynamic "cloudrun_config" { + for_each = local.cluster_cloudrun_config + + content { + disabled = cloudrun_config.value.disabled + } + } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -98,76 +136,150 @@ resource "google_container_cluster" "primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config + + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } } } private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } - remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = ["${var.database_encryption}"] + remove_default_node_pool = var.remove_default_node_pool + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } } /****************************************** Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "google-beta" - count = "${var.regional ? length(var.node_pools) : 0}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - region = "${var.region}" - cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? length(var.node_pools) : 0 + name = var.node_pools[count.index]["name"] + project = var.project_id + region = var.region + cluster = google_container_cluster.primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_regional, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + content { + node_metadata = workload_metadata_config.value.node_metadata + } } - - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -178,16 +290,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/modules/beta-private-cluster/cluster_zonal.tf b/modules/beta-private-cluster/cluster_zonal.tf index 9df66bfbc1..afbc294f0a 100644 --- a/modules/beta-private-cluster/cluster_zonal.tf +++ b/modules/beta-private-cluster/cluster_zonal.tf @@ -20,75 +20,107 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google-beta" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + dynamic "network_policy" { + for_each = local.cluster_network_policy - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + enable_binary_authorization = var.enable_binary_authorization - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } istio_config { - disabled = "${var.istio ? 0 : 1}" + disabled = ! var.istio } - cloudrun_config = "${local.cluster_cloudrun_config["${var.cloudrun ? "enabled" : "disabled"}"]}" + dynamic "cloudrun_config" { + for_each = local.cluster_cloudrun_config + + content { + disabled = cloudrun_config.value.disabled + } + } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -99,76 +131,151 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) + + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } } } private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } - remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = ["${var.database_encryption}"] + remove_default_node_pool = var.remove_default_node_pool + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } } /****************************************** Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + content { + node_metadata = workload_metadata_config.value.node_metadata + } } - - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -179,16 +286,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf index 91b41efac4..7138473ded 100644 --- a/modules/beta-private-cluster/dns.tf +++ b/modules/beta-private-cluster/dns.tf @@ -20,73 +20,94 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config && ! local.upstream_nameservers_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + upstream_nameservers_config = length(var.upstream_nameservers) > 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + + cluster_type = var.regional ? "regional" : "zonal" + + cluster_network_policy = var.network_policy ? [{ + enabled = true + provider = var.network_policy_provider + }] : [{ + enabled = false + provider = null + }] + + cluster_cloudrun_config = var.cloudrun ? [{ disabled = false }] : [] + + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } cluster_type_output_endpoint = { - regional = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.primary.*.endpoint, list("")), 0) - }" + regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0) - }" + zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) } cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } # BETA features cluster_type_output_istio_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) } cluster_type_output_pod_security_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) } - # /BETA features cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } + cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" + + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] # BETA features - cluster_istio_enabled = "${local.cluster_type_output_istio_enabled[local.cluster_type] ? false : true}" - cluster_cloudrun_enabled = "${var.cloudrun}" - cluster_pod_security_policy_enabled = "${local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] ? true : false}" - + cluster_istio_enabled = ! local.cluster_type_output_istio_enabled[local.cluster_type] + cluster_cloudrun_enabled = var.cloudrun + cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] # /BETA features } @@ -203,9 +276,9 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -213,7 +286,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/modules/beta-private-cluster/masq.tf b/modules/beta-private-cluster/masq.tf index 3006578627..1e9dc7791d 100644 --- a/modules/beta-private-cluster/masq.tf +++ b/modules/beta-private-cluster/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.configure_ip_masq ? 1 : 0}" + count = var.configure_ip_masq ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = <` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | string | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"UNSPECIFIED"` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements @@ -214,7 +133,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [Terraform](https://www.terraform.io/downloads.html) 0.12 - [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 ### Configure a Service Account diff --git a/modules/beta-public-cluster/auth.tf b/modules/beta-public-cluster/auth.tf index 0bbafaf4a2..c177eee5a7 100644 --- a/modules/beta-public-cluster/auth.tf +++ b/modules/beta-public-cluster/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "google-beta" + provider = google-beta } /****************************************** @@ -29,6 +29,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/modules/beta-public-cluster/cluster_regional.tf b/modules/beta-public-cluster/cluster_regional.tf index b651323baf..af21dc605a 100644 --- a/modules/beta-public-cluster/cluster_regional.tf +++ b/modules/beta-public-cluster/cluster_regional.tf @@ -20,74 +20,112 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google-beta" - count = "${var.regional ? 1 : 0}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta - region = "${var.region}" - node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 1 : 0 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_regional}" + region = var.region - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + node_locations = coalescelist( + compact(var.zones), + sort(random_shuffle.available_zones.result), + ) - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link + + dynamic "network_policy" { + for_each = local.cluster_network_policy + + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_regional + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } istio_config { - disabled = "${var.istio ? 0 : 1}" + disabled = ! var.istio } - cloudrun_config = "${local.cluster_cloudrun_config["${var.cloudrun ? "enabled" : "disabled"}"]}" + dynamic "cloudrun_config" { + for_each = local.cluster_cloudrun_config + + content { + disabled = cloudrun_config.value.disabled + } + } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -98,70 +136,145 @@ resource "google_container_cluster" "primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) + + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } } } - remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = ["${var.database_encryption}"] + + remove_default_node_pool = var.remove_default_node_pool + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } } /****************************************** Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "google-beta" - count = "${var.regional ? length(var.node_pools) : 0}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - region = "${var.region}" - cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? length(var.node_pools) : 0 + name = var.node_pools[count.index]["name"] + project = var.project_id + region = var.region + cluster = google_container_cluster.primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_regional, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + content { + node_metadata = workload_metadata_config.value.node_metadata + } } - - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -172,16 +285,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/modules/beta-public-cluster/cluster_zonal.tf b/modules/beta-public-cluster/cluster_zonal.tf index dca12fd9ce..c35f11818e 100644 --- a/modules/beta-public-cluster/cluster_zonal.tf +++ b/modules/beta-public-cluster/cluster_zonal.tf @@ -20,75 +20,107 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google-beta" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + dynamic "network_policy" { + for_each = local.cluster_network_policy - enable_binary_authorization = "${var.enable_binary_authorization}" - pod_security_policy_config = "${var.pod_security_policy_config}" + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + enable_binary_authorization = var.enable_binary_authorization + + dynamic "pod_security_policy_config" { + for_each = var.pod_security_policy_config + content { + enabled = pod_security_policy_config.value.enabled + } + } + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } istio_config { - disabled = "${var.istio ? 0 : 1}" + disabled = ! var.istio } - cloudrun_config = "${local.cluster_cloudrun_config["${var.cloudrun ? "enabled" : "disabled"}"]}" + dynamic "cloudrun_config" { + for_each = local.cluster_cloudrun_config + + content { + disabled = cloudrun_config.value.disabled + } + } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -99,70 +131,146 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) + + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" + content { + node_metadata = workload_metadata_config.value.node_metadata + } + } } } - remove_default_node_pool = "${var.remove_default_node_pool}" - database_encryption = ["${var.database_encryption}"] + + remove_default_node_pool = var.remove_default_node_pool + + dynamic "database_encryption" { + for_each = var.database_encryption + + content { + key_name = database_encryption.value.key_name + state = database_encryption.value.state + } + } } /****************************************** Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] + dynamic "workload_metadata_config" { + for_each = local.cluster_node_metadata_config - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + content { + node_metadata = workload_metadata_config.value.node_metadata + } } - - workload_metadata_config = "${local.cluster_node_metadata_config["${var.node_metadata == "UNSPECIFIED" ? "unspecified" : "specified"}"]}" } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -173,16 +281,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf index 91b41efac4..7138473ded 100644 --- a/modules/beta-public-cluster/dns.tf +++ b/modules/beta-public-cluster/dns.tf @@ -20,73 +20,94 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config && ! local.upstream_nameservers_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + upstream_nameservers_config = length(var.upstream_nameservers) > 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + + cluster_type = var.regional ? "regional" : "zonal" + + cluster_network_policy = var.network_policy ? [{ + enabled = true + provider = var.network_policy_provider + }] : [{ + enabled = false + provider = null + }] + + cluster_cloudrun_config = var.cloudrun ? [{ disabled = false }] : [] + + cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ + node_metadata = var.node_metadata + }] cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } cluster_type_output_endpoint = { - regional = "${element(concat(google_container_cluster.primary.*.endpoint, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.endpoint, [""]), + 0, + ) } cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } # BETA features cluster_type_output_istio_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) } cluster_type_output_pod_security_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) + zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) } - # /BETA features cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } + cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" + + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] # BETA features - cluster_istio_enabled = "${local.cluster_type_output_istio_enabled[local.cluster_type] ? false : true}" - cluster_cloudrun_enabled = "${var.cloudrun}" - cluster_pod_security_policy_enabled = "${local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] ? true : false}" - + cluster_istio_enabled = ! local.cluster_type_output_istio_enabled[local.cluster_type] + cluster_cloudrun_enabled = var.cloudrun + cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] # /BETA features } @@ -194,9 +278,9 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -204,7 +288,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/modules/beta-public-cluster/masq.tf b/modules/beta-public-cluster/masq.tf index 3006578627..1e9dc7791d 100644 --- a/modules/beta-public-cluster/masq.tf +++ b/modules/beta-public-cluster/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.configure_ip_masq ? 1 : 0}" + count = var.configure_ip_masq ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = <
### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements @@ -212,7 +136,7 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.11.x +- [Terraform](https://www.terraform.io/downloads.html) 0.12 - [Terraform Provider for GCP Beta][terraform-provider-google-beta] v2.9 ### Configure a Service Account diff --git a/modules/private-cluster/auth.tf b/modules/private-cluster/auth.tf index 0bbafaf4a2..c177eee5a7 100644 --- a/modules/private-cluster/auth.tf +++ b/modules/private-cluster/auth.tf @@ -20,7 +20,7 @@ Retrieve authentication token *****************************************/ data "google_client_config" "default" { - provider = "google-beta" + provider = google-beta } /****************************************** @@ -29,6 +29,6 @@ data "google_client_config" "default" { provider "kubernetes" { load_config_file = false host = "https://${local.cluster_endpoint}" - token = "${data.google_client_config.default.access_token}" - cluster_ca_certificate = "${base64decode(local.cluster_ca_certificate)}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(local.cluster_ca_certificate) } diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index 141c9b9445..f42ab76dab 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -20,66 +20,91 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google-beta" - count = "${var.regional ? 1 : 0}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta - region = "${var.region}" - node_locations = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 1 : 0 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_regional}" + region = var.region - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + node_locations = coalescelist( + compact(var.zones), + sort(random_shuffle.available_zones.result), + ) - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link + + dynamic "network_policy" { + for_each = local.cluster_network_policy + + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_regional + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -90,71 +115,125 @@ resource "google_container_cluster" "primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } - remove_default_node_pool = "${var.remove_default_node_pool}" + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - provider = "google-beta" - count = "${var.regional ? length(var.node_pools) : 0}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - region = "${var.region}" - cluster = "${google_container_cluster.primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_regional)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? length(var.node_pools) : 0 + name = var.node_pools[count.index]["name"] + project = var.project_id + region = var.region + cluster = google_container_cluster.primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_regional, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -165,16 +244,19 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_regional_cluster" { - count = "${var.regional ? 1 : 0}" + count = var.regional ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.primary", "google_container_node_pool.pools"] + depends_on = [ + google_container_cluster.primary, + google_container_node_pool.pools, + ] } diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index 1ee89f7e93..9b44c6d478 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -20,66 +20,86 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google-beta" - count = "${var.regional ? 0 : 1}" - name = "${var.name}" - description = "${var.description}" - project = "${var.project_id}" + provider = google-beta - zone = "${var.zones[0]}" - node_locations = ["${slice(var.zones,1,length(var.zones))}"] - cluster_ipv4_cidr = "${var.cluster_ipv4_cidr}" - network = "${replace(data.google_compute_network.gke_network.self_link, "https://www.googleapis.com/compute/v1/", "")}" - network_policy = "${local.cluster_network_policy["${var.network_policy ? "enabled" : "disabled"}"]}" + count = var.regional ? 0 : 1 + name = var.name + description = var.description + project = var.project_id - subnetwork = "${replace(data.google_compute_subnetwork.gke_subnetwork.self_link, "https://www.googleapis.com/compute/v1/", "")}" - min_master_version = "${local.kubernetes_version_zonal}" + zone = var.zones[0] + node_locations = slice(var.zones, 1, length(var.zones)) + cluster_ipv4_cidr = var.cluster_ipv4_cidr + network = data.google_compute_network.gke_network.self_link - logging_service = "${var.logging_service}" - monitoring_service = "${var.monitoring_service}" + dynamic "network_policy" { + for_each = local.cluster_network_policy - master_authorized_networks_config = ["${var.master_authorized_networks_config}"] + content { + enabled = network_policy.value.enabled + provider = network_policy.value.provider + } + } + + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link + min_master_version = local.kubernetes_version_zonal + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + dynamic "master_authorized_networks_config" { + for_each = var.master_authorized_networks_config + content { + dynamic "cidr_blocks" { + for_each = master_authorized_networks_config.value.cidr_blocks + content { + cidr_block = lookup(cidr_blocks.value, "cidr_block", "") + display_name = lookup(cidr_blocks.value, "display_name", "") + } + } + } + } master_auth { - username = "${var.basic_auth_username}" - password = "${var.basic_auth_password}" + username = var.basic_auth_username + password = var.basic_auth_password client_certificate_config { - issue_client_certificate = "${var.issue_client_certificate}" + issue_client_certificate = var.issue_client_certificate } } addons_config { http_load_balancing { - disabled = "${var.http_load_balancing ? 0 : 1}" + disabled = ! var.http_load_balancing } horizontal_pod_autoscaling { - disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + disabled = ! var.horizontal_pod_autoscaling } kubernetes_dashboard { - disabled = "${var.kubernetes_dashboard ? 0 : 1}" + disabled = ! var.kubernetes_dashboard } network_policy_config { - disabled = "${var.network_policy ? 0 : 1}" + disabled = ! var.network_policy } } ip_allocation_policy { - cluster_secondary_range_name = "${var.ip_range_pods}" - services_secondary_range_name = "${var.ip_range_services}" + cluster_secondary_range_name = var.ip_range_pods + services_secondary_range_name = var.ip_range_services } maintenance_policy { daily_maintenance_window { - start_time = "${var.maintenance_start_time}" + start_time = var.maintenance_start_time } } lifecycle { - ignore_changes = ["node_pool"] + ignore_changes = [node_pool] } timeouts { @@ -90,71 +110,126 @@ resource "google_container_cluster" "zonal_primary" { node_pool { name = "default-pool" - initial_node_count = "${var.initial_node_count}" + initial_node_count = var.initial_node_count node_config { - service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" + service_account = lookup(var.node_pools[0], "service_account", local.service_account) } } private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block } - remove_default_node_pool = "${var.remove_default_node_pool}" + remove_default_node_pool = var.remove_default_node_pool } /****************************************** Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - provider = "google-beta" - count = "${var.regional ? 0 : length(var.node_pools)}" - name = "${lookup(var.node_pools[count.index], "name")}" - project = "${var.project_id}" - zone = "${var.zones[0]}" - cluster = "${google_container_cluster.zonal_primary.name}" - version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version_zonal)}" - initial_node_count = "${lookup(var.node_pools[count.index], "initial_node_count", lookup(var.node_pools[count.index], "min_count", 1))}" + provider = google-beta + count = var.regional ? 0 : length(var.node_pools) + name = var.node_pools[count.index]["name"] + project = var.project_id + zone = var.zones[0] + cluster = google_container_cluster.zonal_primary[0].name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( + var.node_pools[count.index], + "version", + local.node_version_zonal, + ) + initial_node_count = lookup( + var.node_pools[count.index], + "initial_node_count", + lookup(var.node_pools[count.index], "min_count", 1), + ) autoscaling { - min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" - max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + min_node_count = lookup(var.node_pools[count.index], "min_count", 1) + max_node_count = lookup(var.node_pools[count.index], "max_count", 100) } management { - auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" - auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) } node_config { - image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" - machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" - labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" - metadata = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_metadata["all"], var.node_pools_metadata[lookup(var.node_pools[count.index], "name")], map("disable-legacy-endpoints", var.disable_legacy_metadata_endpoints))}" - taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" - tags = ["${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}"] - - disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" - disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" - service_account = "${lookup(var.node_pools[count.index], "service_account", local.service_account)}" - preemptible = "${lookup(var.node_pools[count.index], "preemptible", false)}" - - oauth_scopes = [ - "${concat(var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", - ] - - guest_accelerator { - type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" - count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + image_type = lookup(var.node_pools[count.index], "image_type", "COS") + machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") + labels = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_labels["all"], + var.node_pools_labels[var.node_pools[count.index]["name"]], + ) + metadata = merge( + { + "cluster_name" = var.name + }, + { + "node_pool" = var.node_pools[count.index]["name"] + }, + var.node_pools_metadata["all"], + var.node_pools_metadata[var.node_pools[count.index]["name"]], + { + "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints + }, + ) + dynamic "taint" { + for_each = concat( + var.node_pools_taints["all"], + var.node_pools_taints[var.node_pools[count.index]["name"]], + ) + content { + effect = taint.value.effect + key = taint.value.key + value = taint.value.value + } + } + + tags = concat( + ["gke-${var.name}"], + ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + var.node_pools_tags["all"], + var.node_pools_tags[var.node_pools[count.index]["name"]], + ) + + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( + var.node_pools[count.index], + "service_account", + local.service_account, + ) + preemptible = lookup(var.node_pools[count.index], "preemptible", false) + + oauth_scopes = concat( + var.node_pools_oauth_scopes["all"], + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + ) + + dynamic "guest_accelerator" { + for_each = lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ + type = lookup(var.node_pools[count.index], "accelerator_type", "") + count = lookup(var.node_pools[count.index], "accelerator_count", 0) + }] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } } } lifecycle { - ignore_changes = ["initial_node_count"] + ignore_changes = [initial_node_count] } timeouts { @@ -165,16 +240,19 @@ resource "google_container_node_pool" "zonal_pools" { } resource "null_resource" "wait_for_zonal_cluster" { - count = "${var.regional ? 0 : 1}" + count = var.regional ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" } - depends_on = ["google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index 91b41efac4..7138473ded 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -20,73 +20,94 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = "${local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" } - depends_on = ["data.google_client_config.default", "google_container_cluster.primary", "google_container_node_pool.pools", "google_container_cluster.zonal_primary", "google_container_node_pool.zonal_pools"] + depends_on = [ + data.google_client_config.default, + google_container_cluster.primary, + google_container_node_pool.pools, + google_container_cluster.zonal_primary, + google_container_node_pool.zonal_pools, + ] } /****************************************** Create kube-dns confimap *****************************************/ resource "kubernetes_config_map" "kube-dns" { - count = "${local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0}" + count = local.custom_kube_dns_config && ! local.upstream_nameservers_config ? 1 : 0 metadata { name = "kube-dns" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { stubDomains = < 0 + upstream_nameservers_config = length(var.upstream_nameservers) > 0 + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + + cluster_type = var.regional ? "regional" : "zonal" + + cluster_network_policy = var.network_policy ? [{ + enabled = true + provider = var.network_policy_provider + }] : [{ + enabled = false + provider = null + }] cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.name, [""]), + 0, + ) } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = element( + concat(google_container_cluster.zonal_primary.*.zone, [""]), + 0, + ) } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) + zonal = var.region } - cluster_type_output_regional_zones = "${flatten(google_container_cluster.primary.*.node_locations)}" - cluster_type_output_zonal_zones = "${slice(var.zones, 1, length(var.zones))}" + cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) + cluster_type_output_zonal_zones = slice(var.zones, 1, length(var.zones)) cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones)}" + regional = local.cluster_type_output_regional_zones + zonal = concat( + google_container_cluster.zonal_primary.*.zone, + local.cluster_type_output_zonal_zones, + ) } cluster_type_output_endpoint = { - regional = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.primary.*.endpoint, list("")), 0) - }" + regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = "${ - var.deploy_using_private_endpoint ? - element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, list("")), 0) : - element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0) - }" + zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) } cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = concat(google_container_cluster.primary.*.master_auth, []) + zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.master_version, + [""], + ), + 0, + ) } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.min_master_version, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.min_master_version, + [""], + ), + 0, + ) } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.logging_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.logging_service, + [""], + ), + 0, + ) } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = element( + concat(google_container_cluster.primary.*.monitoring_service, [""]), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.monitoring_service, + [""], + ), + 0, + ) } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, + [""], + ), + 0, + ) } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = element( + concat( + google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) + zonal = element( + concat( + google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, + [""], + ), + 0, + ) } + cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = concat(google_container_node_pool.pools.*.name, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) } cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = concat(google_container_node_pool.pools.*.version, [""]) + zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" - cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" - cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" - + cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] + cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" - cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_name = local.cluster_type_output_name[local.cluster_type] + cluster_location = local.cluster_type_output_location[local.cluster_type] + cluster_region = local.cluster_type_output_region[local.cluster_type] + cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) + cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] + cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] + cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] + cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] + cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] + cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] + cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] + cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] + cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] + cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = "google-beta" - region = "${var.region}" - project = "${var.project_id}" + provider = google-beta + region = var.region + project = var.project_id } data "google_container_engine_versions" "zone" { @@ -189,7 +264,7 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = "${var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0]}" + zone = var.zones[0] == "" ? data.google_compute_zones.available.names[0] : var.zones[0] - project = "${var.project_id}" + project = var.project_id } diff --git a/modules/private-cluster/masq.tf b/modules/private-cluster/masq.tf index 3006578627..1e9dc7791d 100644 --- a/modules/private-cluster/masq.tf +++ b/modules/private-cluster/masq.tf @@ -20,18 +20,18 @@ Create ip-masq-agent confimap *****************************************/ resource "kubernetes_config_map" "ip-masq-agent" { - count = "${var.configure_ip_masq ? 1 : 0}" + count = var.configure_ip_masq ? 1 : 0 metadata { name = "ip-masq-agent" namespace = "kube-system" - labels { + labels = { maintained_by = "terraform" } } - data { + data = { config = <