From 687703b8b91d3fa3b8eaebed0b2ad68e0be92ccc Mon Sep 17 00:00:00 2001 From: Andriy Kopachevskyy Date: Wed, 31 Jul 2019 17:35:51 +0300 Subject: [PATCH] Refactor/flatten duplicate resources #161 Collapsed regional and zonal clusters into one cluster with location attribute instead usage of count, zones and regions. Selection of latest nodes and master version same both for regional and zonal/multi-zonal clusters. For Master version in case or regional cluster following code check latest version by location (region), and for zonal, location is var.zone[0]: latest_master_version = data.google_container_engine_versions.master.latest_master_version For Node version in case of regional cluster used latest master version of first of available zones, and in case of zonal cluster - latest master version of var.zones[0] zone location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] Fixes terraform-google-modules/terraform-google-kubernetes-engine#161 --- Makefile | 6 +- autogen/{cluster_regional.tf => cluster.tf} | 32 +- autogen/cluster_zonal.tf | 328 --------- autogen/dns.tf | 8 - autogen/main.tf | 303 ++------ autogen/masq.tf | 2 - autogen/outputs.tf | 2 - autogen/sa.tf | 2 +- cluster_regional.tf => cluster.tf | 31 +- cluster_zonal.tf | 252 ------- dns.tf | 8 - helpers/terraform_docs | 694 ++++++++++++++++++ helpers/terraform_validate | 23 + main.tf | 253 ++----- masq.tf | 2 - .../{cluster_regional.tf => cluster.tf} | 30 +- modules/beta-private-cluster/cluster_zonal.tf | 310 -------- modules/beta-private-cluster/dns.tf | 8 - modules/beta-private-cluster/main.tf | 281 ++----- modules/beta-private-cluster/masq.tf | 2 - modules/beta-private-cluster/outputs.tf | 2 - modules/beta-private-cluster/sa.tf | 2 +- .../{cluster_regional.tf => cluster.tf} | 31 +- modules/beta-public-cluster/cluster_zonal.tf | 305 -------- modules/beta-public-cluster/dns.tf | 8 - modules/beta-public-cluster/main.tf | 283 ++----- modules/beta-public-cluster/masq.tf | 2 - modules/beta-public-cluster/outputs.tf | 2 - modules/beta-public-cluster/sa.tf | 2 +- .../{cluster_regional.tf => cluster.tf} | 30 +- modules/private-cluster/cluster_zonal.tf | 257 ------- modules/private-cluster/dns.tf | 8 - modules/private-cluster/main.tf | 251 ++----- modules/private-cluster/masq.tf | 2 - modules/private-cluster/outputs.tf | 2 - modules/private-cluster/sa.tf | 2 +- outputs.tf | 2 - sa.tf | 2 +- test/fixtures/simple_regional/example.tf | 1 - test/make.sh | 57 +- 40 files changed, 1113 insertions(+), 2715 deletions(-) rename autogen/{cluster_regional.tf => cluster.tf} (93%) delete mode 100644 autogen/cluster_zonal.tf rename cluster_regional.tf => cluster.tf (92%) delete mode 100644 cluster_zonal.tf create mode 100755 helpers/terraform_docs create mode 100755 helpers/terraform_validate rename modules/beta-private-cluster/{cluster_regional.tf => cluster.tf} (93%) delete mode 100644 modules/beta-private-cluster/cluster_zonal.tf rename modules/beta-public-cluster/{cluster_regional.tf => cluster.tf} (93%) delete mode 100644 modules/beta-public-cluster/cluster_zonal.tf rename modules/private-cluster/{cluster_regional.tf => cluster.tf} (92%) delete mode 100644 modules/private-cluster/cluster_zonal.tf diff --git a/Makefile b/Makefile index 95abe1e74d..21d7a2764f 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCK all: check generate_docs .PHONY: check -check: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace check_generate check_generate_docs +check: check_shell check_python check_golang check_terraform check_base_files test_check_headers check_headers check_trailing_whitespace check_generate check_generate_docs # The .PHONY directive tells make that this isn't a real target and so # the presence of a file named 'check_shell' won't cause this target to stop @@ -47,10 +47,6 @@ check_golang: check_terraform: @source test/make.sh && check_terraform -.PHONY: check_docker -check_docker: - @source test/make.sh && docker - .PHONY: check_base_files check_base_files: @source test/make.sh && basefiles diff --git a/autogen/cluster_regional.tf b/autogen/cluster.tf similarity index 93% rename from autogen/cluster_regional.tf rename to autogen/cluster.tf index 1c20d5c940..87abe433ae 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster.tf @@ -17,7 +17,7 @@ {{ autogeneration_note }} /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { {% if private_cluster or beta_cluster %} @@ -26,18 +26,13 @@ resource "google_container_cluster" "primary" { provider = google {% endif %} - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -51,7 +46,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -173,8 +168,8 @@ resource "google_container_cluster" "primary" { enable_private_nodes = var.enable_private_nodes master_ipv4_cidr_block = var.master_ipv4_cidr_block } - {% endif %} + remove_default_node_pool = var.remove_default_node_pool {% if beta_cluster %} @@ -190,19 +185,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -220,7 +215,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -278,7 +273,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -313,8 +308,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf deleted file mode 100644 index 0ed6af6b5c..0000000000 --- a/autogen/cluster_zonal.tf +++ /dev/null @@ -1,328 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -{{ autogeneration_note }} - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - {% if private_cluster or beta_cluster %} - provider = google-beta - {% else %} - provider = google - {% endif %} - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - -{% if beta_cluster %} - enable_binary_authorization = var.enable_binary_authorization - enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node - - vertical_pod_autoscaling { - enabled = var.enable_vertical_pod_autoscaling - } - - dynamic "pod_security_policy_config" { - for_each = var.pod_security_policy_config - content { - enabled = pod_security_policy_config.value.enabled - } - } - -{% endif %} - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - {% if beta_cluster %} - - istio_config { - disabled = ! var.istio - } - - dynamic "cloudrun_config" { - for_each = local.cluster_cloudrun_config - - content { - disabled = cloudrun_config.value.disabled - } - } - {% endif %} - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - {% if beta_cluster %} - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - {% endif %} - } - } - -{% if private_cluster %} - private_cluster_config { - enable_private_endpoint = var.enable_private_endpoint - enable_private_nodes = var.enable_private_nodes - master_ipv4_cidr_block = var.master_ipv4_cidr_block - } -{% endif %} - - remove_default_node_pool = var.remove_default_node_pool -{% if beta_cluster %} - - dynamic "database_encryption" { - for_each = var.database_encryption - - content { - key_name = database_encryption.value.key_name - state = database_encryption.value.state - } - } -{% endif %} -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - {% if beta_cluster %} - max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - {% endif %} - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - {% if beta_cluster %} - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - {% endif %} - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/autogen/dns.tf b/autogen/dns.tf index 65c8d99d65..d9d4a35395 100644 --- a/autogen/dns.tf +++ b/autogen/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/autogen/main.tf b/autogen/main.tf index 5603e3b54a..170b1570a9 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -26,8 +26,8 @@ data "google_compute_zones" "available" { provider = google {% endif %} - project = var.project_id - region = var.region + project = var.project_id + region = var.region } resource "random_shuffle" "available_zones" { @@ -36,271 +36,106 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider - }] : [{ - enabled = false + }] : [{ + enabled = false provider = null }] {% if beta_cluster %} - cluster_cloudrun_config = var.cloudrun ? [{disabled = false}] : [] + cluster_cloudrun_config = var.cloudrun ? [{ disabled = false }] : [] cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ node_metadata = var.node_metadata }] {% endif %} - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones {% if private_cluster %} - cluster_type_output_endpoint = { - regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - - zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) - } + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint {% else %} - cluster_type_output_endpoint = { - regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.endpoint, [""]), - 0, - ) - } + cluster_output_endpoint = google_container_cluster.primary.endpoint {% endif %} - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } - - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } - - {% if beta_cluster %} +{% if beta_cluster %} # BETA features - cluster_type_output_istio_enabled = { - regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - } - - cluster_type_output_pod_security_policy_enabled = { - regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) - } + cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled - cluster_type_output_intranode_visbility_enabled = { - regional = element(concat(google_container_cluster.primary.*.enable_intranode_visibility, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.enable_intranode_visibility, [""]), 0) - } - - cluster_type_output_vertical_pod_autoscaling_enabled = { - regional = element(concat(google_container_cluster.primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - } # /BETA features {% endif %} - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } - - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = !local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = !local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = !local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = !local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled {% if beta_cluster %} # BETA features - cluster_istio_enabled = !local.cluster_type_output_istio_enabled[local.cluster_type] - cluster_cloudrun_enabled = var.cloudrun - cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] - cluster_intranode_visibility_enabled = local.cluster_type_output_intranode_visbility_enabled[local.cluster_type] - cluster_vertical_pod_autoscaling_enabled = local.cluster_type_output_vertical_pod_autoscaling_enabled[local.cluster_type] + cluster_istio_enabled = ! local.cluster_output_istio_enabled + cluster_cloudrun_enabled = var.cloudrun + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled # /BETA features {% endif %} } @@ -309,8 +144,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -319,7 +153,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/autogen/masq.tf b/autogen/masq.tf index afd34e148d..6deab757c3 100644 --- a/autogen/masq.tf +++ b/autogen/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/autogen/outputs.tf b/autogen/outputs.tf index 4285a37a62..ff8eab1bef 100644 --- a/autogen/outputs.tf +++ b/autogen/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/autogen/sa.tf b/autogen/sa.tf index 6fbc2dfb51..c5f7f132a1 100644 --- a/autogen/sa.tf +++ b/autogen/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/cluster_regional.tf b/cluster.tf similarity index 92% rename from cluster_regional.tf rename to cluster.tf index f6afe63d49..499e689a4b 100644 --- a/cluster_regional.tf +++ b/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -122,23 +117,24 @@ resource "google_container_cluster" "primary" { } } + remove_default_node_pool = var.remove_default_node_pool } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -153,7 +149,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -211,7 +207,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -236,8 +232,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/cluster_zonal.tf b/cluster_zonal.tf deleted file mode 100644 index 36cec81b3f..0000000000 --- a/cluster_zonal.tf +++ /dev/null @@ -1,252 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - } - } - - - remove_default_node_pool = var.remove_default_node_pool -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/dns.tf b/dns.tf index 7138473ded..b240a23e65 100644 --- a/dns.tf +++ b/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/helpers/terraform_docs b/helpers/terraform_docs new file mode 100755 index 0000000000..c33230959b --- /dev/null +++ b/helpers/terraform_docs @@ -0,0 +1,694 @@ +#!/usr/bin/env bash + +set -e + +main() { + declare argv + argv=$(getopt -o a: --long args: -- "$@") || return + eval "set -- $argv" + + declare args + declare files + + for argv; do + case $argv in + (-a|--args) + shift + args="$1" + shift + ;; + (--) + shift + files="$@" + break + ;; + esac + done + + local hack_terraform_docs=$(terraform version | head -1 | grep -c 0.12) + + if [[ "$hack_terraform_docs" == "1" ]]; then + which awk 2>&1 >/dev/null || ( echo "awk is required for terraform-docs hack to work with Terraform 0.12"; exit 1) + + tmp_file_awk=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") + terraform_docs_awk "$tmp_file_awk" + terraform_docs "$tmp_file_awk" "$args" "$files" + rm -f "$tmp_file_awk" + else + terraform_docs "0" "$args" "$files" + fi + +} + +terraform_docs() { + readonly terraform_docs_awk_file="$1" + readonly args="$2" + readonly files="$3" + + declare -a paths + declare -a tfvars_files + + index=0 + + for file_with_path in $files; do + file_with_path="${file_with_path// /__REPLACED__SPACE__}" + + paths[index]=$(dirname "$file_with_path") + + if [[ "$file_with_path" == *".tfvars" ]]; then + tfvars_files+=("$file_with_path") + fi + + ((index+=1)) + done + + readonly tmp_file=$(mktemp) + readonly text_file="README.md" + + for path_uniq in $(echo "${paths[*]}" | tr ' ' '\n' | sort -u); do + path_uniq="${path_uniq//__REPLACED__SPACE__/ }" + + pushd "$path_uniq" > /dev/null + + if [[ ! -f "$text_file" ]]; then + popd > /dev/null + continue + fi + + if [[ "$terraform_docs_awk_file" == "0" ]]; then + terraform-docs $args md ./ > "$tmp_file" + else + # Can't append extension for mktemp, so renaming instead + tmp_file_docs=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") + mv "$tmp_file_docs" "$tmp_file_docs.tf" + tmp_file_docs_tf="$tmp_file_docs.tf" + + awk -f "$terraform_docs_awk_file" ./*.tf > "$tmp_file_docs_tf" + terraform-docs $args md "$tmp_file_docs_tf" > "$tmp_file" + rm -f "$tmp_file_docs_tf" + fi + + # Replace content between markers with the placeholder - https://stackoverflow.com/questions/1212799/how-do-i-extract-lines-between-two-line-delimiters-in-perl#1212834 + perl -i -ne 'if (/BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/../END OF PRE-COMMIT-TERRAFORM DOCS HOOK/) { print $_ if /BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/; print "I_WANT_TO_BE_REPLACED\n$_" if /END OF PRE-COMMIT-TERRAFORM DOCS HOOK/;} else { print $_ }' "$text_file" + + # Replace placeholder with the content of the file + perl -i -e 'open(F, "'"$tmp_file"'"); $f = join "", ; while(<>){if (/I_WANT_TO_BE_REPLACED/) {print $f} else {print $_};}' "$text_file" + + rm -f "$tmp_file" + + popd > /dev/null + done +} + +terraform_docs_awk() { + readonly output_file=$1 + + cat <<"EOF" > $output_file +# This script converts Terraform 0.12 variables/outputs to something suitable for `terraform-docs` +# As of terraform-docs v0.6.0, HCL2 is not supported. This script is a *dirty hack* to get around it. +# https://github.com/segmentio/terraform-docs/ +# https://github.com/segmentio/terraform-docs/issues/62 + +# Script was originally found here: https://github.com/cloudposse/build-harness/blob/master/bin/terraform-docs.awk + +{ + if ( $0 ~ /\{/ ) { + braceCnt++ + } + + if ( $0 ~ /\}/ ) { + braceCnt-- + } + + # [START] variable or output block started + if ($0 ~ /^[[:space:]]*(variable|output)[[:space:]][[:space:]]*"(.*?)"/) { + # Normalize the braceCnt (should be 1 now) + braceCnt = 1 + # [CLOSE] "default" block + if (blockDefCnt > 0) { + blockDefCnt = 0 + } + blockCnt++ + print $0 + } + + # [START] multiline default statement started + if (blockCnt > 0) { + if ($0 ~ /^[[:space:]][[:space:]]*(default)[[:space:]][[:space:]]*=/) { + if ($3 ~ "null") { + print " default = \"null\"" + } else { + print $0 + blockDefCnt++ + blockDefStart=1 + } + } + } + + # [PRINT] single line "description" + if (blockCnt > 0) { + if (blockDefCnt == 0) { + if ($0 ~ /^[[:space:]][[:space:]]*description[[:space:]][[:space:]]*=/) { + # [CLOSE] "default" block + if (blockDefCnt > 0) { + blockDefCnt = 0 + } + print $0 + } + } + } + + # [PRINT] single line "type" + if (blockCnt > 0) { + if ($0 ~ /^[[:space:]][[:space:]]*type[[:space:]][[:space:]]*=/ ) { + # [CLOSE] "default" block + if (blockDefCnt > 0) { + blockDefCnt = 0 + } + type=$3 + if (type ~ "object") { + print " type = \"object\"" + } else { + # legacy quoted types: "string", "list", and "map" + if ($3 ~ /^[[:space:]]*"(.*?)"[[:space:]]*$/) { + print " type = " $3 + } else { + print " type = \"" $3 "\"" + } + } + } + } + + # [CLOSE] variable/output block + if (blockCnt > 0) { + if (braceCnt == 0 && blockCnt > 0) { + blockCnt-- + print $0 + } + } + + # [PRINT] Multiline "default" statement + if (blockCnt > 0 && blockDefCnt > 0) { + if (blockDefStart == 1) { + blockDefStart = 0 + } else { + print $0 + } + } +} +EOF + +} + +getopt() { + # pure-getopt, a drop-in replacement for GNU getopt in pure Bash. + # version 1.4.3 + # + # Copyright 2012-2018 Aron Griffis + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be included + # in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + _getopt_main() { + # Returns one of the following statuses: + # 0 success + # 1 error parsing parameters + # 2 error in getopt invocation + # 3 internal error + # 4 reserved for -T + # + # For statuses 0 and 1, generates normalized and shell-quoted + # "options -- parameters" on stdout. + + declare parsed status + declare short long name flags + declare have_short=false + + # Synopsis from getopt man-page: + # + # getopt optstring parameters + # getopt [options] [--] optstring parameters + # getopt [options] -o|--options optstring [options] [--] parameters + # + # The first form can be normalized to the third form which + # _getopt_parse() understands. The second form can be recognized after + # first parse when $short hasn't been set. + + if [[ -n ${GETOPT_COMPATIBLE+isset} || $1 == [^-]* ]]; then + # Enable compatibility mode + flags=c$flags + # Normalize first to third synopsis form + set -- -o "$1" -- "${@:2}" + fi + + # First parse always uses flags=p since getopt always parses its own + # arguments effectively in this mode. + parsed=$(_getopt_parse getopt ahl:n:o:qQs:TuV \ + alternative,help,longoptions:,name:,options:,quiet,quiet-output,shell:,test,version \ + p "$@") + status=$? + if [[ $status != 0 ]]; then + if [[ $status == 1 ]]; then + echo "Try \`getopt --help' for more information." >&2 + # Since this is the first parse, convert status 1 to 2 + status=2 + fi + return $status + fi + eval "set -- $parsed" + + while [[ $# -gt 0 ]]; do + case $1 in + (-a|--alternative) + flags=a$flags ;; + + (-h|--help) + _getopt_help + return 2 # as does GNU getopt + ;; + + (-l|--longoptions) + long="$long${long:+,}$2" + shift ;; + + (-n|--name) + name=$2 + shift ;; + + (-o|--options) + short=$2 + have_short=true + shift ;; + + (-q|--quiet) + flags=q$flags ;; + + (-Q|--quiet-output) + flags=Q$flags ;; + + (-s|--shell) + case $2 in + (sh|bash) + flags=${flags//t/} ;; + (csh|tcsh) + flags=t$flags ;; + (*) + echo 'getopt: unknown shell after -s or --shell argument' >&2 + echo "Try \`getopt --help' for more information." >&2 + return 2 ;; + esac + shift ;; + + (-u|--unquoted) + flags=u$flags ;; + + (-T|--test) + return 4 ;; + + (-V|--version) + echo "pure-getopt 1.4.3" + return 0 ;; + + (--) + shift + break ;; + esac + + shift + done + + if ! $have_short; then + # $short was declared but never set, not even to an empty string. + # This implies the second form in the synopsis. + if [[ $# == 0 ]]; then + echo 'getopt: missing optstring argument' >&2 + echo "Try \`getopt --help' for more information." >&2 + return 2 + fi + short=$1 + have_short=true + shift + fi + + if [[ $short == -* ]]; then + # Leading dash means generate output in place rather than reordering, + # unless we're already in compatibility mode. + [[ $flags == *c* ]] || flags=i$flags + short=${short#?} + elif [[ $short == +* ]]; then + # Leading plus means POSIXLY_CORRECT, unless we're already in + # compatibility mode. + [[ $flags == *c* ]] || flags=p$flags + short=${short#?} + fi + + # This should fire if POSIXLY_CORRECT is in the environment, even if + # it's an empty string. That's the difference between :+ and + + flags=${POSIXLY_CORRECT+p}$flags + + _getopt_parse "${name:-getopt}" "$short" "$long" "$flags" "$@" + } + + _getopt_parse() { + # Inner getopt parser, used for both first parse and second parse. + # Returns 0 for success, 1 for error parsing, 3 for internal error. + # In the case of status 1, still generates stdout with whatever could + # be parsed. + # + # $flags is a string of characters with the following meanings: + # a - alternative parsing mode + # c - GETOPT_COMPATIBLE + # i - generate output in place rather than reordering + # p - POSIXLY_CORRECT + # q - disable error reporting + # Q - disable normal output + # t - quote for csh/tcsh + # u - unquoted output + + declare name="$1" short="$2" long="$3" flags="$4" + shift 4 + + # Split $long on commas, prepend double-dashes, strip colons; + # for use with _getopt_resolve_abbrev + declare -a longarr + _getopt_split longarr "$long" + longarr=( "${longarr[@]/#/--}" ) + longarr=( "${longarr[@]%:}" ) + longarr=( "${longarr[@]%:}" ) + + # Parse and collect options and parameters + declare -a opts params + declare o alt_recycled=false error=0 + + while [[ $# -gt 0 ]]; do + case $1 in + (--) + params=( "${params[@]}" "${@:2}" ) + break ;; + + (--*=*) + o=${1%%=*} + if ! o=$(_getopt_resolve_abbrev "$o" "${longarr[@]}"); then + error=1 + elif [[ ,"$long", == *,"${o#--}"::,* ]]; then + opts=( "${opts[@]}" "$o" "${1#*=}" ) + elif [[ ,"$long", == *,"${o#--}":,* ]]; then + opts=( "${opts[@]}" "$o" "${1#*=}" ) + elif [[ ,"$long", == *,"${o#--}",* ]]; then + if $alt_recycled; then o=${o#-}; fi + _getopt_err "$name: option '$o' doesn't allow an argument" + error=1 + else + echo "getopt: assertion failed (1)" >&2 + return 3 + fi + alt_recycled=false + ;; + + (--?*) + o=$1 + if ! o=$(_getopt_resolve_abbrev "$o" "${longarr[@]}"); then + error=1 + elif [[ ,"$long", == *,"${o#--}",* ]]; then + opts=( "${opts[@]}" "$o" ) + elif [[ ,"$long", == *,"${o#--}::",* ]]; then + opts=( "${opts[@]}" "$o" '' ) + elif [[ ,"$long", == *,"${o#--}:",* ]]; then + if [[ $# -ge 2 ]]; then + shift + opts=( "${opts[@]}" "$o" "$1" ) + else + if $alt_recycled; then o=${o#-}; fi + _getopt_err "$name: option '$o' requires an argument" + error=1 + fi + else + echo "getopt: assertion failed (2)" >&2 + return 3 + fi + alt_recycled=false + ;; + + (-*) + if [[ $flags == *a* ]]; then + # Alternative parsing mode! + # Try to handle as a long option if any of the following apply: + # 1. There's an equals sign in the mix -x=3 or -xy=3 + # 2. There's 2+ letters and an abbreviated long match -xy + # 3. There's a single letter and an exact long match + # 4. There's a single letter and no short match + o=${1::2} # temp for testing #4 + if [[ $1 == *=* || $1 == -?? || \ + ,$long, == *,"${1#-}"[:,]* || \ + ,$short, != *,"${o#-}"[:,]* ]]; then + o=$(_getopt_resolve_abbrev "${1%%=*}" "${longarr[@]}" 2>/dev/null) + case $? in + (0) + # Unambiguous match. Let the long options parser handle + # it, with a flag to get the right error message. + set -- "-$1" "${@:2}" + alt_recycled=true + continue ;; + (1) + # Ambiguous match, generate error and continue. + _getopt_resolve_abbrev "${1%%=*}" "${longarr[@]}" >/dev/null + error=1 + shift + continue ;; + (2) + # No match, fall through to single-character check. + true ;; + (*) + echo "getopt: assertion failed (3)" >&2 + return 3 ;; + esac + fi + fi + + o=${1::2} + if [[ "$short" == *"${o#-}"::* ]]; then + if [[ ${#1} -gt 2 ]]; then + opts=( "${opts[@]}" "$o" "${1:2}" ) + else + opts=( "${opts[@]}" "$o" '' ) + fi + elif [[ "$short" == *"${o#-}":* ]]; then + if [[ ${#1} -gt 2 ]]; then + opts=( "${opts[@]}" "$o" "${1:2}" ) + elif [[ $# -ge 2 ]]; then + shift + opts=( "${opts[@]}" "$o" "$1" ) + else + _getopt_err "$name: option requires an argument -- '${o#-}'" + error=1 + fi + elif [[ "$short" == *"${o#-}"* ]]; then + opts=( "${opts[@]}" "$o" ) + if [[ ${#1} -gt 2 ]]; then + set -- "$o" "-${1:2}" "${@:2}" + fi + else + if [[ $flags == *a* ]]; then + # Alternative parsing mode! Report on the entire failed + # option. GNU includes =value but we omit it for sanity with + # very long values. + _getopt_err "$name: unrecognized option '${1%%=*}'" + else + _getopt_err "$name: invalid option -- '${o#-}'" + if [[ ${#1} -gt 2 ]]; then + set -- "$o" "-${1:2}" "${@:2}" + fi + fi + error=1 + fi ;; + + (*) + # GNU getopt in-place mode (leading dash on short options) + # overrides POSIXLY_CORRECT + if [[ $flags == *i* ]]; then + opts=( "${opts[@]}" "$1" ) + elif [[ $flags == *p* ]]; then + params=( "${params[@]}" "$@" ) + break + else + params=( "${params[@]}" "$1" ) + fi + esac + + shift + done + + if [[ $flags == *Q* ]]; then + true # generate no output + else + echo -n ' ' + if [[ $flags == *[cu]* ]]; then + printf '%s -- %s' "${opts[*]}" "${params[*]}" + else + if [[ $flags == *t* ]]; then + _getopt_quote_csh "${opts[@]}" -- "${params[@]}" + else + _getopt_quote "${opts[@]}" -- "${params[@]}" + fi + fi + echo + fi + + return $error + } + + _getopt_err() { + if [[ $flags != *q* ]]; then + printf '%s\n' "$1" >&2 + fi + } + + _getopt_resolve_abbrev() { + # Resolves an abbrevation from a list of possibilities. + # If the abbreviation is unambiguous, echoes the expansion on stdout + # and returns 0. If the abbreviation is ambiguous, prints a message on + # stderr and returns 1. (For first parse this should convert to exit + # status 2.) If there is no match at all, prints a message on stderr + # and returns 2. + declare a q="$1" + declare -a matches + shift + for a; do + if [[ $q == "$a" ]]; then + # Exact match. Squash any other partial matches. + matches=( "$a" ) + break + elif [[ $flags == *a* && $q == -[^-]* && $a == -"$q" ]]; then + # Exact alternative match. Squash any other partial matches. + matches=( "$a" ) + break + elif [[ $a == "$q"* ]]; then + # Abbreviated match. + matches=( "${matches[@]}" "$a" ) + elif [[ $flags == *a* && $q == -[^-]* && $a == -"$q"* ]]; then + # Abbreviated alternative match. + matches=( "${matches[@]}" "${a#-}" ) + fi + done + case ${#matches[@]} in + (0) + [[ $flags == *q* ]] || \ + printf "$name: unrecognized option %s\\n" >&2 \ + "$(_getopt_quote "$q")" + return 2 ;; + (1) + printf '%s' "${matches[0]}"; return 0 ;; + (*) + [[ $flags == *q* ]] || \ + printf "$name: option %s is ambiguous; possibilities: %s\\n" >&2 \ + "$(_getopt_quote "$q")" "$(_getopt_quote "${matches[@]}")" + return 1 ;; + esac + } + + _getopt_split() { + # Splits $2 at commas to build array specified by $1 + declare IFS=, + eval "$1=( \$2 )" + } + + _getopt_quote() { + # Quotes arguments with single quotes, escaping inner single quotes + declare s space q=\' + for s; do + printf "$space'%s'" "${s//$q/$q\\$q$q}" + space=' ' + done + } + + _getopt_quote_csh() { + # Quotes arguments with single quotes, escaping inner single quotes, + # bangs, backslashes and newlines + declare s i c space + for s; do + echo -n "$space'" + for ((i=0; i<${#s}; i++)); do + c=${s:i:1} + case $c in + (\\|\'|!) + echo -n "'\\$c'" ;; + ($'\n') + echo -n "\\$c" ;; + (*) + echo -n "$c" ;; + esac + done + echo -n \' + space=' ' + done + } + + _getopt_help() { + cat <<-EOT >&2 + + Usage: + getopt + getopt [options] [--] + getopt [options] -o|--options [options] [--] + + Parse command options. + + Options: + -a, --alternative allow long options starting with single - + -l, --longoptions the long options to be recognized + -n, --name the name under which errors are reported + -o, --options the short options to be recognized + -q, --quiet disable error reporting by getopt(3) + -Q, --quiet-output no normal output + -s, --shell set quoting conventions to those of + -T, --test test for getopt(1) version + -u, --unquoted do not quote the output + + -h, --help display this help and exit + -V, --version output version information and exit + + For more details see getopt(1). + EOT + } + + _getopt_version_check() { + if [[ -z $BASH_VERSION ]]; then + echo "getopt: unknown version of bash might not be compatible" >&2 + return 1 + fi + + # This is a lexical comparison that should be sufficient forever. + if [[ $BASH_VERSION < 2.05b ]]; then + echo "getopt: bash $BASH_VERSION might not be compatible" >&2 + return 1 + fi + + return 0 + } + + _getopt_version_check + _getopt_main "$@" + declare status=$? + unset -f _getopt_main _getopt_err _getopt_parse _getopt_quote \ + _getopt_quote_csh _getopt_resolve_abbrev _getopt_split _getopt_help \ + _getopt_version_check + return $status +} + +[[ $BASH_SOURCE != "$0" ]] || main "$@" \ No newline at end of file diff --git a/helpers/terraform_validate b/helpers/terraform_validate new file mode 100755 index 0000000000..0c284194ac --- /dev/null +++ b/helpers/terraform_validate @@ -0,0 +1,23 @@ +#! /bin/bash +# +# Copyright 2019 Google LLC. This software is provided as-is, without warranty +# or representation for any use or purpose. Your use of it is subject to your +# agreement with Google. +# +# This script initializes modules so that terraform validate as of 0.12 behaves +# as expected and does not issue errors such as: +# +# Error: Module not installed +# +# on test/fixtures/shared_vpc_no_subnets/main.tf line 37: +# 37: module "project-factory" { +# +# This module is not yet installed. Run "terraform init" to install all modules +# required by this configuration. + +# The first and only argument to this script is the directory containing *.tf +# files to validate. This directory is assumed to be a root module. + +cd "$1" +terraform init -backend=false +terraform validate \ No newline at end of file diff --git a/main.tf b/main.tf index 218babf66a..b63d60f884 100644 --- a/main.tf +++ b/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -51,214 +60,57 @@ locals { provider = null }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.endpoint, [""]), - 0, - ) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = google_container_cluster.primary.endpoint - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } - - - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } - - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -267,7 +119,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/masq.tf b/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/masq.tf +++ b/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-private-cluster/cluster_regional.tf b/modules/beta-private-cluster/cluster.tf similarity index 93% rename from modules/beta-private-cluster/cluster_regional.tf rename to modules/beta-private-cluster/cluster.tf index a3a2d10d9c..e32fb0ccb7 100644 --- a/modules/beta-private-cluster/cluster_regional.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google-beta - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -176,19 +171,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -204,7 +199,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -262,7 +257,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -295,8 +290,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/beta-private-cluster/cluster_zonal.tf b/modules/beta-private-cluster/cluster_zonal.tf deleted file mode 100644 index 09f83d002b..0000000000 --- a/modules/beta-private-cluster/cluster_zonal.tf +++ /dev/null @@ -1,310 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google-beta - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - enable_binary_authorization = var.enable_binary_authorization - enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node - - vertical_pod_autoscaling { - enabled = var.enable_vertical_pod_autoscaling - } - - dynamic "pod_security_policy_config" { - for_each = var.pod_security_policy_config - content { - enabled = pod_security_policy_config.value.enabled - } - } - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - - istio_config { - disabled = ! var.istio - } - - dynamic "cloudrun_config" { - for_each = local.cluster_cloudrun_config - - content { - disabled = cloudrun_config.value.disabled - } - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - } - - private_cluster_config { - enable_private_endpoint = var.enable_private_endpoint - enable_private_nodes = var.enable_private_nodes - master_ipv4_cidr_block = var.master_ipv4_cidr_block - } - - remove_default_node_pool = var.remove_default_node_pool - - dynamic "database_encryption" { - for_each = var.database_encryption - - content { - key_name = database_encryption.value.key_name - state = database_encryption.value.state - } - } -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf index 7138473ded..b240a23e65 100644 --- a/modules/beta-private-cluster/dns.tf +++ b/modules/beta-private-cluster/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index d4c5d3f68d..ede006d3e2 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -57,231 +66,63 @@ locals { node_metadata = var.node_metadata }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - - zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features - cluster_type_output_istio_enabled = { - regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - } - - cluster_type_output_pod_security_policy_enabled = { - regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) - } - - cluster_type_output_intranode_visbility_enabled = { - regional = element(concat(google_container_cluster.primary.*.enable_intranode_visibility, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.enable_intranode_visibility, [""]), 0) - } + cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled - cluster_type_output_vertical_pod_autoscaling_enabled = { - regional = element(concat(google_container_cluster.primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - } # /BETA features - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features - cluster_istio_enabled = ! local.cluster_type_output_istio_enabled[local.cluster_type] + cluster_istio_enabled = ! local.cluster_output_istio_enabled cluster_cloudrun_enabled = var.cloudrun - cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] - cluster_intranode_visibility_enabled = local.cluster_type_output_intranode_visbility_enabled[local.cluster_type] - cluster_vertical_pod_autoscaling_enabled = local.cluster_type_output_vertical_pod_autoscaling_enabled[local.cluster_type] + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled # /BETA features } @@ -289,8 +130,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -299,7 +139,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/modules/beta-private-cluster/masq.tf b/modules/beta-private-cluster/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/modules/beta-private-cluster/masq.tf +++ b/modules/beta-private-cluster/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index df21e7717e..4153960069 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/modules/beta-public-cluster/cluster_regional.tf b/modules/beta-public-cluster/cluster.tf similarity index 93% rename from modules/beta-public-cluster/cluster_regional.tf rename to modules/beta-public-cluster/cluster.tf index c295bed608..aeaebfefbb 100644 --- a/modules/beta-public-cluster/cluster_regional.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google-beta - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -157,6 +152,7 @@ resource "google_container_cluster" "primary" { } } + remove_default_node_pool = var.remove_default_node_pool dynamic "database_encryption" { @@ -170,19 +166,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -198,7 +194,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -256,7 +252,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -289,8 +285,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/beta-public-cluster/cluster_zonal.tf b/modules/beta-public-cluster/cluster_zonal.tf deleted file mode 100644 index e62727afb8..0000000000 --- a/modules/beta-public-cluster/cluster_zonal.tf +++ /dev/null @@ -1,305 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google-beta - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - enable_binary_authorization = var.enable_binary_authorization - enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node - - vertical_pod_autoscaling { - enabled = var.enable_vertical_pod_autoscaling - } - - dynamic "pod_security_policy_config" { - for_each = var.pod_security_policy_config - content { - enabled = pod_security_policy_config.value.enabled - } - } - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - - istio_config { - disabled = ! var.istio - } - - dynamic "cloudrun_config" { - for_each = local.cluster_cloudrun_config - - content { - disabled = cloudrun_config.value.disabled - } - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - } - - - remove_default_node_pool = var.remove_default_node_pool - - dynamic "database_encryption" { - for_each = var.database_encryption - - content { - key_name = database_encryption.value.key_name - state = database_encryption.value.state - } - } -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf index 7138473ded..b240a23e65 100644 --- a/modules/beta-public-cluster/dns.tf +++ b/modules/beta-public-cluster/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 1b706fc63e..5e165d4a3b 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -57,233 +66,63 @@ locals { node_metadata = var.node_metadata }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.endpoint, [""]), - 0, - ) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = google_container_cluster.primary.endpoint - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features - cluster_type_output_istio_enabled = { - regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - } - - cluster_type_output_pod_security_policy_enabled = { - regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) - } - - cluster_type_output_intranode_visbility_enabled = { - regional = element(concat(google_container_cluster.primary.*.enable_intranode_visibility, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.enable_intranode_visibility, [""]), 0) - } + cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled - cluster_type_output_vertical_pod_autoscaling_enabled = { - regional = element(concat(google_container_cluster.primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - } # /BETA features - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features - cluster_istio_enabled = ! local.cluster_type_output_istio_enabled[local.cluster_type] + cluster_istio_enabled = ! local.cluster_output_istio_enabled cluster_cloudrun_enabled = var.cloudrun - cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] - cluster_intranode_visibility_enabled = local.cluster_type_output_intranode_visbility_enabled[local.cluster_type] - cluster_vertical_pod_autoscaling_enabled = local.cluster_type_output_vertical_pod_autoscaling_enabled[local.cluster_type] + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled # /BETA features } @@ -291,8 +130,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -301,7 +139,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/modules/beta-public-cluster/masq.tf b/modules/beta-public-cluster/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/modules/beta-public-cluster/masq.tf +++ b/modules/beta-public-cluster/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index df21e7717e..4153960069 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster.tf similarity index 92% rename from modules/private-cluster/cluster_regional.tf rename to modules/private-cluster/cluster.tf index 1e5b6669d3..6ea5394b02 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google-beta - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -132,19 +127,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -159,7 +154,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -217,7 +212,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -242,8 +237,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf deleted file mode 100644 index 4367847af8..0000000000 --- a/modules/private-cluster/cluster_zonal.tf +++ /dev/null @@ -1,257 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google-beta - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - } - } - - private_cluster_config { - enable_private_endpoint = var.enable_private_endpoint - enable_private_nodes = var.enable_private_nodes - master_ipv4_cidr_block = var.master_ipv4_cidr_block - } - - remove_default_node_pool = var.remove_default_node_pool -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index 7138473ded..b240a23e65 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index d39207827d..9c8dc629eb 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -51,212 +60,57 @@ locals { provider = null }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - - zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } - - - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } - - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -265,7 +119,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/modules/private-cluster/masq.tf b/modules/private-cluster/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/modules/private-cluster/masq.tf +++ b/modules/private-cluster/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 3beb362d25..dea7b5c7b5 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/outputs.tf b/outputs.tf index 3beb362d25..dea7b5c7b5 100644 --- a/outputs.tf +++ b/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/sa.tf b/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/sa.tf +++ b/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/test/fixtures/simple_regional/example.tf b/test/fixtures/simple_regional/example.tf index cb37e43427..a03fadb28b 100644 --- a/test/fixtures/simple_regional/example.tf +++ b/test/fixtures/simple_regional/example.tf @@ -26,4 +26,3 @@ module "example" { ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_account } - diff --git a/test/make.sh b/test/make.sh index 3abee2e042..e3937328c0 100755 --- a/test/make.sh +++ b/test/make.sh @@ -38,6 +38,7 @@ find_files() { find "${pth}" '(' \ -path '*/.git' -o \ -path '*/.terraform' -o \ + -path '*/.kitchen' -o \ -path './autogen' -o \ -path './test/fixtures/all_examples' -o \ -path './test/fixtures/shared' ')' \ @@ -46,7 +47,7 @@ find_files() { # Compatibility with both GNU and BSD style xargs. compat_xargs() { - local compat=() + local compat=() rval # Test if xargs is GNU or BSD style. GNU xargs will succeed with status 0 # when given --no-run-if-empty and no input on STDIN. BSD xargs will fail and # exit status non-zero If xargs fails, assume it is BSD style and proceed. @@ -55,6 +56,11 @@ compat_xargs() { compat=("--no-run-if-empty") fi xargs "${compat[@]}" "$@" + rval="$?" + if [[ -z "${NOWARN:-}" ]] && [[ "${rval}" -gt 0 ]]; then + echo "Warning: compat_xargs $* failed with exit code ${rval}" >&2 + fi + return "${rval}" } # This function makes sure that the required files for @@ -67,24 +73,28 @@ function basefiles() { done } -# This function runs the hadolint linter on -# every file named 'Dockerfile' -function docker() { - echo "Running hadolint on Dockerfiles" - find_files . -name "Dockerfile" -print0 \ - | compat_xargs -0 hadolint -} - # This function runs 'terraform validate' and 'terraform fmt' # against all directory paths which contain *.tf files. function check_terraform() { - set -e - echo "Running terraform validate and terraform fmt" + local rval=125 + # fmt is before validate for faster feedback, validate requires terraform + # init which takes time. + echo "Running terraform fmt" find_files . -name "*.tf" -print0 \ | compat_xargs -0 -n1 dirname \ | sort -u \ - | compat_xargs -t -n1 -i{} bash -c \ - 'terraform init "{}" && terraform validate "{}" && terraform fmt -check=true -write=false -diff "{}"' + | compat_xargs -t -n1 terraform fmt -diff -check=true -write=false + rval="$?" + if [[ "${rval}" -gt 0 ]]; then + echo "Error: terraform fmt failed with exit code ${rval}" >&2 + echo "Check the output for diffs and correct using terraform fmt " >&2 + return "${rval}" + fi + echo "Running terraform validate" + find_files . -not -path "./test/fixtures/shared/*" -name "*.tf" -print0 \ + | compat_xargs -0 -n1 dirname \ + | sort -u \ + | compat_xargs -t -n1 helpers/terraform_validate } # This function runs 'go fmt' and 'go vet' on every file @@ -118,7 +128,7 @@ function check_trailing_whitespace() { echo "Checking for trailing whitespace" find_files . -print \ | grep -v -E '\.(pyc|png)$' \ - | compat_xargs grep -H -n '[[:blank:]]$' + | NOWARN=1 compat_xargs grep -H -n '[[:blank:]]$' rc=$? if [[ ${rc} -eq 0 ]]; then return 1 @@ -132,15 +142,18 @@ function generate() { function generate_docs() { echo "Generating markdown docs with terraform-docs" - local path - while read -r path; do - if [[ -e "${path}/README.md" ]]; then - # script seem to be designed to work into current directory - cd "${path}" && echo "Working in ${path} ..." - terraform_docs.sh . && echo Success! || echo "Warning! Exit code: ${?}" - cd - >/dev/null + local pth helper_dir rval + helper_dir="$(pwd)/helpers" + while read -r pth; do + if [[ -e "${pth}/README.md" ]]; then + (cd "${pth}" || return 3; "${helper_dir}"/terraform_docs .;) + rval="$?" + if [[ "${rval}" -gt 0 ]]; then + echo "Error: terraform_docs in ${pth} exit code: ${rval}" >&2 + return "${rval}" + fi else - echo "Skipping ${path} because README.md does not exist." + echo "Skipping ${pth} because README.md does not exist." fi done < <(find_files . -name '*.tf' -print0 \ | compat_xargs -0 -n1 dirname \