diff --git a/Makefile b/Makefile index 95abe1e74d..21d7a2764f 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCK all: check generate_docs .PHONY: check -check: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace check_generate check_generate_docs +check: check_shell check_python check_golang check_terraform check_base_files test_check_headers check_headers check_trailing_whitespace check_generate check_generate_docs # The .PHONY directive tells make that this isn't a real target and so # the presence of a file named 'check_shell' won't cause this target to stop @@ -47,10 +47,6 @@ check_golang: check_terraform: @source test/make.sh && check_terraform -.PHONY: check_docker -check_docker: - @source test/make.sh && docker - .PHONY: check_base_files check_base_files: @source test/make.sh && basefiles diff --git a/autogen/cluster_regional.tf b/autogen/cluster.tf similarity index 93% rename from autogen/cluster_regional.tf rename to autogen/cluster.tf index 1c20d5c940..87abe433ae 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster.tf @@ -17,7 +17,7 @@ {{ autogeneration_note }} /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { {% if private_cluster or beta_cluster %} @@ -26,18 +26,13 @@ resource "google_container_cluster" "primary" { provider = google {% endif %} - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -51,7 +46,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -173,8 +168,8 @@ resource "google_container_cluster" "primary" { enable_private_nodes = var.enable_private_nodes master_ipv4_cidr_block = var.master_ipv4_cidr_block } - {% endif %} + remove_default_node_pool = var.remove_default_node_pool {% if beta_cluster %} @@ -190,19 +185,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -220,7 +215,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -278,7 +273,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -313,8 +308,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf deleted file mode 100644 index 0ed6af6b5c..0000000000 --- a/autogen/cluster_zonal.tf +++ /dev/null @@ -1,328 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -{{ autogeneration_note }} - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - {% if private_cluster or beta_cluster %} - provider = google-beta - {% else %} - provider = google - {% endif %} - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - -{% if beta_cluster %} - enable_binary_authorization = var.enable_binary_authorization - enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node - - vertical_pod_autoscaling { - enabled = var.enable_vertical_pod_autoscaling - } - - dynamic "pod_security_policy_config" { - for_each = var.pod_security_policy_config - content { - enabled = pod_security_policy_config.value.enabled - } - } - -{% endif %} - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - {% if beta_cluster %} - - istio_config { - disabled = ! var.istio - } - - dynamic "cloudrun_config" { - for_each = local.cluster_cloudrun_config - - content { - disabled = cloudrun_config.value.disabled - } - } - {% endif %} - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - {% if beta_cluster %} - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - {% endif %} - } - } - -{% if private_cluster %} - private_cluster_config { - enable_private_endpoint = var.enable_private_endpoint - enable_private_nodes = var.enable_private_nodes - master_ipv4_cidr_block = var.master_ipv4_cidr_block - } -{% endif %} - - remove_default_node_pool = var.remove_default_node_pool -{% if beta_cluster %} - - dynamic "database_encryption" { - for_each = var.database_encryption - - content { - key_name = database_encryption.value.key_name - state = database_encryption.value.state - } - } -{% endif %} -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - {% if beta_cluster %} - max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - {% endif %} - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - {% if beta_cluster %} - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - {% endif %} - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/autogen/dns.tf b/autogen/dns.tf index 65c8d99d65..d9d4a35395 100644 --- a/autogen/dns.tf +++ b/autogen/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/autogen/main.tf b/autogen/main.tf index 5603e3b54a..170b1570a9 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -26,8 +26,8 @@ data "google_compute_zones" "available" { provider = google {% endif %} - project = var.project_id - region = var.region + project = var.project_id + region = var.region } resource "random_shuffle" "available_zones" { @@ -36,271 +36,106 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && !var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true provider = var.network_policy_provider - }] : [{ - enabled = false + }] : [{ + enabled = false provider = null }] {% if beta_cluster %} - cluster_cloudrun_config = var.cloudrun ? [{disabled = false}] : [] + cluster_cloudrun_config = var.cloudrun ? [{ disabled = false }] : [] cluster_node_metadata_config = var.node_metadata == "UNSPECIFIED" ? [] : [{ node_metadata = var.node_metadata }] {% endif %} - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones {% if private_cluster %} - cluster_type_output_endpoint = { - regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - - zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) - } + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint {% else %} - cluster_type_output_endpoint = { - regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.endpoint, [""]), - 0, - ) - } + cluster_output_endpoint = google_container_cluster.primary.endpoint {% endif %} - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } - - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } - - {% if beta_cluster %} +{% if beta_cluster %} # BETA features - cluster_type_output_istio_enabled = { - regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - } - - cluster_type_output_pod_security_policy_enabled = { - regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) - } + cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled - cluster_type_output_intranode_visbility_enabled = { - regional = element(concat(google_container_cluster.primary.*.enable_intranode_visibility, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.enable_intranode_visibility, [""]), 0) - } - - cluster_type_output_vertical_pod_autoscaling_enabled = { - regional = element(concat(google_container_cluster.primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - } # /BETA features {% endif %} - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } - - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = !local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = !local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = !local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = !local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled {% if beta_cluster %} # BETA features - cluster_istio_enabled = !local.cluster_type_output_istio_enabled[local.cluster_type] - cluster_cloudrun_enabled = var.cloudrun - cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] - cluster_intranode_visibility_enabled = local.cluster_type_output_intranode_visbility_enabled[local.cluster_type] - cluster_vertical_pod_autoscaling_enabled = local.cluster_type_output_vertical_pod_autoscaling_enabled[local.cluster_type] + cluster_istio_enabled = ! local.cluster_output_istio_enabled + cluster_cloudrun_enabled = var.cloudrun + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled # /BETA features {% endif %} } @@ -309,8 +144,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -319,7 +153,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/autogen/masq.tf b/autogen/masq.tf index afd34e148d..6deab757c3 100644 --- a/autogen/masq.tf +++ b/autogen/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/autogen/outputs.tf b/autogen/outputs.tf index 4285a37a62..ff8eab1bef 100644 --- a/autogen/outputs.tf +++ b/autogen/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/autogen/sa.tf b/autogen/sa.tf index 6fbc2dfb51..c5f7f132a1 100644 --- a/autogen/sa.tf +++ b/autogen/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/cluster_regional.tf b/cluster.tf similarity index 92% rename from cluster_regional.tf rename to cluster.tf index f6afe63d49..499e689a4b 100644 --- a/cluster_regional.tf +++ b/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -122,23 +117,24 @@ resource "google_container_cluster" "primary" { } } + remove_default_node_pool = var.remove_default_node_pool } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -153,7 +149,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -211,7 +207,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -236,8 +232,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/cluster_zonal.tf b/cluster_zonal.tf deleted file mode 100644 index 36cec81b3f..0000000000 --- a/cluster_zonal.tf +++ /dev/null @@ -1,252 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - } - } - - - remove_default_node_pool = var.remove_default_node_pool -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/dns.tf b/dns.tf index 7138473ded..b240a23e65 100644 --- a/dns.tf +++ b/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/helpers/terraform_docs b/helpers/terraform_docs new file mode 100755 index 0000000000..c33230959b --- /dev/null +++ b/helpers/terraform_docs @@ -0,0 +1,694 @@ +#!/usr/bin/env bash + +set -e + +main() { + declare argv + argv=$(getopt -o a: --long args: -- "$@") || return + eval "set -- $argv" + + declare args + declare files + + for argv; do + case $argv in + (-a|--args) + shift + args="$1" + shift + ;; + (--) + shift + files="$@" + break + ;; + esac + done + + local hack_terraform_docs=$(terraform version | head -1 | grep -c 0.12) + + if [[ "$hack_terraform_docs" == "1" ]]; then + which awk 2>&1 >/dev/null || ( echo "awk is required for terraform-docs hack to work with Terraform 0.12"; exit 1) + + tmp_file_awk=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") + terraform_docs_awk "$tmp_file_awk" + terraform_docs "$tmp_file_awk" "$args" "$files" + rm -f "$tmp_file_awk" + else + terraform_docs "0" "$args" "$files" + fi + +} + +terraform_docs() { + readonly terraform_docs_awk_file="$1" + readonly args="$2" + readonly files="$3" + + declare -a paths + declare -a tfvars_files + + index=0 + + for file_with_path in $files; do + file_with_path="${file_with_path// /__REPLACED__SPACE__}" + + paths[index]=$(dirname "$file_with_path") + + if [[ "$file_with_path" == *".tfvars" ]]; then + tfvars_files+=("$file_with_path") + fi + + ((index+=1)) + done + + readonly tmp_file=$(mktemp) + readonly text_file="README.md" + + for path_uniq in $(echo "${paths[*]}" | tr ' ' '\n' | sort -u); do + path_uniq="${path_uniq//__REPLACED__SPACE__/ }" + + pushd "$path_uniq" > /dev/null + + if [[ ! -f "$text_file" ]]; then + popd > /dev/null + continue + fi + + if [[ "$terraform_docs_awk_file" == "0" ]]; then + terraform-docs $args md ./ > "$tmp_file" + else + # Can't append extension for mktemp, so renaming instead + tmp_file_docs=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") + mv "$tmp_file_docs" "$tmp_file_docs.tf" + tmp_file_docs_tf="$tmp_file_docs.tf" + + awk -f "$terraform_docs_awk_file" ./*.tf > "$tmp_file_docs_tf" + terraform-docs $args md "$tmp_file_docs_tf" > "$tmp_file" + rm -f "$tmp_file_docs_tf" + fi + + # Replace content between markers with the placeholder - https://stackoverflow.com/questions/1212799/how-do-i-extract-lines-between-two-line-delimiters-in-perl#1212834 + perl -i -ne 'if (/BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/../END OF PRE-COMMIT-TERRAFORM DOCS HOOK/) { print $_ if /BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/; print "I_WANT_TO_BE_REPLACED\n$_" if /END OF PRE-COMMIT-TERRAFORM DOCS HOOK/;} else { print $_ }' "$text_file" + + # Replace placeholder with the content of the file + perl -i -e 'open(F, "'"$tmp_file"'"); $f = join "", ; while(<>){if (/I_WANT_TO_BE_REPLACED/) {print $f} else {print $_};}' "$text_file" + + rm -f "$tmp_file" + + popd > /dev/null + done +} + +terraform_docs_awk() { + readonly output_file=$1 + + cat <<"EOF" > $output_file +# This script converts Terraform 0.12 variables/outputs to something suitable for `terraform-docs` +# As of terraform-docs v0.6.0, HCL2 is not supported. This script is a *dirty hack* to get around it. +# https://github.com/segmentio/terraform-docs/ +# https://github.com/segmentio/terraform-docs/issues/62 + +# Script was originally found here: https://github.com/cloudposse/build-harness/blob/master/bin/terraform-docs.awk + +{ + if ( $0 ~ /\{/ ) { + braceCnt++ + } + + if ( $0 ~ /\}/ ) { + braceCnt-- + } + + # [START] variable or output block started + if ($0 ~ /^[[:space:]]*(variable|output)[[:space:]][[:space:]]*"(.*?)"/) { + # Normalize the braceCnt (should be 1 now) + braceCnt = 1 + # [CLOSE] "default" block + if (blockDefCnt > 0) { + blockDefCnt = 0 + } + blockCnt++ + print $0 + } + + # [START] multiline default statement started + if (blockCnt > 0) { + if ($0 ~ /^[[:space:]][[:space:]]*(default)[[:space:]][[:space:]]*=/) { + if ($3 ~ "null") { + print " default = \"null\"" + } else { + print $0 + blockDefCnt++ + blockDefStart=1 + } + } + } + + # [PRINT] single line "description" + if (blockCnt > 0) { + if (blockDefCnt == 0) { + if ($0 ~ /^[[:space:]][[:space:]]*description[[:space:]][[:space:]]*=/) { + # [CLOSE] "default" block + if (blockDefCnt > 0) { + blockDefCnt = 0 + } + print $0 + } + } + } + + # [PRINT] single line "type" + if (blockCnt > 0) { + if ($0 ~ /^[[:space:]][[:space:]]*type[[:space:]][[:space:]]*=/ ) { + # [CLOSE] "default" block + if (blockDefCnt > 0) { + blockDefCnt = 0 + } + type=$3 + if (type ~ "object") { + print " type = \"object\"" + } else { + # legacy quoted types: "string", "list", and "map" + if ($3 ~ /^[[:space:]]*"(.*?)"[[:space:]]*$/) { + print " type = " $3 + } else { + print " type = \"" $3 "\"" + } + } + } + } + + # [CLOSE] variable/output block + if (blockCnt > 0) { + if (braceCnt == 0 && blockCnt > 0) { + blockCnt-- + print $0 + } + } + + # [PRINT] Multiline "default" statement + if (blockCnt > 0 && blockDefCnt > 0) { + if (blockDefStart == 1) { + blockDefStart = 0 + } else { + print $0 + } + } +} +EOF + +} + +getopt() { + # pure-getopt, a drop-in replacement for GNU getopt in pure Bash. + # version 1.4.3 + # + # Copyright 2012-2018 Aron Griffis + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be included + # in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + _getopt_main() { + # Returns one of the following statuses: + # 0 success + # 1 error parsing parameters + # 2 error in getopt invocation + # 3 internal error + # 4 reserved for -T + # + # For statuses 0 and 1, generates normalized and shell-quoted + # "options -- parameters" on stdout. + + declare parsed status + declare short long name flags + declare have_short=false + + # Synopsis from getopt man-page: + # + # getopt optstring parameters + # getopt [options] [--] optstring parameters + # getopt [options] -o|--options optstring [options] [--] parameters + # + # The first form can be normalized to the third form which + # _getopt_parse() understands. The second form can be recognized after + # first parse when $short hasn't been set. + + if [[ -n ${GETOPT_COMPATIBLE+isset} || $1 == [^-]* ]]; then + # Enable compatibility mode + flags=c$flags + # Normalize first to third synopsis form + set -- -o "$1" -- "${@:2}" + fi + + # First parse always uses flags=p since getopt always parses its own + # arguments effectively in this mode. + parsed=$(_getopt_parse getopt ahl:n:o:qQs:TuV \ + alternative,help,longoptions:,name:,options:,quiet,quiet-output,shell:,test,version \ + p "$@") + status=$? + if [[ $status != 0 ]]; then + if [[ $status == 1 ]]; then + echo "Try \`getopt --help' for more information." >&2 + # Since this is the first parse, convert status 1 to 2 + status=2 + fi + return $status + fi + eval "set -- $parsed" + + while [[ $# -gt 0 ]]; do + case $1 in + (-a|--alternative) + flags=a$flags ;; + + (-h|--help) + _getopt_help + return 2 # as does GNU getopt + ;; + + (-l|--longoptions) + long="$long${long:+,}$2" + shift ;; + + (-n|--name) + name=$2 + shift ;; + + (-o|--options) + short=$2 + have_short=true + shift ;; + + (-q|--quiet) + flags=q$flags ;; + + (-Q|--quiet-output) + flags=Q$flags ;; + + (-s|--shell) + case $2 in + (sh|bash) + flags=${flags//t/} ;; + (csh|tcsh) + flags=t$flags ;; + (*) + echo 'getopt: unknown shell after -s or --shell argument' >&2 + echo "Try \`getopt --help' for more information." >&2 + return 2 ;; + esac + shift ;; + + (-u|--unquoted) + flags=u$flags ;; + + (-T|--test) + return 4 ;; + + (-V|--version) + echo "pure-getopt 1.4.3" + return 0 ;; + + (--) + shift + break ;; + esac + + shift + done + + if ! $have_short; then + # $short was declared but never set, not even to an empty string. + # This implies the second form in the synopsis. + if [[ $# == 0 ]]; then + echo 'getopt: missing optstring argument' >&2 + echo "Try \`getopt --help' for more information." >&2 + return 2 + fi + short=$1 + have_short=true + shift + fi + + if [[ $short == -* ]]; then + # Leading dash means generate output in place rather than reordering, + # unless we're already in compatibility mode. + [[ $flags == *c* ]] || flags=i$flags + short=${short#?} + elif [[ $short == +* ]]; then + # Leading plus means POSIXLY_CORRECT, unless we're already in + # compatibility mode. + [[ $flags == *c* ]] || flags=p$flags + short=${short#?} + fi + + # This should fire if POSIXLY_CORRECT is in the environment, even if + # it's an empty string. That's the difference between :+ and + + flags=${POSIXLY_CORRECT+p}$flags + + _getopt_parse "${name:-getopt}" "$short" "$long" "$flags" "$@" + } + + _getopt_parse() { + # Inner getopt parser, used for both first parse and second parse. + # Returns 0 for success, 1 for error parsing, 3 for internal error. + # In the case of status 1, still generates stdout with whatever could + # be parsed. + # + # $flags is a string of characters with the following meanings: + # a - alternative parsing mode + # c - GETOPT_COMPATIBLE + # i - generate output in place rather than reordering + # p - POSIXLY_CORRECT + # q - disable error reporting + # Q - disable normal output + # t - quote for csh/tcsh + # u - unquoted output + + declare name="$1" short="$2" long="$3" flags="$4" + shift 4 + + # Split $long on commas, prepend double-dashes, strip colons; + # for use with _getopt_resolve_abbrev + declare -a longarr + _getopt_split longarr "$long" + longarr=( "${longarr[@]/#/--}" ) + longarr=( "${longarr[@]%:}" ) + longarr=( "${longarr[@]%:}" ) + + # Parse and collect options and parameters + declare -a opts params + declare o alt_recycled=false error=0 + + while [[ $# -gt 0 ]]; do + case $1 in + (--) + params=( "${params[@]}" "${@:2}" ) + break ;; + + (--*=*) + o=${1%%=*} + if ! o=$(_getopt_resolve_abbrev "$o" "${longarr[@]}"); then + error=1 + elif [[ ,"$long", == *,"${o#--}"::,* ]]; then + opts=( "${opts[@]}" "$o" "${1#*=}" ) + elif [[ ,"$long", == *,"${o#--}":,* ]]; then + opts=( "${opts[@]}" "$o" "${1#*=}" ) + elif [[ ,"$long", == *,"${o#--}",* ]]; then + if $alt_recycled; then o=${o#-}; fi + _getopt_err "$name: option '$o' doesn't allow an argument" + error=1 + else + echo "getopt: assertion failed (1)" >&2 + return 3 + fi + alt_recycled=false + ;; + + (--?*) + o=$1 + if ! o=$(_getopt_resolve_abbrev "$o" "${longarr[@]}"); then + error=1 + elif [[ ,"$long", == *,"${o#--}",* ]]; then + opts=( "${opts[@]}" "$o" ) + elif [[ ,"$long", == *,"${o#--}::",* ]]; then + opts=( "${opts[@]}" "$o" '' ) + elif [[ ,"$long", == *,"${o#--}:",* ]]; then + if [[ $# -ge 2 ]]; then + shift + opts=( "${opts[@]}" "$o" "$1" ) + else + if $alt_recycled; then o=${o#-}; fi + _getopt_err "$name: option '$o' requires an argument" + error=1 + fi + else + echo "getopt: assertion failed (2)" >&2 + return 3 + fi + alt_recycled=false + ;; + + (-*) + if [[ $flags == *a* ]]; then + # Alternative parsing mode! + # Try to handle as a long option if any of the following apply: + # 1. There's an equals sign in the mix -x=3 or -xy=3 + # 2. There's 2+ letters and an abbreviated long match -xy + # 3. There's a single letter and an exact long match + # 4. There's a single letter and no short match + o=${1::2} # temp for testing #4 + if [[ $1 == *=* || $1 == -?? || \ + ,$long, == *,"${1#-}"[:,]* || \ + ,$short, != *,"${o#-}"[:,]* ]]; then + o=$(_getopt_resolve_abbrev "${1%%=*}" "${longarr[@]}" 2>/dev/null) + case $? in + (0) + # Unambiguous match. Let the long options parser handle + # it, with a flag to get the right error message. + set -- "-$1" "${@:2}" + alt_recycled=true + continue ;; + (1) + # Ambiguous match, generate error and continue. + _getopt_resolve_abbrev "${1%%=*}" "${longarr[@]}" >/dev/null + error=1 + shift + continue ;; + (2) + # No match, fall through to single-character check. + true ;; + (*) + echo "getopt: assertion failed (3)" >&2 + return 3 ;; + esac + fi + fi + + o=${1::2} + if [[ "$short" == *"${o#-}"::* ]]; then + if [[ ${#1} -gt 2 ]]; then + opts=( "${opts[@]}" "$o" "${1:2}" ) + else + opts=( "${opts[@]}" "$o" '' ) + fi + elif [[ "$short" == *"${o#-}":* ]]; then + if [[ ${#1} -gt 2 ]]; then + opts=( "${opts[@]}" "$o" "${1:2}" ) + elif [[ $# -ge 2 ]]; then + shift + opts=( "${opts[@]}" "$o" "$1" ) + else + _getopt_err "$name: option requires an argument -- '${o#-}'" + error=1 + fi + elif [[ "$short" == *"${o#-}"* ]]; then + opts=( "${opts[@]}" "$o" ) + if [[ ${#1} -gt 2 ]]; then + set -- "$o" "-${1:2}" "${@:2}" + fi + else + if [[ $flags == *a* ]]; then + # Alternative parsing mode! Report on the entire failed + # option. GNU includes =value but we omit it for sanity with + # very long values. + _getopt_err "$name: unrecognized option '${1%%=*}'" + else + _getopt_err "$name: invalid option -- '${o#-}'" + if [[ ${#1} -gt 2 ]]; then + set -- "$o" "-${1:2}" "${@:2}" + fi + fi + error=1 + fi ;; + + (*) + # GNU getopt in-place mode (leading dash on short options) + # overrides POSIXLY_CORRECT + if [[ $flags == *i* ]]; then + opts=( "${opts[@]}" "$1" ) + elif [[ $flags == *p* ]]; then + params=( "${params[@]}" "$@" ) + break + else + params=( "${params[@]}" "$1" ) + fi + esac + + shift + done + + if [[ $flags == *Q* ]]; then + true # generate no output + else + echo -n ' ' + if [[ $flags == *[cu]* ]]; then + printf '%s -- %s' "${opts[*]}" "${params[*]}" + else + if [[ $flags == *t* ]]; then + _getopt_quote_csh "${opts[@]}" -- "${params[@]}" + else + _getopt_quote "${opts[@]}" -- "${params[@]}" + fi + fi + echo + fi + + return $error + } + + _getopt_err() { + if [[ $flags != *q* ]]; then + printf '%s\n' "$1" >&2 + fi + } + + _getopt_resolve_abbrev() { + # Resolves an abbrevation from a list of possibilities. + # If the abbreviation is unambiguous, echoes the expansion on stdout + # and returns 0. If the abbreviation is ambiguous, prints a message on + # stderr and returns 1. (For first parse this should convert to exit + # status 2.) If there is no match at all, prints a message on stderr + # and returns 2. + declare a q="$1" + declare -a matches + shift + for a; do + if [[ $q == "$a" ]]; then + # Exact match. Squash any other partial matches. + matches=( "$a" ) + break + elif [[ $flags == *a* && $q == -[^-]* && $a == -"$q" ]]; then + # Exact alternative match. Squash any other partial matches. + matches=( "$a" ) + break + elif [[ $a == "$q"* ]]; then + # Abbreviated match. + matches=( "${matches[@]}" "$a" ) + elif [[ $flags == *a* && $q == -[^-]* && $a == -"$q"* ]]; then + # Abbreviated alternative match. + matches=( "${matches[@]}" "${a#-}" ) + fi + done + case ${#matches[@]} in + (0) + [[ $flags == *q* ]] || \ + printf "$name: unrecognized option %s\\n" >&2 \ + "$(_getopt_quote "$q")" + return 2 ;; + (1) + printf '%s' "${matches[0]}"; return 0 ;; + (*) + [[ $flags == *q* ]] || \ + printf "$name: option %s is ambiguous; possibilities: %s\\n" >&2 \ + "$(_getopt_quote "$q")" "$(_getopt_quote "${matches[@]}")" + return 1 ;; + esac + } + + _getopt_split() { + # Splits $2 at commas to build array specified by $1 + declare IFS=, + eval "$1=( \$2 )" + } + + _getopt_quote() { + # Quotes arguments with single quotes, escaping inner single quotes + declare s space q=\' + for s; do + printf "$space'%s'" "${s//$q/$q\\$q$q}" + space=' ' + done + } + + _getopt_quote_csh() { + # Quotes arguments with single quotes, escaping inner single quotes, + # bangs, backslashes and newlines + declare s i c space + for s; do + echo -n "$space'" + for ((i=0; i<${#s}; i++)); do + c=${s:i:1} + case $c in + (\\|\'|!) + echo -n "'\\$c'" ;; + ($'\n') + echo -n "\\$c" ;; + (*) + echo -n "$c" ;; + esac + done + echo -n \' + space=' ' + done + } + + _getopt_help() { + cat <<-EOT >&2 + + Usage: + getopt + getopt [options] [--] + getopt [options] -o|--options [options] [--] + + Parse command options. + + Options: + -a, --alternative allow long options starting with single - + -l, --longoptions the long options to be recognized + -n, --name the name under which errors are reported + -o, --options the short options to be recognized + -q, --quiet disable error reporting by getopt(3) + -Q, --quiet-output no normal output + -s, --shell set quoting conventions to those of + -T, --test test for getopt(1) version + -u, --unquoted do not quote the output + + -h, --help display this help and exit + -V, --version output version information and exit + + For more details see getopt(1). + EOT + } + + _getopt_version_check() { + if [[ -z $BASH_VERSION ]]; then + echo "getopt: unknown version of bash might not be compatible" >&2 + return 1 + fi + + # This is a lexical comparison that should be sufficient forever. + if [[ $BASH_VERSION < 2.05b ]]; then + echo "getopt: bash $BASH_VERSION might not be compatible" >&2 + return 1 + fi + + return 0 + } + + _getopt_version_check + _getopt_main "$@" + declare status=$? + unset -f _getopt_main _getopt_err _getopt_parse _getopt_quote \ + _getopt_quote_csh _getopt_resolve_abbrev _getopt_split _getopt_help \ + _getopt_version_check + return $status +} + +[[ $BASH_SOURCE != "$0" ]] || main "$@" \ No newline at end of file diff --git a/helpers/terraform_validate b/helpers/terraform_validate new file mode 100755 index 0000000000..0c284194ac --- /dev/null +++ b/helpers/terraform_validate @@ -0,0 +1,23 @@ +#! /bin/bash +# +# Copyright 2019 Google LLC. This software is provided as-is, without warranty +# or representation for any use or purpose. Your use of it is subject to your +# agreement with Google. +# +# This script initializes modules so that terraform validate as of 0.12 behaves +# as expected and does not issue errors such as: +# +# Error: Module not installed +# +# on test/fixtures/shared_vpc_no_subnets/main.tf line 37: +# 37: module "project-factory" { +# +# This module is not yet installed. Run "terraform init" to install all modules +# required by this configuration. + +# The first and only argument to this script is the directory containing *.tf +# files to validate. This directory is assumed to be a root module. + +cd "$1" +terraform init -backend=false +terraform validate \ No newline at end of file diff --git a/main.tf b/main.tf index 218babf66a..b63d60f884 100644 --- a/main.tf +++ b/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -51,214 +60,57 @@ locals { provider = null }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.endpoint, [""]), - 0, - ) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = google_container_cluster.primary.endpoint - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } - - - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } - - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -267,7 +119,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/masq.tf b/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/masq.tf +++ b/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-private-cluster/cluster_regional.tf b/modules/beta-private-cluster/cluster.tf similarity index 93% rename from modules/beta-private-cluster/cluster_regional.tf rename to modules/beta-private-cluster/cluster.tf index a3a2d10d9c..e32fb0ccb7 100644 --- a/modules/beta-private-cluster/cluster_regional.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google-beta - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -176,19 +171,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -204,7 +199,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -262,7 +257,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -295,8 +290,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/beta-private-cluster/cluster_zonal.tf b/modules/beta-private-cluster/cluster_zonal.tf deleted file mode 100644 index 09f83d002b..0000000000 --- a/modules/beta-private-cluster/cluster_zonal.tf +++ /dev/null @@ -1,310 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google-beta - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - enable_binary_authorization = var.enable_binary_authorization - enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node - - vertical_pod_autoscaling { - enabled = var.enable_vertical_pod_autoscaling - } - - dynamic "pod_security_policy_config" { - for_each = var.pod_security_policy_config - content { - enabled = pod_security_policy_config.value.enabled - } - } - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - - istio_config { - disabled = ! var.istio - } - - dynamic "cloudrun_config" { - for_each = local.cluster_cloudrun_config - - content { - disabled = cloudrun_config.value.disabled - } - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - } - - private_cluster_config { - enable_private_endpoint = var.enable_private_endpoint - enable_private_nodes = var.enable_private_nodes - master_ipv4_cidr_block = var.master_ipv4_cidr_block - } - - remove_default_node_pool = var.remove_default_node_pool - - dynamic "database_encryption" { - for_each = var.database_encryption - - content { - key_name = database_encryption.value.key_name - state = database_encryption.value.state - } - } -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf index 7138473ded..b240a23e65 100644 --- a/modules/beta-private-cluster/dns.tf +++ b/modules/beta-private-cluster/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index d4c5d3f68d..ede006d3e2 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -57,231 +66,63 @@ locals { node_metadata = var.node_metadata }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - - zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features - cluster_type_output_istio_enabled = { - regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - } - - cluster_type_output_pod_security_policy_enabled = { - regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) - } - - cluster_type_output_intranode_visbility_enabled = { - regional = element(concat(google_container_cluster.primary.*.enable_intranode_visibility, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.enable_intranode_visibility, [""]), 0) - } + cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled - cluster_type_output_vertical_pod_autoscaling_enabled = { - regional = element(concat(google_container_cluster.primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - } # /BETA features - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features - cluster_istio_enabled = ! local.cluster_type_output_istio_enabled[local.cluster_type] + cluster_istio_enabled = ! local.cluster_output_istio_enabled cluster_cloudrun_enabled = var.cloudrun - cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] - cluster_intranode_visibility_enabled = local.cluster_type_output_intranode_visbility_enabled[local.cluster_type] - cluster_vertical_pod_autoscaling_enabled = local.cluster_type_output_vertical_pod_autoscaling_enabled[local.cluster_type] + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled # /BETA features } @@ -289,8 +130,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -299,7 +139,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/modules/beta-private-cluster/masq.tf b/modules/beta-private-cluster/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/modules/beta-private-cluster/masq.tf +++ b/modules/beta-private-cluster/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index df21e7717e..4153960069 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/modules/beta-public-cluster/cluster_regional.tf b/modules/beta-public-cluster/cluster.tf similarity index 93% rename from modules/beta-public-cluster/cluster_regional.tf rename to modules/beta-public-cluster/cluster.tf index c295bed608..aeaebfefbb 100644 --- a/modules/beta-public-cluster/cluster_regional.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google-beta - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -157,6 +152,7 @@ resource "google_container_cluster" "primary" { } } + remove_default_node_pool = var.remove_default_node_pool dynamic "database_encryption" { @@ -170,19 +166,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -198,7 +194,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -256,7 +252,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -289,8 +285,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/beta-public-cluster/cluster_zonal.tf b/modules/beta-public-cluster/cluster_zonal.tf deleted file mode 100644 index e62727afb8..0000000000 --- a/modules/beta-public-cluster/cluster_zonal.tf +++ /dev/null @@ -1,305 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google-beta - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - enable_binary_authorization = var.enable_binary_authorization - enable_intranode_visibility = var.enable_intranode_visibility - default_max_pods_per_node = var.default_max_pods_per_node - - vertical_pod_autoscaling { - enabled = var.enable_vertical_pod_autoscaling - } - - dynamic "pod_security_policy_config" { - for_each = var.pod_security_policy_config - content { - enabled = pod_security_policy_config.value.enabled - } - } - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - - istio_config { - disabled = ! var.istio - } - - dynamic "cloudrun_config" { - for_each = local.cluster_cloudrun_config - - content { - disabled = cloudrun_config.value.disabled - } - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - } - - - remove_default_node_pool = var.remove_default_node_pool - - dynamic "database_encryption" { - for_each = var.database_encryption - - content { - key_name = database_encryption.value.key_name - state = database_encryption.value.state - } - } -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - - dynamic "workload_metadata_config" { - for_each = local.cluster_node_metadata_config - - content { - node_metadata = workload_metadata_config.value.node_metadata - } - } - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf index 7138473ded..b240a23e65 100644 --- a/modules/beta-public-cluster/dns.tf +++ b/modules/beta-public-cluster/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 1b706fc63e..5e165d4a3b 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -57,233 +66,63 @@ locals { node_metadata = var.node_metadata }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.endpoint, [""]), - 0, - ) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = google_container_cluster.primary.endpoint - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features - cluster_type_output_istio_enabled = { - regional = element(concat(google_container_cluster.primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.addons_config.0.istio_config.0.disabled, [""]), 0) - } - - cluster_type_output_pod_security_policy_enabled = { - regional = element(concat(google_container_cluster.primary.*.pod_security_policy_config.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.pod_security_policy_config.0.enabled, [""]), 0) - } - - cluster_type_output_intranode_visbility_enabled = { - regional = element(concat(google_container_cluster.primary.*.enable_intranode_visibility, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.enable_intranode_visibility, [""]), 0) - } + cluster_output_istio_enabled = google_container_cluster.primary.addons_config.0.istio_config.0.disabled + cluster_output_pod_security_policy_enabled = google_container_cluster.primary.pod_security_policy_config.0.enabled + cluster_output_intranode_visbility_enabled = google_container_cluster.primary.enable_intranode_visibility + cluster_output_vertical_pod_autoscaling_enabled = google_container_cluster.primary.vertical_pod_autoscaling.0.enabled - cluster_type_output_vertical_pod_autoscaling_enabled = { - regional = element(concat(google_container_cluster.primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - zonal = element(concat(google_container_cluster.zonal_primary.*.vertical_pod_autoscaling.0.enabled, [""]), 0) - } # /BETA features - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features - cluster_istio_enabled = ! local.cluster_type_output_istio_enabled[local.cluster_type] + cluster_istio_enabled = ! local.cluster_output_istio_enabled cluster_cloudrun_enabled = var.cloudrun - cluster_pod_security_policy_enabled = local.cluster_type_output_pod_security_policy_enabled[local.cluster_type] - cluster_intranode_visibility_enabled = local.cluster_type_output_intranode_visbility_enabled[local.cluster_type] - cluster_vertical_pod_autoscaling_enabled = local.cluster_type_output_vertical_pod_autoscaling_enabled[local.cluster_type] + cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled + cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled + cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled # /BETA features } @@ -291,8 +130,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -301,7 +139,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/modules/beta-public-cluster/masq.tf b/modules/beta-public-cluster/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/modules/beta-public-cluster/masq.tf +++ b/modules/beta-public-cluster/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index df21e7717e..4153960069 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster.tf similarity index 92% rename from modules/private-cluster/cluster_regional.tf rename to modules/private-cluster/cluster.tf index 1e5b6669d3..6ea5394b02 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster.tf @@ -17,23 +17,18 @@ // This file was automatically generated from a template in ./autogen /****************************************** - Create regional cluster + Create Container Cluster *****************************************/ resource "google_container_cluster" "primary" { provider = google-beta - count = var.regional ? 1 : 0 name = var.name description = var.description project = var.project_id resource_labels = var.cluster_resource_labels - region = var.region - node_locations = coalescelist( - compact(var.zones), - sort(random_shuffle.available_zones.result), - ) - + location = local.location + node_locations = local.node_locations cluster_ipv4_cidr = var.cluster_ipv4_cidr network = data.google_compute_network.gke_network.self_link @@ -47,7 +42,7 @@ resource "google_container_cluster" "primary" { } subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_regional + min_master_version = local.master_version logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -132,19 +127,19 @@ resource "google_container_cluster" "primary" { } /****************************************** - Create regional node pools + Create Container Cluster node pools *****************************************/ resource "google_container_node_pool" "pools" { provider = google-beta - count = var.regional ? length(var.node_pools) : 0 + count = length(var.node_pools) name = var.node_pools[count.index]["name"] project = var.project_id - region = var.region - cluster = google_container_cluster.primary[0].name + location = local.location + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", - local.node_version_regional, + local.node_version, ) initial_node_count = lookup( var.node_pools[count.index], @@ -159,7 +154,7 @@ resource "google_container_node_pool" "pools" { management { auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", true) + auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", local.default_auto_upgrade) } node_config { @@ -217,7 +212,7 @@ resource "google_container_node_pool" "pools" { oauth_scopes = concat( var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -242,8 +237,7 @@ resource "google_container_node_pool" "pools" { } } -resource "null_resource" "wait_for_regional_cluster" { - count = var.regional ? 1 : 0 +resource "null_resource" "wait_for_cluster" { provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf deleted file mode 100644 index 4367847af8..0000000000 --- a/modules/private-cluster/cluster_zonal.tf +++ /dev/null @@ -1,257 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This file was automatically generated from a template in ./autogen - -/****************************************** - Create zonal cluster - *****************************************/ -resource "google_container_cluster" "zonal_primary" { - provider = google-beta - - count = var.regional ? 0 : 1 - name = var.name - description = var.description - project = var.project_id - resource_labels = var.cluster_resource_labels - - zone = var.zones[0] - node_locations = slice(var.zones, 1, length(var.zones)) - cluster_ipv4_cidr = var.cluster_ipv4_cidr - network = data.google_compute_network.gke_network.self_link - - dynamic "network_policy" { - for_each = local.cluster_network_policy - - content { - enabled = network_policy.value.enabled - provider = network_policy.value.provider - } - } - - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link - min_master_version = local.kubernetes_version_zonal - - logging_service = var.logging_service - monitoring_service = var.monitoring_service - - dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config - content { - dynamic "cidr_blocks" { - for_each = master_authorized_networks_config.value.cidr_blocks - content { - cidr_block = lookup(cidr_blocks.value, "cidr_block", "") - display_name = lookup(cidr_blocks.value, "display_name", "") - } - } - } - } - - master_auth { - username = var.basic_auth_username - password = var.basic_auth_password - - client_certificate_config { - issue_client_certificate = var.issue_client_certificate - } - } - - addons_config { - http_load_balancing { - disabled = ! var.http_load_balancing - } - - horizontal_pod_autoscaling { - disabled = ! var.horizontal_pod_autoscaling - } - - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - - network_policy_config { - disabled = ! var.network_policy - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.ip_range_pods - services_secondary_range_name = var.ip_range_services - } - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - } - - lifecycle { - ignore_changes = [node_pool] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } - - node_pool { - name = "default-pool" - initial_node_count = var.initial_node_count - - node_config { - service_account = lookup(var.node_pools[0], "service_account", local.service_account) - } - } - - private_cluster_config { - enable_private_endpoint = var.enable_private_endpoint - enable_private_nodes = var.enable_private_nodes - master_ipv4_cidr_block = var.master_ipv4_cidr_block - } - - remove_default_node_pool = var.remove_default_node_pool -} - -/****************************************** - Create zonal node pools - *****************************************/ -resource "google_container_node_pool" "zonal_pools" { - provider = google-beta - count = var.regional ? 0 : length(var.node_pools) - name = var.node_pools[count.index]["name"] - project = var.project_id - zone = var.zones[0] - cluster = google_container_cluster.zonal_primary[0].name - version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( - var.node_pools[count.index], - "version", - local.node_version_zonal, - ) - initial_node_count = lookup( - var.node_pools[count.index], - "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) - autoscaling { - min_node_count = lookup(var.node_pools[count.index], "min_count", 1) - max_node_count = lookup(var.node_pools[count.index], "max_count", 100) - } - - management { - auto_repair = lookup(var.node_pools[count.index], "auto_repair", true) - auto_upgrade = lookup(var.node_pools[count.index], "auto_upgrade", false) - } - - node_config { - image_type = lookup(var.node_pools[count.index], "image_type", "COS") - machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") - labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], - ) - metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], - { - "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints - }, - ) - dynamic "taint" { - for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], - ) - content { - effect = taint.value.effect - key = taint.value.key - value = taint.value.value - } - } - - tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], - ) - - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") - service_account = lookup( - var.node_pools[count.index], - "service_account", - local.service_account, - ) - preemptible = lookup(var.node_pools[count.index], "preemptible", false) - - oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], - ) - - guest_accelerator = [ - for guest_accelerator in lookup(var.node_pools[count.index], "accelerator_count", 0) > 0 ? [{ - type = lookup(var.node_pools[count.index], "accelerator_type", "") - count = lookup(var.node_pools[count.index], "accelerator_count", 0) - }] : [] : { - type = guest_accelerator["type"] - count = guest_accelerator["count"] - } - ] - } - - lifecycle { - ignore_changes = [initial_node_count] - } - - timeouts { - create = "30m" - update = "30m" - delete = "30m" - } -} - -resource "null_resource" "wait_for_zonal_cluster" { - count = var.regional ? 0 : 1 - - provisioner "local-exec" { - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - provisioner "local-exec" { - when = destroy - command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" - } - - depends_on = [ - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, - ] -} diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index 7138473ded..b240a23e65 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -30,8 +30,6 @@ resource "null_resource" "delete_default_kube_dns_configmap" { data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -61,8 +59,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } @@ -90,8 +86,6 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } @@ -122,7 +116,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools ] } diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index d39207827d..9c8dc629eb 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -32,16 +32,25 @@ resource "random_shuffle" "available_zones" { } locals { - kubernetes_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version - kubernetes_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version - node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.kubernetes_version_regional - node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.kubernetes_version_zonal + // location + location = var.regional ? var.region : var.zones[0] + // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted + node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) + // kuberentes version + master_version_regional = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.region.latest_master_version + master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version + node_version_regional = var.node_version != "" && var.regional ? var.node_version : local.master_version_regional + node_version_zonal = var.node_version != "" && ! var.regional ? var.node_version : local.master_version_zonal + master_version = var.regional ? local.master_version_regional : local.master_version_zonal + node_version = var.regional ? local.node_version_regional : local.node_version_zonal + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) - - cluster_type = var.regional ? "regional" : "zonal" + cluster_type = var.regional ? "regional" : "zonal" + // auto upgrade by defaults only for regional cluster as long it has multiple masters versus zonal clusters have only have a single master so upgrades are more dangerous. + default_auto_upgrade = var.regional ? true : false cluster_network_policy = var.network_policy ? [{ enabled = true @@ -51,212 +60,57 @@ locals { provider = null }] - cluster_type_output_name = { - regional = element(concat(google_container_cluster.primary.*.name, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.name, [""]), - 0, - ) - } - - cluster_type_output_location = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = element( - concat(google_container_cluster.zonal_primary.*.zone, [""]), - 0, - ) - } - - cluster_type_output_region = { - regional = element(concat(google_container_cluster.primary.*.region, [""]), 0) - zonal = var.region - } - - cluster_type_output_regional_zones = flatten(google_container_cluster.primary.*.node_locations) - cluster_type_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] - - cluster_type_output_zones = { - regional = local.cluster_type_output_regional_zones - zonal = concat( - google_container_cluster.zonal_primary.*.zone, - local.cluster_type_output_zonal_zones, - ) - } - - cluster_type_output_endpoint = { - regional = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.primary.*.endpoint, [""]), 0) - - zonal = var.deploy_using_private_endpoint ? element(concat(google_container_cluster.zonal_primary.*.private_cluster_config.0.private_endpoint, [""]), 0) : element(concat(google_container_cluster.zonal_primary.*.endpoint, [""]), 0) - } - - cluster_type_output_master_auth = { - regional = concat(google_container_cluster.primary.*.master_auth, []) - zonal = concat(google_container_cluster.zonal_primary.*.master_auth, []) - } - - cluster_type_output_master_version = { - regional = element( - concat(google_container_cluster.primary.*.master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.master_version, - [""], - ), - 0, - ) - } - cluster_type_output_min_master_version = { - regional = element( - concat(google_container_cluster.primary.*.min_master_version, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.min_master_version, - [""], - ), - 0, - ) - } + cluster_output_name = google_container_cluster.primary.name + cluster_output_location = google_container_cluster.primary.location + cluster_output_region = google_container_cluster.primary.region + cluster_output_regional_zones = google_container_cluster.primary.node_locations + cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] + cluster_output_zones = local.cluster_output_regional_zones - cluster_type_output_logging_service = { - regional = element( - concat(google_container_cluster.primary.*.logging_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.logging_service, - [""], - ), - 0, - ) - } + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint - cluster_type_output_monitoring_service = { - regional = element( - concat(google_container_cluster.primary.*.monitoring_service, [""]), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.monitoring_service, - [""], - ), - 0, - ) - } + cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) + cluster_output_master_version = google_container_cluster.primary.master_version + cluster_output_min_master_version = google_container_cluster.primary.min_master_version + cluster_output_logging_service = google_container_cluster.primary.logging_service + cluster_output_monitoring_service = google_container_cluster.primary.monitoring_service + cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled + cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled + cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled - cluster_type_output_network_policy_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, - [""], - ), - 0, - ) - } - cluster_type_output_http_load_balancing_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, - [""], - ), - 0, - ) - } + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) + cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) - cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, - [""], - ), - 0, - ) - } - - cluster_type_output_kubernetes_dashboard_enabled = { - regional = element( - concat( - google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - zonal = element( - concat( - google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, - [""], - ), - 0, - ) - } - - - cluster_type_output_node_pools_names = { - regional = concat(google_container_node_pool.pools.*.name, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.name, [""]) - } - - cluster_type_output_node_pools_versions = { - regional = concat(google_container_node_pool.pools.*.version, [""]) - zonal = concat(google_container_node_pool.zonal_pools.*.version, [""]) - } - - cluster_master_auth_list_layer1 = local.cluster_type_output_master_auth[local.cluster_type] + cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] # cluster locals - cluster_name = local.cluster_type_output_name[local.cluster_type] - cluster_location = local.cluster_type_output_location[local.cluster_type] - cluster_region = local.cluster_type_output_region[local.cluster_type] - cluster_zones = sort(local.cluster_type_output_zones[local.cluster_type]) - cluster_endpoint = local.cluster_type_output_endpoint[local.cluster_type] + cluster_name = local.cluster_output_name + cluster_location = local.cluster_output_location + cluster_region = local.cluster_output_region + cluster_zones = sort(local.cluster_output_zones) + cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] - cluster_master_version = local.cluster_type_output_master_version[local.cluster_type] - cluster_min_master_version = local.cluster_type_output_min_master_version[local.cluster_type] - cluster_logging_service = local.cluster_type_output_logging_service[local.cluster_type] - cluster_monitoring_service = local.cluster_type_output_monitoring_service[local.cluster_type] - cluster_node_pools_names = local.cluster_type_output_node_pools_names[local.cluster_type] - cluster_node_pools_versions = local.cluster_type_output_node_pools_versions[local.cluster_type] - cluster_network_policy_enabled = ! local.cluster_type_output_network_policy_enabled[local.cluster_type] - cluster_http_load_balancing_enabled = ! local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] - cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] - cluster_kubernetes_dashboard_enabled = ! local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] + cluster_master_version = local.cluster_output_master_version + cluster_min_master_version = local.cluster_output_min_master_version + cluster_logging_service = local.cluster_output_logging_service + cluster_monitoring_service = local.cluster_output_monitoring_service + cluster_node_pools_names = local.cluster_output_node_pools_names + cluster_node_pools_versions = local.cluster_output_node_pools_versions + cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled + cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled + cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled + cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled } /****************************************** Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - provider = google-beta - region = var.region + location = local.location project = var.project_id } @@ -265,7 +119,6 @@ data "google_container_engine_versions" "zone" { // // data.google_container_engine_versions.zone: Cannot determine zone: set in this resource, or set provider-level zone. // - zone = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] - - project = var.project_id + location = local.zone_count == 0 ? data.google_compute_zones.available.names[0] : var.zones[0] + project = var.project_id } diff --git a/modules/private-cluster/masq.tf b/modules/private-cluster/masq.tf index 1e9dc7791d..b6e411fc42 100644 --- a/modules/private-cluster/masq.tf +++ b/modules/private-cluster/masq.tf @@ -44,7 +44,5 @@ EOF data.google_client_config.default, google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 3beb362d25..dea7b5c7b5 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/outputs.tf b/outputs.tf index 3beb362d25..dea7b5c7b5 100644 --- a/outputs.tf +++ b/outputs.tf @@ -54,8 +54,6 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, - google_container_cluster.zonal_primary, - google_container_node_pool.zonal_pools, ] } diff --git a/sa.tf b/sa.tf index 925ee282a9..0ebe86003d 100644 --- a/sa.tf +++ b/sa.tf @@ -24,7 +24,7 @@ locals { ), ) // if user set var.service_accont it will be used even if var.create_service_account==true, so service account will be created but not used - service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? element(local.service_account_list, 0) : var.service_account + service_account = (var.service_account == "" || var.service_account == "create") && var.create_service_account ? local.service_account_list[0] : var.service_account } resource "random_string" "cluster_service_account_suffix" { diff --git a/test/fixtures/simple_regional/example.tf b/test/fixtures/simple_regional/example.tf index cb37e43427..a03fadb28b 100644 --- a/test/fixtures/simple_regional/example.tf +++ b/test/fixtures/simple_regional/example.tf @@ -26,4 +26,3 @@ module "example" { ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_account } - diff --git a/test/make.sh b/test/make.sh index 3abee2e042..e3937328c0 100755 --- a/test/make.sh +++ b/test/make.sh @@ -38,6 +38,7 @@ find_files() { find "${pth}" '(' \ -path '*/.git' -o \ -path '*/.terraform' -o \ + -path '*/.kitchen' -o \ -path './autogen' -o \ -path './test/fixtures/all_examples' -o \ -path './test/fixtures/shared' ')' \ @@ -46,7 +47,7 @@ find_files() { # Compatibility with both GNU and BSD style xargs. compat_xargs() { - local compat=() + local compat=() rval # Test if xargs is GNU or BSD style. GNU xargs will succeed with status 0 # when given --no-run-if-empty and no input on STDIN. BSD xargs will fail and # exit status non-zero If xargs fails, assume it is BSD style and proceed. @@ -55,6 +56,11 @@ compat_xargs() { compat=("--no-run-if-empty") fi xargs "${compat[@]}" "$@" + rval="$?" + if [[ -z "${NOWARN:-}" ]] && [[ "${rval}" -gt 0 ]]; then + echo "Warning: compat_xargs $* failed with exit code ${rval}" >&2 + fi + return "${rval}" } # This function makes sure that the required files for @@ -67,24 +73,28 @@ function basefiles() { done } -# This function runs the hadolint linter on -# every file named 'Dockerfile' -function docker() { - echo "Running hadolint on Dockerfiles" - find_files . -name "Dockerfile" -print0 \ - | compat_xargs -0 hadolint -} - # This function runs 'terraform validate' and 'terraform fmt' # against all directory paths which contain *.tf files. function check_terraform() { - set -e - echo "Running terraform validate and terraform fmt" + local rval=125 + # fmt is before validate for faster feedback, validate requires terraform + # init which takes time. + echo "Running terraform fmt" find_files . -name "*.tf" -print0 \ | compat_xargs -0 -n1 dirname \ | sort -u \ - | compat_xargs -t -n1 -i{} bash -c \ - 'terraform init "{}" && terraform validate "{}" && terraform fmt -check=true -write=false -diff "{}"' + | compat_xargs -t -n1 terraform fmt -diff -check=true -write=false + rval="$?" + if [[ "${rval}" -gt 0 ]]; then + echo "Error: terraform fmt failed with exit code ${rval}" >&2 + echo "Check the output for diffs and correct using terraform fmt " >&2 + return "${rval}" + fi + echo "Running terraform validate" + find_files . -not -path "./test/fixtures/shared/*" -name "*.tf" -print0 \ + | compat_xargs -0 -n1 dirname \ + | sort -u \ + | compat_xargs -t -n1 helpers/terraform_validate } # This function runs 'go fmt' and 'go vet' on every file @@ -118,7 +128,7 @@ function check_trailing_whitespace() { echo "Checking for trailing whitespace" find_files . -print \ | grep -v -E '\.(pyc|png)$' \ - | compat_xargs grep -H -n '[[:blank:]]$' + | NOWARN=1 compat_xargs grep -H -n '[[:blank:]]$' rc=$? if [[ ${rc} -eq 0 ]]; then return 1 @@ -132,15 +142,18 @@ function generate() { function generate_docs() { echo "Generating markdown docs with terraform-docs" - local path - while read -r path; do - if [[ -e "${path}/README.md" ]]; then - # script seem to be designed to work into current directory - cd "${path}" && echo "Working in ${path} ..." - terraform_docs.sh . && echo Success! || echo "Warning! Exit code: ${?}" - cd - >/dev/null + local pth helper_dir rval + helper_dir="$(pwd)/helpers" + while read -r pth; do + if [[ -e "${pth}/README.md" ]]; then + (cd "${pth}" || return 3; "${helper_dir}"/terraform_docs .;) + rval="$?" + if [[ "${rval}" -gt 0 ]]; then + echo "Error: terraform_docs in ${pth} exit code: ${rval}" >&2 + return "${rval}" + fi else - echo "Skipping ${path} because README.md does not exist." + echo "Skipping ${pth} because README.md does not exist." fi done < <(find_files . -name '*.tf' -print0 \ | compat_xargs -0 -n1 dirname \