Merge remote-tracking branch 'etienne/hieradata_node'

mboisson · Feb 27, 2024 · d63072b · d63072b
2 parents 3c88855 + 2746abf
commit d63072b
Show file tree

Hide file tree

Showing 10 changed files with 166 additions and 47 deletions.
diff --git a/aws/infrastructure.tf b/aws/infrastructure.tf
@@ -34,15 +34,16 @@ module "configuration" {
 }
 
 module "provision" {
-  source          = "../common/provision"
-  bastions        = module.configuration.bastions
-  puppetservers   = module.configuration.puppetservers
-  tf_ssh_key      = module.configuration.ssh_key
-  terraform_data  = module.configuration.terraform_data
-  terraform_facts = module.configuration.terraform_facts
-  hieradata       = var.hieradata
-  sudoer_username = var.sudoer_username
-  depends_on      = [aws_instance.instances, aws_eip.public_ip]
+  source           = "../common/provision"
+  bastions         = module.configuration.bastions
+  puppetservers    = module.configuration.puppetservers
+  tf_ssh_key       = module.configuration.ssh_key
+  terraform_data   = module.configuration.terraform_data
+  terraform_facts  = module.configuration.terraform_facts
+  hieradata        = var.hieradata
+  hieradata_folder = var.hieradata_folder_path
+  sudoer_username  = var.sudoer_username
+  depends_on       = [aws_instance.instances, aws_eip.public_ip]
 }
 
 data "aws_availability_zones" "available" {

diff --git a/azure/infrastructure.tf b/azure/infrastructure.tf
@@ -34,15 +34,16 @@ module "configuration" {
 }
 
 module "provision" {
-  source          = "../common/provision"
-  bastions        = module.configuration.bastions
-  puppetservers   = module.configuration.puppetservers
-  tf_ssh_key      = module.configuration.ssh_key
-  terraform_data  = module.configuration.terraform_data
-  terraform_facts = module.configuration.terraform_facts
-  hieradata       = var.hieradata
-  sudoer_username = var.sudoer_username
-  depends_on      = [ azurerm_linux_virtual_machine.instances ]
+  source           = "../common/provision"
+  bastions         = module.configuration.bastions
+  puppetservers    = module.configuration.puppetservers
+  tf_ssh_key       = module.configuration.ssh_key
+  terraform_data   = module.configuration.terraform_data
+  terraform_facts  = module.configuration.terraform_facts
+  hieradata        = var.hieradata
+  hieradata_folder = var.hieradata_folder_path
+  sudoer_username  = var.sudoer_username
+  depends_on       = [ azurerm_linux_virtual_machine.instances ]
 }
 
 

diff --git a/common/configuration/main.tf b/common/configuration/main.tf
@@ -84,7 +84,7 @@ locals {
   })
 
   terraform_facts = yamlencode({
-    software_stack = var.software_stack
+    software_stack = var.software_stack,
     cloud          = {
       provider = var.cloud_provider
       region = var.cloud_region
@@ -98,6 +98,7 @@ locals {
         cloud_provider        = var.cloud_provider
         tags                  = values.tags
         node_name             = key,
+        node_prefix           = values.prefix,
         domain_name           = var.domain_name
         puppetenv_git         = var.config_git_url,
         puppetenv_rev         = var.config_version,
@@ -165,4 +166,4 @@ output "bastions" {
     for host, values in var.inventory: host => values
     if contains(values.tags, var.bastion_tag) && contains(values.tags, "public") &&  (!contains(values.tags, "pool"))
   }
-}
+}
diff --git a/common/configuration/puppet.yaml b/common/configuration/puppet.yaml
@@ -85,6 +85,7 @@ runcmd:
   - chgrp puppet /etc/puppetlabs/data /etc/puppetlabs/facts
   - ln -sf /etc/puppetlabs/data/terraform_data.yaml /etc/puppetlabs/code/environments/production/data/
   - ln -sf /etc/puppetlabs/data/user_data.yaml /etc/puppetlabs/code/environments/production/data/
+  - ln -sf /etc/puppetlabs/data/hieradata /etc/puppetlabs/code/environments/production/data/
   - ln -sf /etc/puppetlabs/facts/terraform_facts.yaml /etc/puppetlabs/code/environments/production/site/profile/facts.d
 # We use r10k solely to install the modules of the main branch environment.
   - "(cd /etc/puppetlabs/code/environments/production; /opt/puppetlabs/puppet/bin/r10k puppetfile install)"
@@ -139,8 +140,17 @@ write_files:
           %{ if cloud_provider != "gcp" } "GCE", %{ endif }
         ],
       }
+      global : {
+        external-dir : ["/etc/puppetlabs/facts/external"],
+      }
+
     path: /etc/puppetlabs/facter/facter.conf
     permissions: "0644"
+  - path: /etc/puppetlabs/facts/external/prefix.yaml
+    content: |
+      ---
+      prefix : "${node_prefix}"
+    permissions: "0640"
 %{ if contains(tags, "puppet") ~}
   - content: |
       ---

diff --git a/common/outputs.tf b/common/outputs.tf
@@ -45,4 +45,4 @@ output "accounts" {
 output "ssh_private_key" {
   value     = module.configuration.ssh_key.private
   sensitive = true
-}
+}
diff --git a/common/provision/main.tf b/common/provision/main.tf
@@ -2,28 +2,93 @@ variable "bastions" { }
 variable "puppetservers" { }
 variable "terraform_data" { }
 variable "terraform_facts" { }
+variable "hieradata_folder" { }
 variable "hieradata" { }
 variable "sudoer_username" { }
 variable "tf_ssh_key" { }
 
-resource "terraform_data" "deploy_hieradata" {
-  for_each = length(var.bastions) > 0  ? var.puppetservers : { }
+data "local_file" "hieradata_yaml" {
+  for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "*.yaml") : []
+  filename = "${var.hieradata_folder}/${each.value}"
+}
 
-  connection {
-    type                = "ssh"
+data "local_file" "hieradata_subfolder" {
+  for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "{prefix,hostname}/**/*.yaml") : []
+  filename = "${var.hieradata_folder}/${each.value}"
+}
+
+locals {
+  connection_parameters = length(var.bastions) > 0 ? {
     bastion_host        = var.bastions[keys(var.bastions)[0]].public_ip
     bastion_user        = var.sudoer_username
     bastion_private_key = var.tf_ssh_key.private
     user                = var.sudoer_username
-    host                = each.value
     private_key         = var.tf_ssh_key.private
+  } : null
+
+  hieradata_md5 = merge(
+    {for value in data.local_file.hieradata_yaml: replace(value.filename, var.hieradata_folder, "") => value.content_md5},
+    {for value in data.local_file.hieradata_subfolder: replace(value.filename, var.hieradata_folder, "") => value.content_md5},
+  )
+  triggers_hieradata_folder = {
+    hieradata_yaml  = local.hieradata_md5
   }
 
-  triggers_replace = {
+  triggers_hieradata = {
     user_data      = md5(var.hieradata)
     terraform_data = md5(var.terraform_data)
     facts          = md5(var.terraform_facts)
   }
+}
+
+resource "terraform_data" "deploy_hieradata_folder" {
+  for_each = local.connection_parameters != null && length(local.hieradata_md5) > 0 ? var.puppetservers : { }
+
+  connection {
+    type                = "ssh"
+    host                = each.value
+    bastion_host        = local.connection_parameters["bastion_host"]
+    bastion_user        = local.connection_parameters["bastion_user"]
+    bastion_private_key = local.connection_parameters["bastion_private_key"]
+    user                = local.connection_parameters["user"]
+    private_key         = local.connection_parameters["private_key"]
+  }
+
+  triggers_replace = local.triggers_hieradata_folder
+
+  provisioner "file" {
+    source      = "${path.cwd}/hieradata"
+    destination = "hieradata"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      # clean up
+      "sudo rm -rf /etc/puppetlabs/data/hieradata || true",
+      "sudo mkdir -p /etc/puppetlabs/data",
+      # puppet user and group have been assigned the reserved UID/GID 52
+      "sudo cp -r hieradata /etc/puppetlabs/data/",
+      "sudo chown -R root:52 /etc/puppetlabs/data/hieradata",
+      "sudo chmod -R 650 /etc/puppetlabs/data/hieradata",
+      "rm -rf hieradata",
+    ]
+  }
+}
+
+resource "terraform_data" "deploy_hieradata" {
+  for_each = local.connection_parameters != null ? var.puppetservers : { }
+
+  connection {
+    type                = "ssh"
+    host                = each.value
+    bastion_host        = local.connection_parameters["bastion_host"]
+    bastion_user        = local.connection_parameters["bastion_user"]
+    bastion_private_key = local.connection_parameters["bastion_private_key"]
+    user                = local.connection_parameters["user"]
+    private_key         = local.connection_parameters["private_key"]
+  }
+
+  triggers_replace = local.triggers_hieradata
 
   provisioner "file" {
     content     = var.terraform_data
@@ -47,7 +112,36 @@ resource "terraform_data" "deploy_hieradata" {
       "sudo install -o root -g 52 -m 640 terraform_data.yaml user_data.yaml /etc/puppetlabs/data/",
       "sudo install -o root -g 52 -m 640 terraform_facts.yaml /etc/puppetlabs/facts/",
       "rm -f terraform_data.yaml user_data.yaml terraform_facts.yaml",
-      "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true",
+    ]
+  }
+}
+
+resource "terraform_data" "update_consul" {
+  for_each = local.connection_parameters != null ? var.puppetservers : { }
+
+  connection {
+    type                = "ssh"
+    host                = each.value
+    bastion_host        = local.connection_parameters["bastion_host"]
+    bastion_user        = local.connection_parameters["bastion_user"]
+    bastion_private_key = local.connection_parameters["bastion_private_key"]
+    user                = local.connection_parameters["user"]
+    private_key         = local.connection_parameters["private_key"]
+  }
+
+  triggers_replace = merge(
+    local.triggers_hieradata,
+    local.triggers_hieradata_folder,
+  )
+
+  depends_on = [
+      terraform_data.deploy_hieradata,
+      terraform_data.deploy_hieradata_folder,
+    ]
+
+  provisioner "remote-exec" {
+    inline = [
+      "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true"
     ]
   }
 }
diff --git a/common/variables.tf b/common/variables.tf
@@ -94,6 +94,12 @@ variable "hieradata" {
   }
 }
 
+variable "hieradata_folder_path" {
+  type        = string
+  default     = ""
+  description = "Path to hieradata folder containing YAML files to be included in the puppet environment"
+}
+
 variable "sudoer_username" {
   type        = string
   default     = "centos"

diff --git a/docs/README.md b/docs/README.md
@@ -884,6 +884,10 @@ the provided content will replace entirely the Magic Castle environment's
 **Post build modification effect**: None. To modify the Puppetfile after the cluster is initialized, log
 on the Puppet server and modify `/etc/puppetlabs/code/environments/production/Puppetfile`.
 
+### 4.20 hieradata_folder_path (optional)
+
+**default value**: empty string
+
 ## 5. Cloud Specific Configuration
 
 ### 5.1 Amazon Web Services

diff --git a/gcp/infrastructure.tf b/gcp/infrastructure.tf
@@ -34,15 +34,16 @@ module "configuration" {
 }
 
 module "provision" {
-  source          = "../common/provision"
-  bastions        = module.configuration.bastions
-  puppetservers   = module.configuration.puppetservers
-  tf_ssh_key      = module.configuration.ssh_key
-  terraform_data  = module.configuration.terraform_data
-  terraform_facts = module.configuration.terraform_facts
-  hieradata       = var.hieradata
-  sudoer_username = var.sudoer_username
-  depends_on      = [ google_compute_instance.instances ]
+  source           = "../common/provision"
+  bastions         = module.configuration.bastions
+  puppetservers    = module.configuration.puppetservers
+  tf_ssh_key       = module.configuration.ssh_key
+  terraform_data   = module.configuration.terraform_data
+  terraform_facts  = module.configuration.terraform_facts
+  hieradata        = var.hieradata
+  hieradata_folder = var.hieradata_folder_path
+  sudoer_username  = var.sudoer_username
+  depends_on       = [ google_compute_instance.instances ]
 }
 
 

diff --git a/openstack/infrastructure.tf b/openstack/infrastructure.tf
@@ -29,15 +29,16 @@ module "configuration" {
 }
 
 module "provision" {
-  source          = "../common/provision"
-  bastions        = module.configuration.bastions
-  puppetservers   = module.configuration.puppetservers
-  tf_ssh_key      = module.configuration.ssh_key
-  terraform_data  = module.configuration.terraform_data
-  terraform_facts = module.configuration.terraform_facts
-  hieradata       = var.hieradata
-  sudoer_username = var.sudoer_username
-  depends_on      = [local.network_provision_dep]
+  source           = "../common/provision"
+  bastions         = module.configuration.bastions
+  puppetservers    = module.configuration.puppetservers
+  tf_ssh_key       = module.configuration.ssh_key
+  terraform_data   = module.configuration.terraform_data
+  terraform_facts  = module.configuration.terraform_facts
+  hieradata        = var.hieradata
+  hieradata_folder = var.hieradata_folder_path
+  sudoer_username  = var.sudoer_username
+  depends_on       = [local.network_provision_dep]
 }
 
 data "openstack_images_image_v2" "image" {
@@ -137,4 +138,4 @@ locals {
     host => merge(module.configuration.inventory[host], {id=openstack_compute_instance_v2.instances[host].id})
     if contains(module.configuration.inventory[host].tags, "public")
   }
-}
+}