From cf7db00062107ee33164fd1f42ff838a70ef41dd Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 30 Sep 2021 16:03:26 +0200 Subject: [PATCH 01/20] Add ha option --- defaults/main.yml | 4 +++- tasks/main.yaml | 17 +++++++++++++++++ tasks/wn.yaml | 19 ++++++------------- templates/kubeadm-config.j2 | 5 ++++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 80aa6b04..1fb5c0d1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,7 +1,9 @@ # Version to install or latest kube_version: 1.19.14 -# Type of node front or wn +# Type of node front, control_plane or wn kube_type_of_node: front +# Endpoint for the control plane in case of HA mode with multiple master +kube_control_plane_endpoint: "" # IP address or name of the Kube front node kube_server: "{{ ansible_default_ipv4.address }}" # Token diff --git a/tasks/main.yaml b/tasks/main.yaml index 0eee5def..f760c598 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -50,5 +50,22 @@ - name: Include "{{ansible_os_family}}" Kubernetes recipe include_tasks: "{{ansible_os_family}}.yaml" +- name: Add KUBELET_EXTRA_ARGS + lineinfile: + dest: "{{item}}/kubelet" + line: 'KUBELET_EXTRA_ARGS=--cgroup-driver=systemd {{kubelet_extra_args}}' + regexp: 'KUBELET_EXTRA_ARGS=' + create: yes + notify: restart kubelet + with_first_found: + - files: + - /etc/sysconfig/ + - /etc/default/ + - name: Include "{{kube_type_of_node}}" tasks include_tasks: "{{kube_type_of_node}}.yaml" + when: kube_type_of_node == "front" or kube_type_of_node == "wn" + +- name: Include control_plane tasks + include_tasks: "wn.yaml" + when: kube_type_of_node == "control_plane" \ No newline at end of file diff --git a/tasks/wn.yaml b/tasks/wn.yaml index d3daa3a3..a4dfa1ab 100644 --- a/tasks/wn.yaml +++ b/tasks/wn.yaml @@ -5,17 +5,10 @@ search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf" delegate_to: "{{kube_server}}" -- name: Add KUBELET_EXTRA_ARGS - lineinfile: - dest: "{{item}}/kubelet" - line: 'KUBELET_EXTRA_ARGS=--cgroup-driver=systemd {{kubelet_extra_args}}' - regexp: 'KUBELET_EXTRA_ARGS=' - create: yes - notify: restart kubelet - with_first_found: - - files: - - /etc/sysconfig/ - - /etc/default/ - -- name: Add node to kube cluster +- name: Add WN to kube cluster command: kubeadm join --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf + when: kube_type_of_node == "wn" + +- name: Add node to kube cluster control_plane + command: kubeadm join --control-plane --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf + when: kube_type_of_node == "control_plane" \ No newline at end of file diff --git a/templates/kubeadm-config.j2 b/templates/kubeadm-config.j2 index 0330ff97..273208eb 100644 --- a/templates/kubeadm-config.j2 +++ b/templates/kubeadm-config.j2 @@ -16,4 +16,7 @@ apiVersion: kubeadm.k8s.io/v1beta2 bootstrapTokens: - token: "{{kube_token}}" # --token description: "kubeadm bootstrap token" - ttl: "{{kube_token_ttl}}" # --token-ttl \ No newline at end of file + ttl: "{{kube_token_ttl}}" # --token-ttl +{% if kube_control_plane_endpoint != "" %} +controlPlaneEndpoint: {{kube_control_plane_endpoint}} +{% endif %} \ No newline at end of file From f157b604cde784a5663c7d4eca677457129e9406 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 5 Oct 2021 09:04:26 +0200 Subject: [PATCH 02/20] Fix --- tasks/front.yaml | 2 +- templates/kubeadm-config.j2 | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/front.yaml b/tasks/front.yaml index 4bf2b04a..cff35fe1 100644 --- a/tasks/front.yaml +++ b/tasks/front.yaml @@ -9,7 +9,7 @@ template: src=kubeadm-config.j2 dest=/tmp/kubeadm-config.yml - name: Kubeadm init - command: kubeadm init --config /tmp/kubeadm-config.yml creates=/etc/kubernetes/admin.conf + command: kubeadm init --config /tmp/kubeadm-config.yml --upload-certs creates=/etc/kubernetes/admin.conf - name: Set kube_wait_api_server_ip set_fact: diff --git a/templates/kubeadm-config.j2 b/templates/kubeadm-config.j2 index 273208eb..511cdece 100644 --- a/templates/kubeadm-config.j2 +++ b/templates/kubeadm-config.j2 @@ -6,6 +6,9 @@ networking: apiServer: extraArgs: advertise-address: "{{kube_api_server}}" # --apiserver-advertise-address +{% if kube_control_plane_endpoint != "" %} +controlPlaneEndpoint: {{kube_control_plane_endpoint}} +{% endif %} --- kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 @@ -17,6 +20,3 @@ bootstrapTokens: - token: "{{kube_token}}" # --token description: "kubeadm bootstrap token" ttl: "{{kube_token_ttl}}" # --token-ttl -{% if kube_control_plane_endpoint != "" %} -controlPlaneEndpoint: {{kube_control_plane_endpoint}} -{% endif %} \ No newline at end of file From e1f94eb4a0132271a3975923a4543844144b8180 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 10:46:33 +0200 Subject: [PATCH 03/20] Ad HA support --- defaults/main.yml | 15 ++++++++-- files/kube-vip.yaml | 30 +++++++++++++++++++ tasks/control_plane.yaml | 21 +++++++++++++ tasks/etcd.yaml | 51 ++++++++++++++++++++++++++++++++ tasks/front.yaml | 8 +++++ tasks/lb.yaml | 10 +++++++ templates/kube-vip-config.j2 | 27 +++++++++++++++++ templates/kubeadm-config-etcd.j2 | 32 ++++++++++++++++++++ templates/kubeadm-config.j2 | 14 +++++++-- 9 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 files/kube-vip.yaml create mode 100644 tasks/control_plane.yaml create mode 100644 tasks/etcd.yaml create mode 100644 tasks/lb.yaml create mode 100644 templates/kube-vip-config.j2 create mode 100644 templates/kubeadm-config-etcd.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 1fb5c0d1..8972046a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -2,8 +2,6 @@ kube_version: 1.19.14 # Type of node front, control_plane or wn kube_type_of_node: front -# Endpoint for the control plane in case of HA mode with multiple master -kube_control_plane_endpoint: "" # IP address or name of the Kube front node kube_server: "{{ ansible_default_ipv4.address }}" # Token @@ -82,4 +80,15 @@ docker_nvidia_options: path: /usr/bin/nvidia-container-runtime runtimeArgs: [] # Install docker with pip -kube_install_docker_pip: false \ No newline at end of file +kube_install_docker_pip: false + +# Endpoint for the control plane in case of HA mode with multiple master +kube_control_plane_ip: "" +kube_control_plane_port: 8443 +kube_control_plane_peer_ip: "" +kube_control_plane_peer_iface: "" +kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] +# ETCD Peer adress +kube_etcd_peer_address: "kube_etcd_peer_address" +kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"} +kube_etcd_peer_name: "kubeserver.localdomain" diff --git a/files/kube-vip.yaml b/files/kube-vip.yaml new file mode 100644 index 00000000..ff3b4bcc --- /dev/null +++ b/files/kube-vip.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system +spec: + containers: + - command: + - /kube-vip + - start + - -c + - /vip.yaml + image: 'plndr/kube-vip:0.1.1' + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_TIME + volumeMounts: + - mountPath: /vip.yaml + name: config + hostNetwork: true + volumes: + - hostPath: + path: /etc/kube-vip/config.yaml + name: config +status: {} \ No newline at end of file diff --git a/tasks/control_plane.yaml b/tasks/control_plane.yaml new file mode 100644 index 00000000..091c44eb --- /dev/null +++ b/tasks/control_plane.yaml @@ -0,0 +1,21 @@ +--- +- name: Configure VIP LB + import_tasks: lb.yaml + +- name: Configure HA ETDC + import_tasks: etcd.yaml + +- name: Wait for Kube master + wait_for: + path: /etc/environment + search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf" + delegate_to: "{{kube_server}}" + +- name: Transfer certificates from server node + synchronize: + src: /etc/kubernetes/pki + dest: /etc/kubernetes/pki + delegate_to: "{{kube_server}}" + +- name: Add node to kube cluster control_plane + command: kubeadm join --control-plane --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf \ No newline at end of file diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml new file mode 100644 index 00000000..7665867a --- /dev/null +++ b/tasks/etcd.yaml @@ -0,0 +1,51 @@ +- name: Create kubeadm-config-etcd file + template: src=kubeadm-config-etcd.j2 dest=/tmp/kubeadm-config-etcd.yml + +- name: Create /etc/kubernetes/pki/etcd file dir + file: path=/etc/kubernetes/pki/etcd state=directory mode=755 recurse=yes + +- name: Create etcd CA cert + command: kubeadm init phase certs etcd-ca creates=/etc/kubernetes/pki/etcd/ca.crt + when: kube_type_of_node == "front" + +- block: + + - name: Wait for etcd CA cert + wait_for: + path: /etc/kubernetes/pki/etcd/ca.key + delegate_to: "{{kube_server}}" + + - name: Copy etcd CA files from master + synchronize: + src: "{{item}}" + dest: /etc/kubernetes/pki/etcd/ca.key + delegate_to: "{{kube_server}}" + with_items: + - "/etc/kubernetes/pki/etcd/ca.key" + - "/etc/kubernetes/pki/etcd/ca.crt" + + when: kube_type_of_node == "control_plane" + +- name: Create etcd server cert + command: kubeadm init phase certs etcd-server --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/server.crt + +- name: Create etcd peer cert + command: kubeadm init phase certs etcd-peer --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/peer.crt + +- name: Create etcd healthcheck client cert + command: kubeadm init phase certs etcd-healthcheck-client --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/healthcheck-client.crt + +- name: Create apiserver-etcd-client server cert + command: kubeadm init phase certs apiserver-etcd-client --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/apiserver-etcd-client.crt + +- name: Create etcd manifest + command: kubeadm init phase etcd local --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/manifests/etcd.yaml + +- name: Create k8s ca cert + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/ca.crt + +- name: Configure kubelet + command: kubeadm init phase kubeconfig kubelet --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/kubelet.conf + +- name: Configure kubelet + command: kubeadm init phase kubelet-start --config=/tmp/kubeadm-config-etcd.yaml creates=/var/lib/kubelet/config.yaml diff --git a/tasks/front.yaml b/tasks/front.yaml index cff35fe1..88eaf934 100644 --- a/tasks/front.yaml +++ b/tasks/front.yaml @@ -5,6 +5,14 @@ - name: force handlers meta: flush_handlers +- block: + - name: Configure VIP LB + import_tasks: lb.yaml + + - name: Configure HA ETDC + import_tasks: etcd.yaml + when: kube_control_plane_ip != "" + - name: Create kubeadm-config file template: src=kubeadm-config.j2 dest=/tmp/kubeadm-config.yml diff --git a/tasks/lb.yaml b/tasks/lb.yaml new file mode 100644 index 00000000..a804cae2 --- /dev/null +++ b/tasks/lb.yaml @@ -0,0 +1,10 @@ +- name: Create /etc/kube-vip file dir + file: path=/etc/kube-vip state=directory mode=755 + +- name: Create kube-vip config file + template: src=kube-vip-config.j2 dest=/etc/kube-vip/config.yaml + +- name: Create /etc/kubernetes/manifests file dir + file: path=/etc/kubernetes/manifests state=directory mode=755 recurse=yes + +- copy: src=kube-vip.yaml dest=/etc/kubernetes/manifests/kube-vip.yaml \ No newline at end of file diff --git a/templates/kube-vip-config.j2 b/templates/kube-vip-config.j2 new file mode 100644 index 00000000..69f504f1 --- /dev/null +++ b/templates/kube-vip-config.j2 @@ -0,0 +1,27 @@ +localPeer: + id: 0 + address: {{kube_control_plane_peer_ip}} + port: 10000 +remotePeers: +{% for peer_ip in kube_control_plane_remote_peer_list %} +- id: {{loop.index + 1}} + address: {{peer_ip}} + port: 10000 +{% endfor %}" +vip: {{kube_control_plane_ip}} +gratuitousARP: true +singleNode: false +startAsLeader: true +interface: {{kube_control_plane_peer_iface}} +loadBalancers: +- name: API Server Load Balancer + type: tcp + port: {{kube_control_plane_port}} + bindToVip: false + backends: + - port: 6443 + address: {{kube_control_plane_peer_ip}} +{% for peer_ip in kube_control_plane_remote_peer_list %} + - port: 6443 + address: {{peer_ip}} +{% endfor %}" \ No newline at end of file diff --git a/templates/kubeadm-config-etcd.j2 b/templates/kubeadm-config-etcd.j2 new file mode 100644 index 00000000..ae3098f5 --- /dev/null +++ b/templates/kubeadm-config-etcd.j2 @@ -0,0 +1,32 @@ + +--- +kind: ClusterConfiguration +apiVersion: kubeadm.k8s.io/v1beta2 +networking: + podSubnet: "{{kube_pod_network_cidr}}" +apiServer: + extraArgs: + advertise-address: "{{kube_api_server}}" +controlPlaneEndpoint: {{kube_control_plane_ip}}:{{kube_control_plane_port}} +etcd: + local: + extraArgs: + listen-client-urls: "https://{{kube_etcd_peer_address}}:2379" + listen-peer-urls: "https://{{kube_etcd_peer_address}}:2380" + advertise-client-urls: "https://{{kube_etcd_peer_address}}:2379" + initial-advertise-peer-urls: "https://{{kube_etcd_peer_address}}:2380" + initial-cluster: " + {%- for peer_name, peer_ip in kube_etcd_peer_list.items() -%} + {%- if loop.index -%} , {%- endif -%}" + {{peer_name}}=https://{{peer_ip}}:2380 + {%- endfor -%}" + name: "{{kube_etcd_peer_name}}" + initial-cluster-state: "new" + peerCertSANs: + - "{{kube_etcd_peer_address}}" + serverCertSANs: + - "{{kube_etcd_peer_address}}" +--- +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +cgroupDriver: systemd \ No newline at end of file diff --git a/templates/kubeadm-config.j2 b/templates/kubeadm-config.j2 index 511cdece..e4c9e30a 100644 --- a/templates/kubeadm-config.j2 +++ b/templates/kubeadm-config.j2 @@ -6,8 +6,17 @@ networking: apiServer: extraArgs: advertise-address: "{{kube_api_server}}" # --apiserver-advertise-address -{% if kube_control_plane_endpoint != "" %} -controlPlaneEndpoint: {{kube_control_plane_endpoint}} +{% if kube_control_plane_ip != "" %} +controlPlaneEndpoint: {{kube_control_plane_ip}}:{{kube_control_plane_port}} +etcd: + external: + endpoints: + {%- for peer_name, peer_ip in kube_etcd_peer_list.items() %} + - http://{{peer_ip}}:2379 + {% endfor -%} + caFile: /etc/kubernetes/pki/etcd/ca.crt + certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt + keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key {% endif %} --- kind: KubeletConfiguration @@ -20,3 +29,4 @@ bootstrapTokens: - token: "{{kube_token}}" # --token description: "kubeadm bootstrap token" ttl: "{{kube_token_ttl}}" # --token-ttl +certificateKey: {{}} From 3b55fea5df1192955d0c3ab054d48f825da60cdf Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 11:27:15 +0200 Subject: [PATCH 04/20] Ad HA support --- defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 8972046a..fe502db0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -89,6 +89,6 @@ kube_control_plane_peer_ip: "" kube_control_plane_peer_iface: "" kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] # ETCD Peer adress -kube_etcd_peer_address: "kube_etcd_peer_address" +kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}" kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"} kube_etcd_peer_name: "kubeserver.localdomain" From c36be5d1b43c5cc2982c77d61b30af1c43bf2143 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 12:52:31 +0200 Subject: [PATCH 05/20] Ad HA support --- defaults/main.yml | 1 + tasks/control_plane.yaml | 8 +------- tasks/etcd.yaml | 2 +- tasks/front.yaml | 9 ++++++++- templates/kubeadm-config-etcd.j2 | 6 ++---- templates/kubeadm-config.j2 | 8 +++++--- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index fe502db0..fc01ff8b 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -88,6 +88,7 @@ kube_control_plane_port: 8443 kube_control_plane_peer_ip: "" kube_control_plane_peer_iface: "" kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] +kube_certificate_key: "2a22bafb08c4d49a5f8d587134bd5647cf5c0f9eeb9291c935a6495cdc59a03c" # ETCD Peer adress kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}" kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"} diff --git a/tasks/control_plane.yaml b/tasks/control_plane.yaml index 091c44eb..c8170d73 100644 --- a/tasks/control_plane.yaml +++ b/tasks/control_plane.yaml @@ -11,11 +11,5 @@ search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf" delegate_to: "{{kube_server}}" -- name: Transfer certificates from server node - synchronize: - src: /etc/kubernetes/pki - dest: /etc/kubernetes/pki - delegate_to: "{{kube_server}}" - - name: Add node to kube cluster control_plane - command: kubeadm join --control-plane --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf \ No newline at end of file + command: kubeadm join --control-plane --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf \ No newline at end of file diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml index 7665867a..6c937bda 100644 --- a/tasks/etcd.yaml +++ b/tasks/etcd.yaml @@ -1,5 +1,5 @@ - name: Create kubeadm-config-etcd file - template: src=kubeadm-config-etcd.j2 dest=/tmp/kubeadm-config-etcd.yml + template: src=kubeadm-config-etcd.j2 dest=/tmp/kubeadm-config-etcd.yaml - name: Create /etc/kubernetes/pki/etcd file dir file: path=/etc/kubernetes/pki/etcd state=directory mode=755 recurse=yes diff --git a/tasks/front.yaml b/tasks/front.yaml index 88eaf934..d98c3c47 100644 --- a/tasks/front.yaml +++ b/tasks/front.yaml @@ -5,19 +5,26 @@ - name: force handlers meta: flush_handlers +- set_fact: + init_params: "" + - block: - name: Configure VIP LB import_tasks: lb.yaml - name: Configure HA ETDC import_tasks: etcd.yaml + + - set_fact: + init_params: "--upload-certs --ignore-preflight-errors=all" + when: kube_control_plane_ip != "" - name: Create kubeadm-config file template: src=kubeadm-config.j2 dest=/tmp/kubeadm-config.yml - name: Kubeadm init - command: kubeadm init --config /tmp/kubeadm-config.yml --upload-certs creates=/etc/kubernetes/admin.conf + command: kubeadm init --config /tmp/kubeadm-config.yml {{init_params}} creates=/etc/kubernetes/admin.conf - name: Set kube_wait_api_server_ip set_fact: diff --git a/templates/kubeadm-config-etcd.j2 b/templates/kubeadm-config-etcd.j2 index ae3098f5..aaabf4bb 100644 --- a/templates/kubeadm-config-etcd.j2 +++ b/templates/kubeadm-config-etcd.j2 @@ -1,4 +1,3 @@ - --- kind: ClusterConfiguration apiVersion: kubeadm.k8s.io/v1beta2 @@ -15,9 +14,8 @@ etcd: listen-peer-urls: "https://{{kube_etcd_peer_address}}:2380" advertise-client-urls: "https://{{kube_etcd_peer_address}}:2379" initial-advertise-peer-urls: "https://{{kube_etcd_peer_address}}:2380" - initial-cluster: " - {%- for peer_name, peer_ip in kube_etcd_peer_list.items() -%} - {%- if loop.index -%} , {%- endif -%}" + initial-cluster: "{%- for peer_name, peer_ip in kube_etcd_peer_list.items() -%} + {%- if loop.index > 1 -%} , {%- endif -%} {{peer_name}}=https://{{peer_ip}}:2380 {%- endfor -%}" name: "{{kube_etcd_peer_name}}" diff --git a/templates/kubeadm-config.j2 b/templates/kubeadm-config.j2 index e4c9e30a..8f30e989 100644 --- a/templates/kubeadm-config.j2 +++ b/templates/kubeadm-config.j2 @@ -11,8 +11,8 @@ controlPlaneEndpoint: {{kube_control_plane_ip}}:{{kube_control_plane_port}} etcd: external: endpoints: - {%- for peer_name, peer_ip in kube_etcd_peer_list.items() %} - - http://{{peer_ip}}:2379 + {% for peer_name, peer_ip in kube_etcd_peer_list.items() %} + - https://{{peer_ip}}:2379 {% endfor -%} caFile: /etc/kubernetes/pki/etcd/ca.crt certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt @@ -29,4 +29,6 @@ bootstrapTokens: - token: "{{kube_token}}" # --token description: "kubeadm bootstrap token" ttl: "{{kube_token_ttl}}" # --token-ttl -certificateKey: {{}} +{% if kube_control_plane_ip != "" %} +certificateKey: {{kube_certificate_key}} +{% endif %} \ No newline at end of file From 4cff80048a42160ee14c8116af51e445ed475e56 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 13:00:06 +0200 Subject: [PATCH 06/20] Ad HA support --- templates/kube-vip-config.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/kube-vip-config.j2 b/templates/kube-vip-config.j2 index 69f504f1..7f9e7481 100644 --- a/templates/kube-vip-config.j2 +++ b/templates/kube-vip-config.j2 @@ -7,7 +7,7 @@ remotePeers: - id: {{loop.index + 1}} address: {{peer_ip}} port: 10000 -{% endfor %}" +{% endfor %} vip: {{kube_control_plane_ip}} gratuitousARP: true singleNode: false @@ -24,4 +24,4 @@ loadBalancers: {% for peer_ip in kube_control_plane_remote_peer_list %} - port: 6443 address: {{peer_ip}} -{% endfor %}" \ No newline at end of file +{% endfor %} \ No newline at end of file From 1959062ee3d5d5059dac0bf3dc29bb98d2e43c03 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 13:23:34 +0200 Subject: [PATCH 07/20] Ad HA support --- tasks/main.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tasks/main.yaml b/tasks/main.yaml index f760c598..6c91a9bb 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -63,9 +63,4 @@ - /etc/default/ - name: Include "{{kube_type_of_node}}" tasks - include_tasks: "{{kube_type_of_node}}.yaml" - when: kube_type_of_node == "front" or kube_type_of_node == "wn" - -- name: Include control_plane tasks - include_tasks: "wn.yaml" - when: kube_type_of_node == "control_plane" \ No newline at end of file + include_tasks: "{{kube_type_of_node}}.yaml" \ No newline at end of file From 10a0deb5eeca187b184356517b3ec44ba95d2e00 Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 13:42:52 +0200 Subject: [PATCH 08/20] Ad HA support --- tasks/etcd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml index 6c937bda..d2e2af93 100644 --- a/tasks/etcd.yaml +++ b/tasks/etcd.yaml @@ -18,7 +18,7 @@ - name: Copy etcd CA files from master synchronize: src: "{{item}}" - dest: /etc/kubernetes/pki/etcd/ca.key + dest: "{{item}}" delegate_to: "{{kube_server}}" with_items: - "/etc/kubernetes/pki/etcd/ca.key" From d29b1f92c947e5df5dab0f3d43b0a87d93f9083e Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 15:48:17 +0200 Subject: [PATCH 09/20] Ad HA support --- tasks/control_plane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/control_plane.yaml b/tasks/control_plane.yaml index c8170d73..c2c42778 100644 --- a/tasks/control_plane.yaml +++ b/tasks/control_plane.yaml @@ -12,4 +12,4 @@ delegate_to: "{{kube_server}}" - name: Add node to kube cluster control_plane - command: kubeadm join --control-plane --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf \ No newline at end of file + command: kubeadm join --control-plane --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification --certificate-key={{kube_certificate_key}} --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf \ No newline at end of file From 8b4a8126f87557dfd3b75a7ddcdd05712364710f Mon Sep 17 00:00:00 2001 From: micafer Date: Thu, 7 Oct 2021 16:25:33 +0200 Subject: [PATCH 10/20] Ad HA support --- tasks/front.yaml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tasks/front.yaml b/tasks/front.yaml index d98c3c47..2a8655d7 100644 --- a/tasks/front.yaml +++ b/tasks/front.yaml @@ -26,6 +26,18 @@ - name: Kubeadm init command: kubeadm init --config /tmp/kubeadm-config.yml {{init_params}} creates=/etc/kubernetes/admin.conf +- name: Add Kube API server options + lineinfile: + dest: /etc/kubernetes/manifests/kube-apiserver.yaml + line: ' - {{item.option}}={{item.value}}' + regexp: '^ - {{item.option}}=' + insertafter: ' - kube-apiserver' + notify: restart kubeapi + with_items: "{{ kube_apiserver_options }}" + +- name: force handlers + meta: flush_handlers + - name: Set kube_wait_api_server_ip set_fact: kube_wait_api_server_ip: "{{kube_api_server}}" @@ -36,6 +48,11 @@ - import_tasks: kube_nets.yaml +- name: Set KUBECONFIG environment variable + lineinfile: + dest: /etc/environment + line: "KUBECONFIG=/etc/kubernetes/admin.conf" + - import_tasks: helm.yaml environment: KUBECONFIG: /etc/kubernetes/admin.conf @@ -82,23 +99,6 @@ KUBECONFIG: /etc/kubernetes/admin.conf with_items: "{{ kube_apply_repos }}" -- name: Add Kube API server options - lineinfile: - dest: /etc/kubernetes/manifests/kube-apiserver.yaml - line: ' - {{item.option}}={{item.value}}' - regexp: '^ - {{item.option}}=' - insertafter: ' - kube-apiserver' - notify: restart kubeapi - with_items: "{{ kube_apiserver_options }}" - -- name: Set KUBECONFIG environment variable - lineinfile: - dest: /etc/environment - line: "KUBECONFIG=/etc/kubernetes/admin.conf" - -- name: force handlers - meta: flush_handlers - - import_tasks: ingress.yaml when: kube_install_ingress | bool From 444acf77b064c723699f71625b5cbc5bf1a37913 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 8 Oct 2021 08:41:17 +0200 Subject: [PATCH 11/20] Add HA support --- defaults/main.yml | 1 - tasks/control_plane.yaml | 2 +- tasks/etcd.yaml | 27 +++++++++++++++++++++------ tasks/front.yaml | 2 +- templates/kubeadm-config.j2 | 5 +---- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index fc01ff8b..fe502db0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -88,7 +88,6 @@ kube_control_plane_port: 8443 kube_control_plane_peer_ip: "" kube_control_plane_peer_iface: "" kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] -kube_certificate_key: "2a22bafb08c4d49a5f8d587134bd5647cf5c0f9eeb9291c935a6495cdc59a03c" # ETCD Peer adress kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}" kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"} diff --git a/tasks/control_plane.yaml b/tasks/control_plane.yaml index c2c42778..f52ba7d1 100644 --- a/tasks/control_plane.yaml +++ b/tasks/control_plane.yaml @@ -12,4 +12,4 @@ delegate_to: "{{kube_server}}" - name: Add node to kube cluster control_plane - command: kubeadm join --control-plane --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification --certificate-key={{kube_certificate_key}} --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf \ No newline at end of file + command: kubeadm join --control-plane --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf \ No newline at end of file diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml index d2e2af93..d48eb1da 100644 --- a/tasks/etcd.yaml +++ b/tasks/etcd.yaml @@ -4,8 +4,20 @@ - name: Create /etc/kubernetes/pki/etcd file dir file: path=/etc/kubernetes/pki/etcd state=directory mode=755 recurse=yes -- name: Create etcd CA cert - command: kubeadm init phase certs etcd-ca creates=/etc/kubernetes/pki/etcd/ca.crt +- block: + + - name: Create k8s ca cert + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/ca.crt + + - name: Create k8s sa cert + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/sa.crt + + - name: Create k8s front-proxy-ca cert + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/front-proxy-ca.crt + + - name: Create etcd CA cert + command: kubeadm init phase certs etcd-ca creates=/etc/kubernetes/pki/etcd/ca.crt + when: kube_type_of_node == "front" - block: @@ -15,7 +27,7 @@ path: /etc/kubernetes/pki/etcd/ca.key delegate_to: "{{kube_server}}" - - name: Copy etcd CA files from master + - name: Copy CA files from master synchronize: src: "{{item}}" dest: "{{item}}" @@ -23,6 +35,12 @@ with_items: - "/etc/kubernetes/pki/etcd/ca.key" - "/etc/kubernetes/pki/etcd/ca.crt" + - "/etc/kubernetes/pki/ca.crt" + - "/etc/kubernetes/pki/ca.key" + - "/etc/kubernetes/pki/sa.crt" + - "/etc/kubernetes/pki/sa.key" + - "/etc/kubernetes/pki/front-proxy-ca.crt" + - "/etc/kubernetes/pki/front-proxy-ca.key" when: kube_type_of_node == "control_plane" @@ -41,9 +59,6 @@ - name: Create etcd manifest command: kubeadm init phase etcd local --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/manifests/etcd.yaml -- name: Create k8s ca cert - command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/ca.crt - - name: Configure kubelet command: kubeadm init phase kubeconfig kubelet --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/kubelet.conf diff --git a/tasks/front.yaml b/tasks/front.yaml index 2a8655d7..bc12e6aa 100644 --- a/tasks/front.yaml +++ b/tasks/front.yaml @@ -16,7 +16,7 @@ import_tasks: etcd.yaml - set_fact: - init_params: "--upload-certs --ignore-preflight-errors=all" + init_params: "--ignore-preflight-errors=all" when: kube_control_plane_ip != "" diff --git a/templates/kubeadm-config.j2 b/templates/kubeadm-config.j2 index 8f30e989..143fecc6 100644 --- a/templates/kubeadm-config.j2 +++ b/templates/kubeadm-config.j2 @@ -28,7 +28,4 @@ apiVersion: kubeadm.k8s.io/v1beta2 bootstrapTokens: - token: "{{kube_token}}" # --token description: "kubeadm bootstrap token" - ttl: "{{kube_token_ttl}}" # --token-ttl -{% if kube_control_plane_ip != "" %} -certificateKey: {{kube_certificate_key}} -{% endif %} \ No newline at end of file + ttl: "{{kube_token_ttl}}" # --token-ttl \ No newline at end of file From 15296cf7d821c66018867f6162e2b940c4698c06 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 8 Oct 2021 09:17:57 +0200 Subject: [PATCH 12/20] Add HA support --- tasks/etcd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml index d48eb1da..54ad9e4a 100644 --- a/tasks/etcd.yaml +++ b/tasks/etcd.yaml @@ -37,7 +37,7 @@ - "/etc/kubernetes/pki/etcd/ca.crt" - "/etc/kubernetes/pki/ca.crt" - "/etc/kubernetes/pki/ca.key" - - "/etc/kubernetes/pki/sa.crt" + - "/etc/kubernetes/pki/sa.pub" - "/etc/kubernetes/pki/sa.key" - "/etc/kubernetes/pki/front-proxy-ca.crt" - "/etc/kubernetes/pki/front-proxy-ca.key" From 355a9fdb8964c09dece49a1ab024274f0171e019 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 8 Oct 2021 09:24:44 +0200 Subject: [PATCH 13/20] Add HA support --- tasks/front.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/front.yaml b/tasks/front.yaml index bc12e6aa..5a0c9c48 100644 --- a/tasks/front.yaml +++ b/tasks/front.yaml @@ -46,13 +46,13 @@ - name: wait Kube to start on "{{kube_wait_api_server_ip}}" wait_for: port=6443 host="{{kube_wait_api_server_ip}}" -- import_tasks: kube_nets.yaml - - name: Set KUBECONFIG environment variable lineinfile: dest: /etc/environment line: "KUBECONFIG=/etc/kubernetes/admin.conf" +- import_tasks: kube_nets.yaml + - import_tasks: helm.yaml environment: KUBECONFIG: /etc/kubernetes/admin.conf From 9093b5289b09e4a354f03183bb71d11a4586dca6 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 8 Oct 2021 09:26:32 +0200 Subject: [PATCH 14/20] Add HA support --- tasks/etcd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml index 54ad9e4a..d5404733 100644 --- a/tasks/etcd.yaml +++ b/tasks/etcd.yaml @@ -10,7 +10,7 @@ command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/ca.crt - name: Create k8s sa cert - command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/sa.crt + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/sa.pub - name: Create k8s front-proxy-ca cert command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/front-proxy-ca.crt From 2ae291f49ff40ffa7e802e88539bfbc9c9d981b8 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 8 Oct 2021 11:18:29 +0200 Subject: [PATCH 15/20] Add HA support --- defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index fe502db0..a8fec07a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -85,8 +85,8 @@ kube_install_docker_pip: false # Endpoint for the control plane in case of HA mode with multiple master kube_control_plane_ip: "" kube_control_plane_port: 8443 -kube_control_plane_peer_ip: "" -kube_control_plane_peer_iface: "" +kube_control_plane_peer_ip: "{{ ansible_default_ipv4.address }}" +kube_control_plane_peer_iface: "{{ ansible_default_ipv4.interface }}" kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] # ETCD Peer adress kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}" From ccd0daaba65e1df799f5cb1a22cd444e51b1ef23 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 8 Oct 2021 12:05:50 +0200 Subject: [PATCH 16/20] Add HA support --- tasks/etcd.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml index d5404733..9692c6f7 100644 --- a/tasks/etcd.yaml +++ b/tasks/etcd.yaml @@ -22,12 +22,21 @@ - block: - - name: Wait for etcd CA cert + - name: Wait for certs wait_for: - path: /etc/kubernetes/pki/etcd/ca.key + path: "{{item}}" delegate_to: "{{kube_server}}" + with_items: + - "/etc/kubernetes/pki/etcd/ca.key" + - "/etc/kubernetes/pki/etcd/ca.crt" + - "/etc/kubernetes/pki/ca.crt" + - "/etc/kubernetes/pki/ca.key" + - "/etc/kubernetes/pki/sa.pub" + - "/etc/kubernetes/pki/sa.key" + - "/etc/kubernetes/pki/front-proxy-ca.crt" + - "/etc/kubernetes/pki/front-proxy-ca.key" - - name: Copy CA files from master + - name: Copy cert files from master synchronize: src: "{{item}}" dest: "{{item}}" From b2ecb59aad7ed3ef131e8d753fb1302bbe76ba41 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 11 Oct 2021 09:23:27 +0200 Subject: [PATCH 17/20] Add HA support --- tasks/control_plane.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tasks/control_plane.yaml b/tasks/control_plane.yaml index f52ba7d1..96bd25be 100644 --- a/tasks/control_plane.yaml +++ b/tasks/control_plane.yaml @@ -12,4 +12,13 @@ delegate_to: "{{kube_server}}" - name: Add node to kube cluster control_plane - command: kubeadm join --control-plane --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf \ No newline at end of file + command: kubeadm join --control-plane --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf + +- name: Add Kube API server options + lineinfile: + dest: /etc/kubernetes/manifests/kube-apiserver.yaml + line: ' - {{item.option}}={{item.value}}' + regexp: '^ - {{item.option}}=' + insertafter: ' - kube-apiserver' + notify: restart kubeapi + with_items: "{{ kube_apiserver_options }}" \ No newline at end of file From 582f0b050d5203aee10f26634eb78296b9e5628c Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 11 Oct 2021 12:42:48 +0200 Subject: [PATCH 18/20] Add HA support --- tasks/wn.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks/wn.yaml b/tasks/wn.yaml index a4dfa1ab..a98b6159 100644 --- a/tasks/wn.yaml +++ b/tasks/wn.yaml @@ -5,10 +5,10 @@ search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf" delegate_to: "{{kube_server}}" -- name: Add WN to kube cluster +- name: Add WN to kube cluster to master command: kubeadm join --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf - when: kube_type_of_node == "wn" + when: kube_control_plane_ip == "" -- name: Add node to kube cluster control_plane - command: kubeadm join --control-plane --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf - when: kube_type_of_node == "control_plane" \ No newline at end of file +- name: Add WN to kube cluster control plane + command: kubeadm join --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf + when: kube_control_plane_ip != "" From ef56c56e0c804e72f0f63d69370ae6773d1346ef Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 11 Oct 2021 12:44:01 +0200 Subject: [PATCH 19/20] Add HA support --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 8b7b4766..adba3c30 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,16 @@ The variables that can be passed to this role and a brief description about them kube_docker_compatible_versions: ['17.03.', '18.06.', '18.09.', '19.03.'] # Install docker with pip kube_install_docker_pip + # Endpoint for the control plane in case of HA mode with multiple master + kube_control_plane_ip: "" + kube_control_plane_port: 8443 + kube_control_plane_peer_ip: "{{ ansible_default_ipv4.address }}" + kube_control_plane_peer_iface: "{{ ansible_default_ipv4.interface }}" + kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] + # ETCD Peer adress + kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}" + kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"} + kube_etcd_peer_name: "kubeserver.localdomain" Example Playbook ---------------- From b9bffcf2fe24e26e2c3dd2d86306429c468213a7 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 15 Oct 2021 09:32:46 +0200 Subject: [PATCH 20/20] Update kube-vip image --- files/kube-vip.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/kube-vip.yaml b/files/kube-vip.yaml index ff3b4bcc..9710d249 100644 --- a/files/kube-vip.yaml +++ b/files/kube-vip.yaml @@ -11,7 +11,7 @@ spec: - start - -c - /vip.yaml - image: 'plndr/kube-vip:0.1.1' + image: 'ghcr.io/kube-vip/kube-vip:v0.3.8' name: kube-vip resources: {} securityContext: