diff --git a/README.md b/README.md index 8b7b4766..adba3c30 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,16 @@ The variables that can be passed to this role and a brief description about them kube_docker_compatible_versions: ['17.03.', '18.06.', '18.09.', '19.03.'] # Install docker with pip kube_install_docker_pip + # Endpoint for the control plane in case of HA mode with multiple master + kube_control_plane_ip: "" + kube_control_plane_port: 8443 + kube_control_plane_peer_ip: "{{ ansible_default_ipv4.address }}" + kube_control_plane_peer_iface: "{{ ansible_default_ipv4.interface }}" + kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] + # ETCD Peer adress + kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}" + kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"} + kube_etcd_peer_name: "kubeserver.localdomain" Example Playbook ---------------- diff --git a/defaults/main.yml b/defaults/main.yml index f38fea1d..29d06aee 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,6 @@ # Version to install or latest kube_version: 1.19.14 -# Type of node front or wn +# Type of node front, control_plane or wn kube_type_of_node: front # IP address or name of the Kube front node kube_server: "{{ ansible_default_ipv4.address }}" @@ -82,6 +82,19 @@ docker_nvidia_options: # Install docker with pip kube_install_docker_pip: false + +# Endpoint for the control plane in case of HA mode with multiple master +kube_control_plane_ip: "" +kube_control_plane_port: 8443 +kube_control_plane_peer_ip: "{{ ansible_default_ipv4.address }}" +kube_control_plane_peer_iface: "{{ ansible_default_ipv4.interface }}" +kube_control_plane_remote_peer_list: ["{{ ansible_default_ipv4.address }}"] +# ETCD Peer adress +kube_etcd_peer_address: "{{ ansible_default_ipv4.address }}" +kube_etcd_peer_list: {"kubeserver.localdomain": "{{ ansible_default_ipv4.address }}"} +kube_etcd_peer_name: "kubeserver.localdomain" + +# Kyverno options kube_install_kyverno: false kyverno_crds_helm_chart_version: "2.0.3" kyverno_helm_chart_version: "2.0.3" @@ -98,4 +111,5 @@ kyverno_podSecurityStandard: baseline # Set desired pod security severity low, medium, high. Used severity level in PolicyReportResults for the selected pod security policie kyverno_podSecuritySeverity: high # Set to get response in failed validation check. Supported values are audit and enforce. See: https://kyverno.io/docs/writing-policies/validate/ -kyverno_validationFailureAction: ensure \ No newline at end of file +kyverno_validationFailureAction: ensure + diff --git a/files/kube-vip.yaml b/files/kube-vip.yaml new file mode 100644 index 00000000..9710d249 --- /dev/null +++ b/files/kube-vip.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system +spec: + containers: + - command: + - /kube-vip + - start + - -c + - /vip.yaml + image: 'ghcr.io/kube-vip/kube-vip:v0.3.8' + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_TIME + volumeMounts: + - mountPath: /vip.yaml + name: config + hostNetwork: true + volumes: + - hostPath: + path: /etc/kube-vip/config.yaml + name: config +status: {} \ No newline at end of file diff --git a/tasks/control_plane.yaml b/tasks/control_plane.yaml new file mode 100644 index 00000000..96bd25be --- /dev/null +++ b/tasks/control_plane.yaml @@ -0,0 +1,24 @@ +--- +- name: Configure VIP LB + import_tasks: lb.yaml + +- name: Configure HA ETDC + import_tasks: etcd.yaml + +- name: Wait for Kube master + wait_for: + path: /etc/environment + search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf" + delegate_to: "{{kube_server}}" + +- name: Add node to kube cluster control_plane + command: kubeadm join --control-plane --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=all creates=/etc/kubernetes/admin.conf + +- name: Add Kube API server options + lineinfile: + dest: /etc/kubernetes/manifests/kube-apiserver.yaml + line: ' - {{item.option}}={{item.value}}' + regexp: '^ - {{item.option}}=' + insertafter: ' - kube-apiserver' + notify: restart kubeapi + with_items: "{{ kube_apiserver_options }}" \ No newline at end of file diff --git a/tasks/etcd.yaml b/tasks/etcd.yaml new file mode 100644 index 00000000..9692c6f7 --- /dev/null +++ b/tasks/etcd.yaml @@ -0,0 +1,75 @@ +- name: Create kubeadm-config-etcd file + template: src=kubeadm-config-etcd.j2 dest=/tmp/kubeadm-config-etcd.yaml + +- name: Create /etc/kubernetes/pki/etcd file dir + file: path=/etc/kubernetes/pki/etcd state=directory mode=755 recurse=yes + +- block: + + - name: Create k8s ca cert + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/ca.crt + + - name: Create k8s sa cert + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/sa.pub + + - name: Create k8s front-proxy-ca cert + command: kubeadm init phase certs ca creates=/etc/kubernetes/pki/front-proxy-ca.crt + + - name: Create etcd CA cert + command: kubeadm init phase certs etcd-ca creates=/etc/kubernetes/pki/etcd/ca.crt + + when: kube_type_of_node == "front" + +- block: + + - name: Wait for certs + wait_for: + path: "{{item}}" + delegate_to: "{{kube_server}}" + with_items: + - "/etc/kubernetes/pki/etcd/ca.key" + - "/etc/kubernetes/pki/etcd/ca.crt" + - "/etc/kubernetes/pki/ca.crt" + - "/etc/kubernetes/pki/ca.key" + - "/etc/kubernetes/pki/sa.pub" + - "/etc/kubernetes/pki/sa.key" + - "/etc/kubernetes/pki/front-proxy-ca.crt" + - "/etc/kubernetes/pki/front-proxy-ca.key" + + - name: Copy cert files from master + synchronize: + src: "{{item}}" + dest: "{{item}}" + delegate_to: "{{kube_server}}" + with_items: + - "/etc/kubernetes/pki/etcd/ca.key" + - "/etc/kubernetes/pki/etcd/ca.crt" + - "/etc/kubernetes/pki/ca.crt" + - "/etc/kubernetes/pki/ca.key" + - "/etc/kubernetes/pki/sa.pub" + - "/etc/kubernetes/pki/sa.key" + - "/etc/kubernetes/pki/front-proxy-ca.crt" + - "/etc/kubernetes/pki/front-proxy-ca.key" + + when: kube_type_of_node == "control_plane" + +- name: Create etcd server cert + command: kubeadm init phase certs etcd-server --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/server.crt + +- name: Create etcd peer cert + command: kubeadm init phase certs etcd-peer --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/peer.crt + +- name: Create etcd healthcheck client cert + command: kubeadm init phase certs etcd-healthcheck-client --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/etcd/healthcheck-client.crt + +- name: Create apiserver-etcd-client server cert + command: kubeadm init phase certs apiserver-etcd-client --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/pki/apiserver-etcd-client.crt + +- name: Create etcd manifest + command: kubeadm init phase etcd local --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/manifests/etcd.yaml + +- name: Configure kubelet + command: kubeadm init phase kubeconfig kubelet --config=/tmp/kubeadm-config-etcd.yaml creates=/etc/kubernetes/kubelet.conf + +- name: Configure kubelet + command: kubeadm init phase kubelet-start --config=/tmp/kubeadm-config-etcd.yaml creates=/var/lib/kubelet/config.yaml diff --git a/tasks/front.yaml b/tasks/front.yaml index 7216c0c5..f04c40aa 100644 --- a/tasks/front.yaml +++ b/tasks/front.yaml @@ -5,11 +5,26 @@ - name: force handlers meta: flush_handlers +- set_fact: + init_params: "" + +- block: + - name: Configure VIP LB + import_tasks: lb.yaml + + - name: Configure HA ETDC + import_tasks: etcd.yaml + + - set_fact: + init_params: "--ignore-preflight-errors=all" + + when: kube_control_plane_ip != "" + - name: Create kubeadm-config file template: src=kubeadm-config.j2 dest=/tmp/kubeadm-config.yml - name: Kubeadm init - command: kubeadm init --config /tmp/kubeadm-config.yml creates=/etc/kubernetes/admin.conf + command: kubeadm init --config /tmp/kubeadm-config.yml {{init_params}} creates=/etc/kubernetes/admin.conf - name: Set kube_wait_api_server_ip set_fact: diff --git a/tasks/lb.yaml b/tasks/lb.yaml new file mode 100644 index 00000000..a804cae2 --- /dev/null +++ b/tasks/lb.yaml @@ -0,0 +1,10 @@ +- name: Create /etc/kube-vip file dir + file: path=/etc/kube-vip state=directory mode=755 + +- name: Create kube-vip config file + template: src=kube-vip-config.j2 dest=/etc/kube-vip/config.yaml + +- name: Create /etc/kubernetes/manifests file dir + file: path=/etc/kubernetes/manifests state=directory mode=755 recurse=yes + +- copy: src=kube-vip.yaml dest=/etc/kubernetes/manifests/kube-vip.yaml \ No newline at end of file diff --git a/tasks/main.yaml b/tasks/main.yaml index 0eee5def..6c91a9bb 100644 --- a/tasks/main.yaml +++ b/tasks/main.yaml @@ -50,5 +50,17 @@ - name: Include "{{ansible_os_family}}" Kubernetes recipe include_tasks: "{{ansible_os_family}}.yaml" +- name: Add KUBELET_EXTRA_ARGS + lineinfile: + dest: "{{item}}/kubelet" + line: 'KUBELET_EXTRA_ARGS=--cgroup-driver=systemd {{kubelet_extra_args}}' + regexp: 'KUBELET_EXTRA_ARGS=' + create: yes + notify: restart kubelet + with_first_found: + - files: + - /etc/sysconfig/ + - /etc/default/ + - name: Include "{{kube_type_of_node}}" tasks - include_tasks: "{{kube_type_of_node}}.yaml" + include_tasks: "{{kube_type_of_node}}.yaml" \ No newline at end of file diff --git a/tasks/wn.yaml b/tasks/wn.yaml index d3daa3a3..a98b6159 100644 --- a/tasks/wn.yaml +++ b/tasks/wn.yaml @@ -5,17 +5,10 @@ search_regex: "KUBECONFIG=/etc/kubernetes/admin.conf" delegate_to: "{{kube_server}}" -- name: Add KUBELET_EXTRA_ARGS - lineinfile: - dest: "{{item}}/kubelet" - line: 'KUBELET_EXTRA_ARGS=--cgroup-driver=systemd {{kubelet_extra_args}}' - regexp: 'KUBELET_EXTRA_ARGS=' - create: yes - notify: restart kubelet - with_first_found: - - files: - - /etc/sysconfig/ - - /etc/default/ - -- name: Add node to kube cluster +- name: Add WN to kube cluster to master command: kubeadm join --token {{kube_token}} {{kube_server}}:6443 --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf + when: kube_control_plane_ip == "" + +- name: Add WN to kube cluster control plane + command: kubeadm join --token {{kube_token}} {{kube_control_plane_ip}}:{{kube_control_plane_port}} --discovery-token-unsafe-skip-ca-verification creates=/etc/kubernetes/kubelet.conf + when: kube_control_plane_ip != "" diff --git a/templates/kube-vip-config.j2 b/templates/kube-vip-config.j2 new file mode 100644 index 00000000..7f9e7481 --- /dev/null +++ b/templates/kube-vip-config.j2 @@ -0,0 +1,27 @@ +localPeer: + id: 0 + address: {{kube_control_plane_peer_ip}} + port: 10000 +remotePeers: +{% for peer_ip in kube_control_plane_remote_peer_list %} +- id: {{loop.index + 1}} + address: {{peer_ip}} + port: 10000 +{% endfor %} +vip: {{kube_control_plane_ip}} +gratuitousARP: true +singleNode: false +startAsLeader: true +interface: {{kube_control_plane_peer_iface}} +loadBalancers: +- name: API Server Load Balancer + type: tcp + port: {{kube_control_plane_port}} + bindToVip: false + backends: + - port: 6443 + address: {{kube_control_plane_peer_ip}} +{% for peer_ip in kube_control_plane_remote_peer_list %} + - port: 6443 + address: {{peer_ip}} +{% endfor %} \ No newline at end of file diff --git a/templates/kubeadm-config-etcd.j2 b/templates/kubeadm-config-etcd.j2 new file mode 100644 index 00000000..aaabf4bb --- /dev/null +++ b/templates/kubeadm-config-etcd.j2 @@ -0,0 +1,30 @@ +--- +kind: ClusterConfiguration +apiVersion: kubeadm.k8s.io/v1beta2 +networking: + podSubnet: "{{kube_pod_network_cidr}}" +apiServer: + extraArgs: + advertise-address: "{{kube_api_server}}" +controlPlaneEndpoint: {{kube_control_plane_ip}}:{{kube_control_plane_port}} +etcd: + local: + extraArgs: + listen-client-urls: "https://{{kube_etcd_peer_address}}:2379" + listen-peer-urls: "https://{{kube_etcd_peer_address}}:2380" + advertise-client-urls: "https://{{kube_etcd_peer_address}}:2379" + initial-advertise-peer-urls: "https://{{kube_etcd_peer_address}}:2380" + initial-cluster: "{%- for peer_name, peer_ip in kube_etcd_peer_list.items() -%} + {%- if loop.index > 1 -%} , {%- endif -%} + {{peer_name}}=https://{{peer_ip}}:2380 + {%- endfor -%}" + name: "{{kube_etcd_peer_name}}" + initial-cluster-state: "new" + peerCertSANs: + - "{{kube_etcd_peer_address}}" + serverCertSANs: + - "{{kube_etcd_peer_address}}" +--- +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +cgroupDriver: systemd \ No newline at end of file diff --git a/templates/kubeadm-config.j2 b/templates/kubeadm-config.j2 index 0330ff97..143fecc6 100644 --- a/templates/kubeadm-config.j2 +++ b/templates/kubeadm-config.j2 @@ -6,6 +6,18 @@ networking: apiServer: extraArgs: advertise-address: "{{kube_api_server}}" # --apiserver-advertise-address +{% if kube_control_plane_ip != "" %} +controlPlaneEndpoint: {{kube_control_plane_ip}}:{{kube_control_plane_port}} +etcd: + external: + endpoints: + {% for peer_name, peer_ip in kube_etcd_peer_list.items() %} + - https://{{peer_ip}}:2379 + {% endfor -%} + caFile: /etc/kubernetes/pki/etcd/ca.crt + certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt + keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key +{% endif %} --- kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1