From e5920b3ddf145c55778de3704739f1ae51864050 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 17 Mar 2023 15:57:48 +0100 Subject: [PATCH 01/20] Add network-plugin option --- kubernetes/defaults/main.yml | 4 ++ .../tasks/install/server/install_k3s.yml | 4 ++ .../server/network-plugin/deploy_calico.yml | 20 +++++++++ .../network-plugin/deploy_calico_helper.yml | 6 +++ .../templates/k3s/server/config.yaml.jinja2 | 5 +++ .../calico/custom-ressource.yml.jinja2 | 32 ++++++++++++++ .../calico/routingtabletowg.yml.jinja2 | 43 +++++++++++++++++++ 7 files changed, 114 insertions(+) create mode 100644 kubernetes/tasks/install/server/network-plugin/deploy_calico.yml create mode 100644 kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml create mode 100644 kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 create mode 100644 kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml index 57e82a4..378fb96 100644 --- a/kubernetes/defaults/main.yml +++ b/kubernetes/defaults/main.yml @@ -15,3 +15,7 @@ kubernetes: dns_name: token: + + # One of [flannel, calico] + network_plugin: calico + diff --git a/kubernetes/tasks/install/server/install_k3s.yml b/kubernetes/tasks/install/server/install_k3s.yml index e060725..262fb93 100644 --- a/kubernetes/tasks/install/server/install_k3s.yml +++ b/kubernetes/tasks/install/server/install_k3s.yml @@ -34,3 +34,7 @@ # block: | # export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" # create: true + +- name: Deploy calico + import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml + when: "kubernetes.network_plugin == 'calico'" diff --git a/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml b/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml new file mode 100644 index 0000000..3491142 --- /dev/null +++ b/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml @@ -0,0 +1,20 @@ +- name: Deploy calico operator + command: kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.24.3/manifests/tigera-operator.yaml + register: command + changed_when: "'created' in command.stdout" + run_once: true + +- name: Deploy calico ressource template + ansible.builtin.template: + src: ./k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 + dest: /root/calico-ressource.yml + run_once: true + +- name: Deploy calico ressource + command: kubectl apply -f /root/calico-ressource.yml + register: command + changed_when: "'created' in command.stdout" + run_once: true + +- name: Deploy calico-helpers + import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml diff --git a/kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml b/kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml new file mode 100644 index 0000000..ec8a684 --- /dev/null +++ b/kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml @@ -0,0 +1,6 @@ +- name: Deploy service-file for routing-table to wireguard-translation + ansible.builtin.template: + src: ./k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 + dest: /var/lib/rancher/k3s/server/manifests/routingtabletowg.yml + mode: u=rw,g=r,o=r + run_once: true diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index 134ac05..a6c05a6 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -17,7 +17,12 @@ service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }},{{ kubernetes.ipPool.ipv egress-selector-mode: disabled # Network-plugin +{% if kubernetes.network_plugin == "flannel" %} flannel-backend: vxlan +{% else %} +disable-network-policy: true +flannel-backend: none +{% endif %} # FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses kubelet-arg: "--node-ip=0.0.0.0" diff --git a/kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 b/kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 new file mode 100644 index 0000000..fbe7473 --- /dev/null +++ b/kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 @@ -0,0 +1,32 @@ +# This section includes base Calico installation configuration. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + # Note: The ipPools section cannot be modified post-install. + ipPools: + - blockSize: 26 + cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }} + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() + + - blockSize: 122 + cidr: {{ kubernetes.ipPool.ipv6.cluster_cidr }} + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() + +--- + +# This section configures the Calico API server. +# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer +apiVersion: operator.tigera.io/v1 +kind: APIServer +metadata: + name: default +spec: {} diff --git a/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 b/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 new file mode 100644 index 0000000..93bc420 --- /dev/null +++ b/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 @@ -0,0 +1,43 @@ +# https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: routingtabletowg + namespace: calico-system + labels: + app: routingtabletowg +spec: + selector: + matchLabels: + app: routingtabletowg + template: + metadata: + labels: + app: routingtabletowg + spec: + tolerations: + # this toleration is to have the daemonset runnable on master nodes + # remove it if your masters can't run pods + - key: node-role.kubernetes.io/master + effect: NoSchedule + hostNetwork: true + containers: + - name: routingtabletowg + image: "ruakij/routingtabletowg:0.1.2" + env: + - name: INTERFACE + value: {{ kubernetes.ipPool.nodeIp_interface }} + - name: FILTER_PROTOCOL + value: bird + securityContext: + capabilities: + add: + - NET_ADMIN + resources: + requests: + cpu: 10m + memory: 10Mi + limits: + cpu: 20m + memory: 20Mi +--- From f9a859e95c380cc4434d61a2e40e3f6fcba8e989 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 13:56:26 +0100 Subject: [PATCH 02/20] Add ingress-option --- kubernetes/defaults/main.yml | 2 ++ kubernetes/templates/k3s/server/config.yaml.jinja2 | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml index 378fb96..414e0ea 100644 --- a/kubernetes/defaults/main.yml +++ b/kubernetes/defaults/main.yml @@ -19,3 +19,5 @@ kubernetes: # One of [flannel, calico] network_plugin: calico + # One of [traefik-ingress] + ingress_controller: traefik-ingress diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index a6c05a6..3dd95aa 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -24,5 +24,7 @@ disable-network-policy: true flannel-backend: none {% endif %} -# FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses -kubelet-arg: "--node-ip=0.0.0.0" +# Ingress-plugin +{% if kubernetes.ingress_controller != "traefik-ingress" %} +disable: traefik +{% endif %} From 4321d78cf85c5f52f7e5407e1424098bf05a22e5 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 13:57:57 +0100 Subject: [PATCH 03/20] Add comments to variables --- kubernetes/defaults/main.yml | 4 +++- kubernetes/templates/k3s/server/config.yaml.jinja2 | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml index 414e0ea..7966b04 100644 --- a/kubernetes/defaults/main.yml +++ b/kubernetes/defaults/main.yml @@ -2,13 +2,15 @@ kubernetes: ipPool: ipv4: + # Minimum: /24 cluster_cidr: 10.42.0.0/16 service_cidr: 10.43.0.0/16 ipv6: + # Minimum: /120 cluster_cidr: fd42::/56 service_cidr: fd43::/112 - # Replace - with _ + # Interface to grab node-IPv4/v6 from nodeIp_interface: control_plane: diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index 3dd95aa..f348279 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -1,5 +1,6 @@ ## Base ## {% if inventory_hostname == groups['kubernetes'][0] %} +# Initialize with internal etcd cluster-init: true {% else %} server: https://{{ groups['kubernetes'][0] }}:6443 From 5452303992b370140562fcf3cd0cc73f193a47c6 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 13:58:05 +0100 Subject: [PATCH 04/20] Remove netmaker from dependency --- kubernetes/meta/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/kubernetes/meta/main.yml b/kubernetes/meta/main.yml index 613d878..3b2d8d8 100644 --- a/kubernetes/meta/main.yml +++ b/kubernetes/meta/main.yml @@ -1,4 +1,3 @@ --- dependencies: - role: docker - - role: netmaker From fb44c399695af19620f10f2713046c9cc955a028 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 13:58:47 +0100 Subject: [PATCH 05/20] Add install of often-used packets --- kubernetes/tasks/prerequisites.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/kubernetes/tasks/prerequisites.yml b/kubernetes/tasks/prerequisites.yml index 5ac2bae..2565f85 100644 --- a/kubernetes/tasks/prerequisites.yml +++ b/kubernetes/tasks/prerequisites.yml @@ -17,12 +17,15 @@ #- name: Disable swap # command: swapoff -a -#- name: Install iptables -# package: -# name: -# #- containerd -# - iptables -# state: latest +- name: Install required packages + package: + name: + #- containerd + #- iptables + # For Longhorn: + - nfs-common + - open-iscsi + state: latest - import_tasks: ./prerequisites/containerd.yml From 2fee9a174751d065f11ae39fe2def37d2a76eb1a Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 14:00:15 +0100 Subject: [PATCH 06/20] Only enable ipv6 when available and activated --- kubernetes/templates/k3s/agent/config.yaml.jinja2 | 6 +++++- kubernetes/templates/k3s/server/config.yaml.jinja2 | 11 ++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/kubernetes/templates/k3s/agent/config.yaml.jinja2 b/kubernetes/templates/k3s/agent/config.yaml.jinja2 index 417f719..0507054 100644 --- a/kubernetes/templates/k3s/agent/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/agent/config.yaml.jinja2 @@ -1,7 +1,11 @@ server: https://{{ kubernetes.control_plane.dns_name }}:6443 token: {{ kubernetes.token }} -node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }} +{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %} +node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }} # FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses kubelet-arg: "--node-ip=0.0.0.0" +{% else %} +node-ip: {{ nodeip_ipv4 }} +{% endif %} diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index f348279..e57e4a4 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -11,9 +11,18 @@ tls-san: - {{ kubernetes.control_plane.dns_name }} # Networking +{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %} node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }} cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }},{{ kubernetes.ipPool.ipv6.cluster_cidr }} -service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }},{{ kubernetes.ipPool.ipv6.service_cidr }} +service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }},{{ kubernetes.ipPool.ipv6.service_cidr }} + +# FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses +kubelet-arg: "--node-ip=0.0.0.0" +{% else %} +node-ip: {{ nodeip_ipv4 }} +cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }} +service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }} +{% endif %} egress-selector-mode: disabled From 0c82504299514cc18dd84cf92b5dfb6893a58d9b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 14:00:46 +0100 Subject: [PATCH 07/20] Separate getting name and ips to fix bug easily --- kubernetes/tasks/prerequisites.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kubernetes/tasks/prerequisites.yml b/kubernetes/tasks/prerequisites.yml index 2565f85..98060e7 100644 --- a/kubernetes/tasks/prerequisites.yml +++ b/kubernetes/tasks/prerequisites.yml @@ -29,10 +29,14 @@ - import_tasks: ./prerequisites/containerd.yml +- name: Gather interface-name + set_fact: + interface: "{{ kubernetes.ipPool.nodeIp_interface | replace('-', '_') }}" + - name: Getting nodeIp-data from interface set_fact: - nodeip_ipv4: "{{ ansible_facts[ kubernetes.ipPool.nodeIp_interface ].ipv4.address }}" - nodeip_ipv6: "{{ ansible_facts[ kubernetes.ipPool.nodeIp_interface ].ipv6[0].address }}" + nodeip_ipv4: "{{ ansible_facts[ interface ].ipv4.address }}" + nodeip_ipv6: "{{ ansible_facts[ interface ].ipv6[0].address if ansible_facts[ interface ].ipv6 is defined }}" - name: Run handlers to reload configurations meta: flush_handlers From 590b75ac231e0105c3779726bf0e30e6ab9b9723 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 14:01:19 +0100 Subject: [PATCH 08/20] Add quotes to token-usage for special chars --- kubernetes/templates/k3s/agent/config.yaml.jinja2 | 2 +- kubernetes/templates/k3s/server/config.yaml.jinja2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/templates/k3s/agent/config.yaml.jinja2 b/kubernetes/templates/k3s/agent/config.yaml.jinja2 index 0507054..3900243 100644 --- a/kubernetes/templates/k3s/agent/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/agent/config.yaml.jinja2 @@ -1,5 +1,5 @@ server: https://{{ kubernetes.control_plane.dns_name }}:6443 -token: {{ kubernetes.token }} +token: '{{ kubernetes.token }}' {% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %} node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }} diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index e57e4a4..4ea7d95 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -6,7 +6,7 @@ cluster-init: true server: https://{{ groups['kubernetes'][0] }}:6443 {% endif %} -token: {{ kubernetes.token }} +token: '{{ kubernetes.token }}' tls-san: - {{ kubernetes.control_plane.dns_name }} From fc3d9845d6ac7cda7df7f9b4a6a4dfc331734bc9 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 15:17:02 +0100 Subject: [PATCH 09/20] Fix undeterministic node-selection but uses group --- kubernetes/templates/k3s/server/config.yaml.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index 4ea7d95..8cea4fc 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -3,7 +3,7 @@ # Initialize with internal etcd cluster-init: true {% else %} -server: https://{{ groups['kubernetes'][0] }}:6443 +server: https://{{ hostvars[groups['kubernetes'][0]]['nodeip_ipv4'] }}:6443 {% endif %} token: '{{ kubernetes.token }}' From 791ad96849cc0da9b9a67752ae56cd7816086881 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 19 Mar 2023 15:17:45 +0100 Subject: [PATCH 10/20] Add ipv6-check to calico deploy --- .../server/network-plugin/calico/custom-ressource.yml.jinja2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 b/kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 index fbe7473..fea2301 100644 --- a/kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 +++ b/kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2 @@ -15,11 +15,13 @@ spec: natOutgoing: Enabled nodeSelector: all() +{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %} - blockSize: 122 cidr: {{ kubernetes.ipPool.ipv6.cluster_cidr }} encapsulation: None natOutgoing: Enabled nodeSelector: all() +{% endif %} --- From b46d35c8a54445bc2b6c8d74411e7887b7aecd8e Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 30 Mar 2023 14:54:28 +0200 Subject: [PATCH 11/20] Add labels --- kubernetes/templates/k3s/agent/config.yaml.jinja2 | 10 ++++++++++ kubernetes/templates/k3s/server/config.yaml.jinja2 | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/kubernetes/templates/k3s/agent/config.yaml.jinja2 b/kubernetes/templates/k3s/agent/config.yaml.jinja2 index 3900243..f42ec8e 100644 --- a/kubernetes/templates/k3s/agent/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/agent/config.yaml.jinja2 @@ -9,3 +9,13 @@ kubelet-arg: "--node-ip=0.0.0.0" {% else %} node-ip: {{ nodeip_ipv4 }} {% endif %} + +## Label +# Region & DC +node-label: +{% if region is defined %} + - topology.kubernetes.io/region={{ region }} +{% endif %} +{% if zone is defined %} + - topology.kubernetes.io/zone={{ zone }} +{% endif %} diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index 8cea4fc..ba68c97 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -38,3 +38,14 @@ flannel-backend: none {% if kubernetes.ingress_controller != "traefik-ingress" %} disable: traefik {% endif %} + +## Label +# Region & DC +node-label: +{% if region is defined %} + - topology.kubernetes.io/region={{ region }} +{% endif %} +{% if zone is defined %} + - topology.kubernetes.io/zone={{ zone }} +{% endif %} + From 8c4e3c2401daffee839365e1a57a7dc00995bc0b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 30 Mar 2023 14:54:49 +0200 Subject: [PATCH 12/20] Update routingtabletowg and use new sync feature --- .../server/network-plugin/calico/routingtabletowg.yml.jinja2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 b/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 index 93bc420..ae54834 100644 --- a/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 +++ b/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 @@ -23,12 +23,14 @@ spec: hostNetwork: true containers: - name: routingtabletowg - image: "ruakij/routingtabletowg:0.1.2" + image: "ruakij/routingtabletowg:0.2.0" env: - name: INTERFACE value: {{ kubernetes.ipPool.nodeIp_interface }} - name: FILTER_PROTOCOL value: bird + - name: PERIODIC_SYNC + value: 300 securityContext: capabilities: add: From 7c86a5d77d233851d8316cbce95e59cda7df0abd Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 13:30:22 +0200 Subject: [PATCH 13/20] Add register for config --- kubernetes/tasks/install.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/tasks/install.yml b/kubernetes/tasks/install.yml index d06ac15..e1837d1 100644 --- a/kubernetes/tasks/install.yml +++ b/kubernetes/tasks/install.yml @@ -8,6 +8,7 @@ ansible.builtin.template: src: k3s/{{ type }}/config.yaml.jinja2 dest: /etc/rancher/k3s/config.yaml + register: config - name: Download install-script get_url: From d6f8f975bb868b09ef9d8dc4dd75b36abdea1fc7 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 13:32:14 +0200 Subject: [PATCH 14/20] Reload when config changed, but install already done --- kubernetes/tasks/install/agent/install_k3s.yml | 7 ++++++- kubernetes/tasks/install/server/install_k3s.yml | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/kubernetes/tasks/install/agent/install_k3s.yml b/kubernetes/tasks/install/agent/install_k3s.yml index b465095..10f7ce3 100644 --- a/kubernetes/tasks/install/agent/install_k3s.yml +++ b/kubernetes/tasks/install/agent/install_k3s.yml @@ -1,7 +1,12 @@ - name: Install K3s agent - command: /root/k3s_install.sh {{ type }} + command: /root/k3s_install.sh {{ type }} register: command changed_when: "'No change detected' in command.stdout" until: "command is not failed" retries: 2 delay: 10 + +- name: Restart when config changed, but install already done + service: k3s + status: restarted + when: "inventory_hostname != groups['kubernetes'][0] && !command.changed && config.changed" diff --git a/kubernetes/tasks/install/server/install_k3s.yml b/kubernetes/tasks/install/server/install_k3s.yml index 262fb93..f5a34e5 100644 --- a/kubernetes/tasks/install/server/install_k3s.yml +++ b/kubernetes/tasks/install/server/install_k3s.yml @@ -4,6 +4,11 @@ register: command changed_when: "'No change detected' in command.stdout" +- name: Restart when config changed, but install already done + service: k3s + status: restarted + when: "inventory_hostname == groups['kubernetes'][0] && !command.changed && config.changed" + - name: Waiting for K3s-server to accept connections ansible.builtin.wait_for: host: "{{ inventory_hostname }}" @@ -20,6 +25,11 @@ retries: 2 delay: 10 +- name: Restart when config changed, but install already done + service: k3s + status: restarted + when: "inventory_hostname != groups['kubernetes'][0] && !command.changed && config.changed" + - name: Waiting for K3s-server to accept connections on other nodes ansible.builtin.wait_for: host: "{{ inventory_hostname }}" From 0d43d07ad4d057bc717afde508090d3381fbaf2f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 14:12:56 +0200 Subject: [PATCH 15/20] Add extra-config option --- kubernetes/defaults/main.yml | 8 ++++++++ kubernetes/templates/k3s/server/config.yaml.jinja2 | 1 + 2 files changed, 9 insertions(+) diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml index 7966b04..c34d963 100644 --- a/kubernetes/defaults/main.yml +++ b/kubernetes/defaults/main.yml @@ -23,3 +23,11 @@ kubernetes: # One of [traefik-ingress] ingress_controller: traefik-ingress + + config_extra: + # etcd-tuning + # heartbeat: 0.5-1.5x of rtt + # election: 10x- of heartbeat + etcd-arg: + heartbeat-interval: 500 + election-timeout: 5000 diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index ba68c97..ab58922 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -49,3 +49,4 @@ node-label: - topology.kubernetes.io/zone={{ zone }} {% endif %} +{{ kubernetes.config_extra | to_yaml }} From dadd077723f490f8e4c48d870c44bf6651bd2117 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 14:13:22 +0200 Subject: [PATCH 16/20] Fix service and conditional --- kubernetes/tasks/install/agent/install_k3s.yml | 7 ++++--- kubernetes/tasks/install/server/install_k3s.yml | 16 +++++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/kubernetes/tasks/install/agent/install_k3s.yml b/kubernetes/tasks/install/agent/install_k3s.yml index 10f7ce3..94a064a 100644 --- a/kubernetes/tasks/install/agent/install_k3s.yml +++ b/kubernetes/tasks/install/agent/install_k3s.yml @@ -7,6 +7,7 @@ delay: 10 - name: Restart when config changed, but install already done - service: k3s - status: restarted - when: "inventory_hostname != groups['kubernetes'][0] && !command.changed && config.changed" + service: + name: k3s + status: restarted + when: "inventory_hostname != groups['kubernetes'][0] and not command.changed and config.changed" diff --git a/kubernetes/tasks/install/server/install_k3s.yml b/kubernetes/tasks/install/server/install_k3s.yml index f5a34e5..d8d4a75 100644 --- a/kubernetes/tasks/install/server/install_k3s.yml +++ b/kubernetes/tasks/install/server/install_k3s.yml @@ -1,13 +1,14 @@ - name: Install K3s-server for 1st-node - command: /root/k3s_install.sh {{ type }} + command: /root/k3s_install.sh {{ type }} when: "inventory_hostname == groups['kubernetes'][0]" register: command changed_when: "'No change detected' in command.stdout" - name: Restart when config changed, but install already done - service: k3s - status: restarted - when: "inventory_hostname == groups['kubernetes'][0] && !command.changed && config.changed" + service: + name: k3s + status: restarted + when: "inventory_hostname == groups['kubernetes'][0] and not command.changed and config.changed" - name: Waiting for K3s-server to accept connections ansible.builtin.wait_for: @@ -26,9 +27,10 @@ delay: 10 - name: Restart when config changed, but install already done - service: k3s - status: restarted - when: "inventory_hostname != groups['kubernetes'][0] && !command.changed && config.changed" + service: + name: k3s + status: restarted + when: "inventory_hostname != groups['kubernetes'][0] and not command.changed and config.changed" - name: Waiting for K3s-server to accept connections on other nodes ansible.builtin.wait_for: From d113625fa83b6ab442be07e387a7a953fa885049 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 20:08:25 +0200 Subject: [PATCH 17/20] Fix env-value not being string --- .../server/network-plugin/calico/routingtabletowg.yml.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 b/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 index ae54834..382bbff 100644 --- a/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 +++ b/kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2 @@ -30,7 +30,7 @@ spec: - name: FILTER_PROTOCOL value: bird - name: PERIODIC_SYNC - value: 300 + value: '300' securityContext: capabilities: add: From a3c887748aefc3054869c426415c4827e1cec0c1 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 20:14:12 +0200 Subject: [PATCH 18/20] Move network-helper to own file independend from calico --- kubernetes/tasks/install/server/install_k3s.yml | 4 +++- .../tasks/install/server/network-plugin/deploy_calico.yml | 3 --- .../{deploy_calico_helper.yml => deploy_network_helper.yml} | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) rename kubernetes/tasks/install/server/network-plugin/{deploy_calico_helper.yml => deploy_network_helper.yml} (84%) diff --git a/kubernetes/tasks/install/server/install_k3s.yml b/kubernetes/tasks/install/server/install_k3s.yml index d8d4a75..b397c51 100644 --- a/kubernetes/tasks/install/server/install_k3s.yml +++ b/kubernetes/tasks/install/server/install_k3s.yml @@ -49,4 +49,6 @@ - name: Deploy calico import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml - when: "kubernetes.network_plugin == 'calico'" + +- name: Deploy network-helpers + import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml diff --git a/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml b/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml index 3491142..da366da 100644 --- a/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml +++ b/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml @@ -15,6 +15,3 @@ register: command changed_when: "'created' in command.stdout" run_once: true - -- name: Deploy calico-helpers - import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml diff --git a/kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml b/kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml similarity index 84% rename from kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml rename to kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml index ec8a684..acc3205 100644 --- a/kubernetes/tasks/install/server/network-plugin/deploy_calico_helper.yml +++ b/kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml @@ -4,3 +4,4 @@ dest: /var/lib/rancher/k3s/server/manifests/routingtabletowg.yml mode: u=rw,g=r,o=r run_once: true + when: "kubernetes.network.helper.routingtabletowg" From 41570ea40d7595efe07f329d06fd1e0886ed6251 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 20:14:36 +0200 Subject: [PATCH 19/20] Create new block for network-stuff --- kubernetes/defaults/main.yml | 12 ++++++++++-- kubernetes/tasks/install/server/install_k3s.yml | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml index c34d963..fafd96d 100644 --- a/kubernetes/defaults/main.yml +++ b/kubernetes/defaults/main.yml @@ -18,8 +18,16 @@ kubernetes: token: - # One of [flannel, calico] - network_plugin: calico + network: + # One of [flannel, calico] + plugin: calico + + # Helper for networking + helper: + # https://github.com/Ruakij/RoutingTableToWg + # Translates received-routes from e.g. BGP to wireguard-allowedips + # Helpful, when nodeIp_interface is a wireguard-interface + routingtabletowg: false # One of [traefik-ingress] ingress_controller: traefik-ingress diff --git a/kubernetes/tasks/install/server/install_k3s.yml b/kubernetes/tasks/install/server/install_k3s.yml index b397c51..b4a3076 100644 --- a/kubernetes/tasks/install/server/install_k3s.yml +++ b/kubernetes/tasks/install/server/install_k3s.yml @@ -49,6 +49,7 @@ - name: Deploy calico import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml + when: "kubernetes.network.plugin == 'calico'" - name: Deploy network-helpers import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml From c8f90f0f8d48df4bea50cb7988b01d1cd1ba7720 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 5 Apr 2023 20:15:31 +0200 Subject: [PATCH 20/20] Update calico --- .../tasks/install/server/network-plugin/deploy_calico.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml b/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml index da366da..7cd2955 100644 --- a/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml +++ b/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml @@ -1,5 +1,5 @@ - name: Deploy calico operator - command: kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.24.3/manifests/tigera-operator.yaml + command: kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/tigera-operator.yaml register: command changed_when: "'created' in command.stdout" run_once: true