From 23ed82b113bad838c7838ac82a105294efc81ca8 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 9 Oct 2026 14:13:21 +0000 Subject: [PATCH] Own the k3s config.yaml in the kubernetes role template, write it only on a semantic change and run the installer only when the pinned k3s version differs --- kubernetes/defaults/main.yml | 9 +++++++ kubernetes/tasks/install.yml | 19 ++++++++++++++ .../tasks/install/agent/install_k3s.yml | 3 +++ .../tasks/install/server/install_k3s.yml | 12 +++++++-- kubernetes/tasks/node.yml | 26 ------------------- kubernetes/tasks/prerequisites.yml | 2 +- .../templates/k3s/server/config.yaml.jinja2 | 14 ++++++++++ 7 files changed, 56 insertions(+), 29 deletions(-) diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml index fafd96d..5c38dc0 100644 --- a/kubernetes/defaults/main.yml +++ b/kubernetes/defaults/main.yml @@ -32,6 +32,15 @@ kubernetes: # One of [traefik-ingress] ingress_controller: traefik-ingress + # k3s version to install, e.g. v1.37.1+k3s1. An installed k3s is replaced only + # when this is set and differs; empty installs the latest when k3s is missing. + version: "" + + # Extra config of the nodes running etcd, left out with disable-etcd + etcd_config_extra: + # Without it etcd serves metrics on 127.0.0.1 only, out of reach of vmagent + etcd-expose-metrics: true + config_extra: # etcd-tuning # heartbeat: 0.5-1.5x of rtt diff --git a/kubernetes/tasks/install.yml b/kubernetes/tasks/install.yml index e1837d1..feccac7 100644 --- a/kubernetes/tasks/install.yml +++ b/kubernetes/tasks/install.yml @@ -4,11 +4,30 @@ state: directory mode: '0755' +- name: Read the live k3s config + slurp: + src: /etc/rancher/k3s/config.yaml + register: live_config + failed_when: false + +# A rewrite restarts k3s, so the file is written only when k3s would read other +# settings: the live files differ from the rendering in comments, order and +# whitespace. - name: Deploy k3s config ansible.builtin.template: src: k3s/{{ type }}/config.yaml.jinja2 dest: /etc/rancher/k3s/config.yaml register: config + when: >- + live_config.content is not defined or + (live_config.content | b64decode | from_yaml) != (lookup('template', 'k3s/' ~ type ~ '/config.yaml.jinja2') | from_yaml) + +- name: Read the installed k3s version + command: k3s --version + register: k3s_installed + changed_when: false + failed_when: false + check_mode: false - name: Download install-script get_url: diff --git a/kubernetes/tasks/install/agent/install_k3s.yml b/kubernetes/tasks/install/agent/install_k3s.yml index 2fd0ec6..6241efe 100644 --- a/kubernetes/tasks/install/agent/install_k3s.yml +++ b/kubernetes/tasks/install/agent/install_k3s.yml @@ -1,5 +1,8 @@ - name: Install K3s agent command: /root/k3s_install.sh {{ type }} + environment: + INSTALL_K3S_VERSION: "{{ kubernetes.version }}" + when: k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout) register: command changed_when: "'No change detected' not in command.stdout" until: "command is not failed" diff --git a/kubernetes/tasks/install/server/install_k3s.yml b/kubernetes/tasks/install/server/install_k3s.yml index 5454e48..d323b9f 100644 --- a/kubernetes/tasks/install/server/install_k3s.yml +++ b/kubernetes/tasks/install/server/install_k3s.yml @@ -1,6 +1,10 @@ - name: Install K3s-server for 1st-node command: /root/k3s_install.sh {{ type }} - when: "inventory_hostname == groups['kubernetes'][0]" + environment: + INSTALL_K3S_VERSION: "{{ kubernetes.version }}" + when: + - "inventory_hostname == groups['kubernetes'][0]" + - k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout) register: command changed_when: "'No change detected' not in command.stdout" @@ -19,7 +23,11 @@ - name: Install K3s-server for other nodes command: /root/k3s_install.sh {{ type }} - when: "inventory_hostname != groups['kubernetes'][0]" + environment: + INSTALL_K3S_VERSION: "{{ kubernetes.version }}" + when: + - "inventory_hostname != groups['kubernetes'][0]" + - k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout) register: command changed_when: "'No change detected' not in command.stdout" until: "command is not failed" diff --git a/kubernetes/tasks/node.yml b/kubernetes/tasks/node.yml index 59e3024..40ab3c7 100644 --- a/kubernetes/tasks/node.yml +++ b/kubernetes/tasks/node.yml @@ -1,7 +1,3 @@ -# config.yaml is edited in place, not templated: the live files carry hand -# settings the template does not know. k3s reads them on its next restart, -# which stays manual so etcd members restart one at a time. - - name: Kubelet image garbage collection tags: kubelet_gc block: @@ -22,19 +18,6 @@ imageGCLowThresholdPercent: 55 imageMaximumGCAge: 168h - - name: Declare the kubelet-arg list in the k3s config - lineinfile: - path: /etc/rancher/k3s/config.yaml - regexp: '^kubelet-arg:' - line: 'kubelet-arg:' - - - name: Point the kubelet at the drop-in directory - lineinfile: - path: /etc/rancher/k3s/config.yaml - regexp: '^\s*- config-dir=' - line: ' - config-dir=/etc/rancher/k3s/kubelet.conf.d' - insertafter: '^kubelet-arg:' - # Every pod watching its config files takes inotify instances and watches from # the per-user pool. An LXC container inherits these from its host. Only this # file is loaded: sysctl --system would also apply distribution defaults such as @@ -47,12 +30,3 @@ value: "{{ item.value }}" sysctl_file: /etc/sysctl.d/90-inotify.conf loop: "{{ {'fs.inotify.max_user_instances': 65536, 'fs.inotify.max_user_watches': 4194304} | dict2items }}" - -# Without it etcd serves metrics on 127.0.0.1 only, out of reach of vmagent. -- name: Expose the etcd metrics on the node IP - tags: etcd_metrics - when: not (kubernetes.config_extra['disable-etcd'] | default(false)) - lineinfile: - path: /etc/rancher/k3s/config.yaml - regexp: '^etcd-expose-metrics:' - line: 'etcd-expose-metrics: true' diff --git a/kubernetes/tasks/prerequisites.yml b/kubernetes/tasks/prerequisites.yml index 98060e7..32e8e7d 100644 --- a/kubernetes/tasks/prerequisites.yml +++ b/kubernetes/tasks/prerequisites.yml @@ -25,7 +25,7 @@ # For Longhorn: - nfs-common - open-iscsi - state: latest + state: present - import_tasks: ./prerequisites/containerd.yml diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 index d9168b2..8f791c7 100644 --- a/kubernetes/templates/k3s/server/config.yaml.jinja2 +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -21,6 +21,11 @@ cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }} service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }} {% endif %} +# Pin the apiserver advertise-address to the mesh IP: node-external-ip above +# otherwise gets picked, leaking a LAN-only address into the default/kubernetes +# Endpoints object that other nodes cannot route to. +advertise-address: {{ nodeip_ipv4 }} + egress-selector-mode: disabled # Network-plugin @@ -47,3 +52,12 @@ node-label: {% endif %} {{ kubernetes.config_extra | to_yaml }} + +# Image garbage collection tuning (drop-in KubeletConfiguration) +kubelet-arg: + - config-dir=/etc/rancher/k3s/kubelet.conf.d +{% if not kubernetes.config_extra['disable-etcd'] | default(false) %} + +# etcd members only +{{ kubernetes.etcd_config_extra | to_nice_yaml(indent=2) }} +{% endif %}