Own the k3s config.yaml in the kubernetes role template, write it only on a semantic change and run the installer only when the pinned k3s version differs

kubernetes/defaults/main.yml

@@ -32,6 +32,15 @@ kubernetes:
   # One of [traefik-ingress]
   ingress_controller: traefik-ingress
 
+  # k3s version to install, e.g. v1.37.1+k3s1. An installed k3s is replaced only
+  # when this is set and differs; empty installs the latest when k3s is missing.
+  version: ""
+
+  # Extra config of the nodes running etcd, left out with disable-etcd
+  etcd_config_extra:
+    # Without it etcd serves metrics on 127.0.0.1 only, out of reach of vmagent
+    etcd-expose-metrics: true
+
   config_extra:
     # etcd-tuning
     #  heartbeat: 0.5-1.5x of rtt

kubernetes/tasks/install.yml

@@ -4,11 +4,30 @@
     state: directory
     mode: '0755'
 
+- name: Read the live k3s config
+  slurp:
+    src: /etc/rancher/k3s/config.yaml
+  register: live_config
+  failed_when: false
+
+# A rewrite restarts k3s, so the file is written only when k3s would read other
+# settings: the live files differ from the rendering in comments, order and
+# whitespace.
 - name: Deploy k3s config
   ansible.builtin.template:
     src: k3s/{{ type }}/config.yaml.jinja2
     dest: /etc/rancher/k3s/config.yaml
   register: config
+  when: >-
+    live_config.content is not defined or
+    (live_config.content | b64decode | from_yaml) != (lookup('template', 'k3s/' ~ type ~ '/config.yaml.jinja2') | from_yaml)
+
+- name: Read the installed k3s version
+  command: k3s --version
+  register: k3s_installed
+  changed_when: false
+  failed_when: false
+  check_mode: false
 
 - name: Download install-script
   get_url:

kubernetes/tasks/install/agent/install_k3s.yml

@@ -1,5 +1,8 @@
 - name: Install K3s agent
   command: /root/k3s_install.sh {{ type }}
+  environment:
+    INSTALL_K3S_VERSION: "{{ kubernetes.version }}"
+  when: k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout)
   register: command
   changed_when: "'No change detected' not in command.stdout"
   until: "command is not failed"

kubernetes/tasks/install/server/install_k3s.yml

@@ -1,6 +1,10 @@
 - name: Install K3s-server for 1st-node
   command: /root/k3s_install.sh {{ type }}
-  when: "inventory_hostname == groups['kubernetes'][0]"
+  environment:
+    INSTALL_K3S_VERSION: "{{ kubernetes.version }}"
+  when:
+    - "inventory_hostname == groups['kubernetes'][0]"
+    - k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout)
   register: command
   changed_when: "'No change detected' not in command.stdout"
 
@@ -19,7 +23,11 @@
 
 - name: Install K3s-server for other nodes
   command: /root/k3s_install.sh {{ type }}
-  when: "inventory_hostname != groups['kubernetes'][0]"
+  environment:
+    INSTALL_K3S_VERSION: "{{ kubernetes.version }}"
+  when:
+    - "inventory_hostname != groups['kubernetes'][0]"
+    - k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout)
   register: command
   changed_when: "'No change detected' not in command.stdout"
   until: "command is not failed"

kubernetes/tasks/node.yml

@@ -1,7 +1,3 @@
-# config.yaml is edited in place, not templated: the live files carry hand
-# settings the template does not know. k3s reads them on its next restart,
-# which stays manual so etcd members restart one at a time.
-
 - name: Kubelet image garbage collection
   tags: kubelet_gc
   block:
@@ -22,19 +18,6 @@
           imageGCLowThresholdPercent: 55
           imageMaximumGCAge: 168h
 
-    - name: Declare the kubelet-arg list in the k3s config
-      lineinfile:
-        path: /etc/rancher/k3s/config.yaml
-        regexp: '^kubelet-arg:'
-        line: 'kubelet-arg:'
-
-    - name: Point the kubelet at the drop-in directory
-      lineinfile:
-        path: /etc/rancher/k3s/config.yaml
-        regexp: '^\s*- config-dir='
-        line: '  - config-dir=/etc/rancher/k3s/kubelet.conf.d'
-        insertafter: '^kubelet-arg:'
-
 # Every pod watching its config files takes inotify instances and watches from
 # the per-user pool. An LXC container inherits these from its host. Only this
 # file is loaded: sysctl --system would also apply distribution defaults such as
@@ -47,12 +30,3 @@
     value: "{{ item.value }}"
     sysctl_file: /etc/sysctl.d/90-inotify.conf
   loop: "{{ {'fs.inotify.max_user_instances': 65536, 'fs.inotify.max_user_watches': 4194304} | dict2items }}"
-
-# Without it etcd serves metrics on 127.0.0.1 only, out of reach of vmagent.
-- name: Expose the etcd metrics on the node IP
-  tags: etcd_metrics
-  when: not (kubernetes.config_extra['disable-etcd'] | default(false))
-  lineinfile:
-    path: /etc/rancher/k3s/config.yaml
-    regexp: '^etcd-expose-metrics:'
-    line: 'etcd-expose-metrics: true'

kubernetes/tasks/prerequisites.yml

@@ -25,7 +25,7 @@
           # For Longhorn:
           - nfs-common
           - open-iscsi
-      state: latest
+      state: present
 
 -  import_tasks: ./prerequisites/containerd.yml
 

kubernetes/templates/k3s/server/config.yaml.jinja2

@@ -21,6 +21,11 @@ cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }}
 service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }}
 {% endif %}
 
+# Pin the apiserver advertise-address to the mesh IP: node-external-ip above
+# otherwise gets picked, leaking a LAN-only address into the default/kubernetes
+# Endpoints object that other nodes cannot route to.
+advertise-address: {{ nodeip_ipv4 }}
+
 egress-selector-mode: disabled
 
 # Network-plugin
@@ -47,3 +52,12 @@ node-label:
 {% endif %}
 
 {{ kubernetes.config_extra | to_yaml }}
+
+# Image garbage collection tuning (drop-in KubeletConfiguration)
+kubelet-arg:
+  - config-dir=/etc/rancher/k3s/kubelet.conf.d
+{% if not kubernetes.config_extra['disable-etcd'] | default(false) %}
+
+# etcd members only
+{{ kubernetes.etcd_config_extra | to_nice_yaml(indent=2) }}
+{% endif %}