kubernetes: kubelet_reserved drop-in 30-reserved.conf and a k3s restart handler that runs only with k3s_restart_allowed

kubernetes/tasks/node.yml

@@ -18,29 +18,82 @@
           imageGCLowThresholdPercent: 55
           imageMaximumGCAge: 168h
 
-# KubeletConfiguration fields from kubernetes.kubelet_memory; the file is removed
-# while the variable is empty.
+# KubeletConfiguration fields from kubernetes.kubelet_memory (eviction) and
+# kubernetes.kubelet_reserved (systemReserved, kubeReserved); a file is removed
+# while its variable is empty. The kubelet reads the drop-ins only when k3s
+# starts, so a change takes effect after a k3s restart. The restart_k3s handler
+# restarts only with -e k3s_restart_allowed=true, on one node per run.
 - name: Kubelet memory reservation and eviction
-  tags: kubelet_memory
+  tags: [kubelet_memory, kubelet_reserved]
   block:
+    - name: Refuse a k3s restart on several nodes at once
+      ansible.builtin.assert:
+        that: ansible_play_hosts_all | length == 1
+        fail_msg: k3s_restart_allowed restarts k3s on every changed node of the play; limit the run to one node (--limit <node>) to keep the etcd quorum
+      when: k3s_restart_allowed | default(false) | bool
+
     - name: Create the kubelet drop-in directory
       file:
         path: /etc/rancher/k3s/kubelet.conf.d
         state: directory
         mode: "0755"
 
-    - name: Set the kubelet memory reservation and eviction thresholds
+    - name: Set the kubelet eviction thresholds
       copy:
         dest: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf
         mode: "0644"
         content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_memory) | to_nice_yaml(indent=2) }}"
       when: kubernetes.kubelet_memory | length > 0
+      register: kubelet_memory_dropin
+      notify: restart_k3s
 
     - name: Remove the kubelet memory drop-in
       file:
         path: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf
         state: absent
       when: kubernetes.kubelet_memory | length == 0
+      notify: restart_k3s
+
+    - name: Set the kubelet system and kube reservations
+      copy:
+        dest: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf
+        mode: "0644"
+        content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_reserved) | to_nice_yaml(indent=2) }}"
+      when: kubernetes.kubelet_reserved | length > 0
+      register: kubelet_reserved_dropin
+      notify: restart_k3s
+
+    - name: Remove the kubelet reservation drop-in
+      file:
+        path: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf
+        state: absent
+      when: kubernetes.kubelet_reserved | length == 0
+      notify: restart_k3s
+
+    # Compares the file times with the start of the running k3s, so a pending
+    # restart is reported on every run, not only on the run that wrote the file.
+    - name: List the kubelet drop-ins written after k3s started
+      shell: |
+        since=$(systemctl show k3s -p ActiveEnterTimestamp --value)
+        [ -n "$since" ] || exit 0
+        start=$(date -d "$since" +%s) || exit 1
+        for f in /etc/rancher/k3s/kubelet.conf.d/*; do
+          [ "$(stat -c %Y "$f")" -gt "$start" ] && echo "$f"
+        done
+        exit 0
+      register: kubelet_dropins_pending
+      changed_when: false
+      check_mode: false
+
+    - name: Report the pending k3s restart
+      ansible.builtin.debug:
+        msg: >-
+          k3s restart pending on {{ inventory_hostname }}: the kubelet reads
+          /etc/rancher/k3s/kubelet.conf.d only at start. Restart one node at a
+          time with --limit {{ inventory_hostname }} -e k3s_restart_allowed=true.
+      when:
+        - not (k3s_restart_allowed | default(false) | bool)
+        - kubelet_dropins_pending.stdout | length > 0 or kubelet_memory_dropin is changed or kubelet_reserved_dropin is changed
 
 # Every pod watching its config files takes inotify instances and watches from
 # the per-user pool. An LXC container inherits these from its host. Only this