kubernetes: kubelet_reserved drop-in 30-reserved.conf and a k3s restart handler that runs only with k3s_restart_allowed

kubernetes/defaults/main.yml

@@ -40,10 +40,18 @@ kubernetes:
   # when this is set and differs; empty installs the latest when k3s is missing.
   version: ""
 
-  # KubeletConfiguration fields (systemReserved, evictionHard, ...) written to
+  # KubeletConfiguration fields (evictionHard, evictionSoft, ...) written to
   # /etc/rancher/k3s/kubelet.conf.d/20-memory.conf; empty writes no file
   kubelet_memory: {}
 
+  # KubeletConfiguration fields systemReserved and kubeReserved written to
+  # /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf; empty writes no file.
+  # Both only shrink the allocatable of the node (enforceNodeAllocatable stays
+  # pods). Sized per node in host_vars from the measured memory and CPU of
+  # k3s.service (kubeReserved) and of everything else outside kubepods.slice
+  # (systemReserved).
+  kubelet_reserved: {}
+
   # Extra config of the nodes running etcd, left out with disable-etcd
   etcd_config_extra:
     # Without it etcd serves metrics on 127.0.0.1 only, out of reach of vmagent

kubernetes/handlers/main.yml

@@ -17,3 +17,23 @@
 - name: wait_for_connection
   wait_for_connection:
     delay: 5
+
+# Restarts the kubelet to read changed drop-ins. Runs only with
+# -e k3s_restart_allowed=true; node.yml refuses it for more than one host.
+- name: restart_k3s
+  ansible.builtin.service:
+    name: k3s
+    state: restarted
+  when:
+    - k3s_restart_allowed | default(false) | bool
+    - not ansible_check_mode
+  notify: wait_for_k3s
+
+- name: wait_for_k3s
+  command: k3s kubectl get --raw /readyz
+  register: k3s_readyz
+  until: k3s_readyz.rc == 0 and k3s_readyz.stdout == 'ok'
+  retries: 30
+  delay: 10
+  changed_when: false
+  when: not ansible_check_mode

kubernetes/tasks/node.yml

@@ -18,29 +18,82 @@
           imageGCLowThresholdPercent: 55
           imageMaximumGCAge: 168h
 
-# KubeletConfiguration fields from kubernetes.kubelet_memory; the file is removed
+# KubeletConfiguration fields from kubernetes.kubelet_memory (eviction) and
-# while the variable is empty.
+# kubernetes.kubelet_reserved (systemReserved, kubeReserved); a file is removed
+# while its variable is empty. The kubelet reads the drop-ins only when k3s
+# starts, so a change takes effect after a k3s restart. The restart_k3s handler
+# restarts only with -e k3s_restart_allowed=true, on one node per run.
 - name: Kubelet memory reservation and eviction
-  tags: kubelet_memory
+  tags: [kubelet_memory, kubelet_reserved]
   block:
+    - name: Refuse a k3s restart on several nodes at once
+      ansible.builtin.assert:
+        that: ansible_play_hosts_all | length == 1
+        fail_msg: k3s_restart_allowed restarts k3s on every changed node of the play; limit the run to one node (--limit <node>) to keep the etcd quorum
+      when: k3s_restart_allowed | default(false) | bool
+
     - name: Create the kubelet drop-in directory
       file:
         path: /etc/rancher/k3s/kubelet.conf.d
         state: directory
         mode: "0755"
 
-    - name: Set the kubelet memory reservation and eviction thresholds
+    - name: Set the kubelet eviction thresholds
       copy:
         dest: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf
         mode: "0644"
         content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_memory) | to_nice_yaml(indent=2) }}"
       when: kubernetes.kubelet_memory | length > 0
+      register: kubelet_memory_dropin
+      notify: restart_k3s
 
     - name: Remove the kubelet memory drop-in
       file:
         path: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf
         state: absent
       when: kubernetes.kubelet_memory | length == 0
+      notify: restart_k3s
+
+    - name: Set the kubelet system and kube reservations
+      copy:
+        dest: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf
+        mode: "0644"
+        content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_reserved) | to_nice_yaml(indent=2) }}"
+      when: kubernetes.kubelet_reserved | length > 0
+      register: kubelet_reserved_dropin
+      notify: restart_k3s
+
+    - name: Remove the kubelet reservation drop-in
+      file:
+        path: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf
+        state: absent
+      when: kubernetes.kubelet_reserved | length == 0
+      notify: restart_k3s
+
+    # Compares the file times with the start of the running k3s, so a pending
+    # restart is reported on every run, not only on the run that wrote the file.
+    - name: List the kubelet drop-ins written after k3s started
+      shell: |
+        since=$(systemctl show k3s -p ActiveEnterTimestamp --value)
+        [ -n "$since" ] || exit 0
+        start=$(date -d "$since" +%s) || exit 1
+        for f in /etc/rancher/k3s/kubelet.conf.d/*; do
+          [ "$(stat -c %Y "$f")" -gt "$start" ] && echo "$f"
+        done
+        exit 0
+      register: kubelet_dropins_pending
+      changed_when: false
+      check_mode: false
+
+    - name: Report the pending k3s restart
+      ansible.builtin.debug:
+        msg: >-
+          k3s restart pending on {{ inventory_hostname }}: the kubelet reads
+          /etc/rancher/k3s/kubelet.conf.d only at start. Restart one node at a
+          time with --limit {{ inventory_hostname }} -e k3s_restart_allowed=true.
+      when:
+        - not (k3s_restart_allowed | default(false) | bool)
+        - kubelet_dropins_pending.stdout | length > 0 or kubelet_memory_dropin is changed or kubelet_reserved_dropin is changed
 
 # Every pod watching its config files takes inotify instances and watches from
 # the per-user pool. An LXC container inherits these from its host. Only this