diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml index ab258db..e4d2c62 100644 --- a/kubernetes/defaults/main.yml +++ b/kubernetes/defaults/main.yml @@ -40,10 +40,18 @@ kubernetes: # when this is set and differs; empty installs the latest when k3s is missing. version: "" - # KubeletConfiguration fields (systemReserved, evictionHard, ...) written to + # KubeletConfiguration fields (evictionHard, evictionSoft, ...) written to # /etc/rancher/k3s/kubelet.conf.d/20-memory.conf; empty writes no file kubelet_memory: {} + # KubeletConfiguration fields systemReserved and kubeReserved written to + # /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf; empty writes no file. + # Both only shrink the allocatable of the node (enforceNodeAllocatable stays + # pods). Sized per node in host_vars from the measured memory and CPU of + # k3s.service (kubeReserved) and of everything else outside kubepods.slice + # (systemReserved). + kubelet_reserved: {} + # Extra config of the nodes running etcd, left out with disable-etcd etcd_config_extra: # Without it etcd serves metrics on 127.0.0.1 only, out of reach of vmagent diff --git a/kubernetes/handlers/main.yml b/kubernetes/handlers/main.yml index 5b55d71..79c578f 100644 --- a/kubernetes/handlers/main.yml +++ b/kubernetes/handlers/main.yml @@ -17,3 +17,23 @@ - name: wait_for_connection wait_for_connection: delay: 5 + +# Restarts the kubelet to read changed drop-ins. Runs only with +# -e k3s_restart_allowed=true; node.yml refuses it for more than one host. +- name: restart_k3s + ansible.builtin.service: + name: k3s + state: restarted + when: + - k3s_restart_allowed | default(false) | bool + - not ansible_check_mode + notify: wait_for_k3s + +- name: wait_for_k3s + command: k3s kubectl get --raw /readyz + register: k3s_readyz + until: k3s_readyz.rc == 0 and k3s_readyz.stdout == 'ok' + retries: 30 + delay: 10 + changed_when: false + when: not ansible_check_mode diff --git a/kubernetes/tasks/node.yml b/kubernetes/tasks/node.yml index 40c5919..a45a6d0 100644 --- a/kubernetes/tasks/node.yml +++ b/kubernetes/tasks/node.yml @@ -18,29 +18,82 @@ imageGCLowThresholdPercent: 55 imageMaximumGCAge: 168h -# KubeletConfiguration fields from kubernetes.kubelet_memory; the file is removed -# while the variable is empty. +# KubeletConfiguration fields from kubernetes.kubelet_memory (eviction) and +# kubernetes.kubelet_reserved (systemReserved, kubeReserved); a file is removed +# while its variable is empty. The kubelet reads the drop-ins only when k3s +# starts, so a change takes effect after a k3s restart. The restart_k3s handler +# restarts only with -e k3s_restart_allowed=true, on one node per run. - name: Kubelet memory reservation and eviction - tags: kubelet_memory + tags: [kubelet_memory, kubelet_reserved] block: + - name: Refuse a k3s restart on several nodes at once + ansible.builtin.assert: + that: ansible_play_hosts_all | length == 1 + fail_msg: k3s_restart_allowed restarts k3s on every changed node of the play; limit the run to one node (--limit ) to keep the etcd quorum + when: k3s_restart_allowed | default(false) | bool + - name: Create the kubelet drop-in directory file: path: /etc/rancher/k3s/kubelet.conf.d state: directory mode: "0755" - - name: Set the kubelet memory reservation and eviction thresholds + - name: Set the kubelet eviction thresholds copy: dest: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf mode: "0644" content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_memory) | to_nice_yaml(indent=2) }}" when: kubernetes.kubelet_memory | length > 0 + register: kubelet_memory_dropin + notify: restart_k3s - name: Remove the kubelet memory drop-in file: path: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf state: absent when: kubernetes.kubelet_memory | length == 0 + notify: restart_k3s + + - name: Set the kubelet system and kube reservations + copy: + dest: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf + mode: "0644" + content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_reserved) | to_nice_yaml(indent=2) }}" + when: kubernetes.kubelet_reserved | length > 0 + register: kubelet_reserved_dropin + notify: restart_k3s + + - name: Remove the kubelet reservation drop-in + file: + path: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf + state: absent + when: kubernetes.kubelet_reserved | length == 0 + notify: restart_k3s + + # Compares the file times with the start of the running k3s, so a pending + # restart is reported on every run, not only on the run that wrote the file. + - name: List the kubelet drop-ins written after k3s started + shell: | + since=$(systemctl show k3s -p ActiveEnterTimestamp --value) + [ -n "$since" ] || exit 0 + start=$(date -d "$since" +%s) || exit 1 + for f in /etc/rancher/k3s/kubelet.conf.d/*; do + [ "$(stat -c %Y "$f")" -gt "$start" ] && echo "$f" + done + exit 0 + register: kubelet_dropins_pending + changed_when: false + check_mode: false + + - name: Report the pending k3s restart + ansible.builtin.debug: + msg: >- + k3s restart pending on {{ inventory_hostname }}: the kubelet reads + /etc/rancher/k3s/kubelet.conf.d only at start. Restart one node at a + time with --limit {{ inventory_hostname }} -e k3s_restart_allowed=true. + when: + - not (k3s_restart_allowed | default(false) | bool) + - kubelet_dropins_pending.stdout | length > 0 or kubelet_memory_dropin is changed or kubelet_reserved_dropin is changed # Every pod watching its config files takes inotify instances and watches from # the per-user pool. An LXC container inherits these from its host. Only this