crowdsec: disabled_scenarios removes hub scenarios replaced by local ones, absent_files removes retired local files

crowdsec/defaults/main.yml

@@ -12,6 +12,10 @@ crowdsec:
   scenarios: {}
   # path below /etc/crowdsec -> content, for local parsers and scenarios.
   local_files: {}
+  # name -> installed hub scenario removed from the agent (null drops one); needed where a local scenario replaces it.
+  disabled_scenarios: {}
+  # name -> path below /etc/crowdsec of a retired local file, removed when present (null drops one).
+  absent_files: {}
   # name -> CIDR (single addresses as /32 or /128) never acted upon. Joined by
   # the public_ips (same format) of every inventory host, then whitelisted by
   # this agent and, when lapi_allowlist is set, allowlisted on the LAPI.

crowdsec/tasks/main.yml

@@ -53,6 +53,7 @@
     - /etc/crowdsec/bouncers
     - /etc/crowdsec/parsers/s02-enrich
     - /etc/systemd/system/crowdsec.service.d
+    - /etc/systemd/system/crowdsec-hubupdate.service.d
     - /etc/systemd/system/crowdsec-firewall-bouncer.service.d
     - /etc/systemd/system/nftables.service.d
 
@@ -171,6 +172,37 @@
   when: crowdsec_missing | length > 0
   notify: Restart crowdsec
 
+# The script upgrades every hub item with --force (a scenario removed with --force taints its collection, and a plain upgrade skips
+# tainted collections) and removes the disabled scenarios again. It runs here once and daily as crowdsec-hubupdate.service.
+- name: Install the hub update script
+  ansible.builtin.template:
+    src: crowdsec-hubupdate.sh.j2
+    dest: /usr/local/sbin/crowdsec-hubupdate
+    mode: "0755"
+
+- name: Run the hub update script instead of the package one
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/crowdsec-hubupdate.service.d/override.conf
+    mode: "0644"
+    content: |
+      [Service]
+      ExecStart=
+      ExecStart=/usr/local/sbin/crowdsec-hubupdate
+  register: crowdsec_hubupdate_dropin
+
+- name: Reload systemd for the hub update unit
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: crowdsec_hubupdate_dropin is changed
+
+- name: Upgrade the hub items and remove the disabled scenarios
+  ansible.builtin.command: /usr/local/sbin/crowdsec-hubupdate
+  environment:
+    NO_RELOAD: "1"
+  register: crowdsec_hubupdate
+  changed_when: "'changed' in crowdsec_hubupdate.stdout"
+  notify: Restart crowdsec
+
 - name: Install the local parsers and scenarios
   ansible.builtin.copy:
     dest: "/etc/crowdsec/{{ item.key }}"
@@ -181,6 +213,15 @@
     label: "{{ item.key }}"
   notify: Restart crowdsec
 
+- name: Remove retired local files
+  ansible.builtin.file:
+    path: "/etc/crowdsec/{{ item.value }}"
+    state: absent
+  loop: "{{ crowdsec.absent_files | dict2items | rejectattr('value', 'none') }}"
+  loop_control:
+    label: "{{ item.value }}"
+  notify: Restart crowdsec
+
 - name: Enable the agent and the bouncer
   ansible.builtin.systemd:
     name: "{{ item }}"

crowdsec/templates/crowdsec-hubupdate.sh.j2

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Managed by Ansible. crowdsec-hubupdate.service runs this script instead of /usr/libexec/crowdsec/hubupdate.sh.
+#
+# "cscli scenarios remove --force" taints the collection that holds the scenario, and a plain "cscli hub upgrade" skips tainted
+# collections for good. The upgrade therefore runs with --force, which also brings the removed scenarios back, and the removal is
+# applied again. crowdsec is reloaded only when the installed hub files or the data files differ afterwards; with NO_RELOAD=1
+# the script prints "changed" instead, for the role, which restarts crowdsec once at the end of its run.
+set -eu
+
+test -x /usr/bin/cscli || exit 0
+
+fingerprint() {
+  {
+    find -L /etc/crowdsec -path /etc/crowdsec/hub -prune -o -type f -exec sha256sum {} +
+    find /var/lib/crowdsec/data -maxdepth 1 -type f ! -name 'crowdsec.db*' ! -name trace -exec sha256sum {} +
+  } 2>/dev/null | sort -k2 | sha256sum
+}
+
+before="$(fingerprint)"
+
+/usr/bin/cscli --error hub update >/dev/null
+/usr/bin/cscli --error hub upgrade --force >/dev/null
+
+{% for scenario in crowdsec.disabled_scenarios.values() | reject('none') | unique | sort %}
+if /usr/bin/cscli scenarios list -o raw | cut -d, -f1 | grep -qxF '{{ scenario }}'; then
+  /usr/bin/cscli --error scenarios remove '{{ scenario }}' --force >/dev/null
+fi
+{% endfor %}
+
+if [ "$before" != "$(fingerprint)" ]; then
+  if [ -n "${NO_RELOAD:-}" ]; then
+    echo changed
+  else
+    systemctl reload crowdsec.service
+  fi
+fi