host_firewall: block Spamhaus DROP, DROPv6 and ASN-DROP networks in, out and forward

host_firewall/defaults/main.yml

@@ -54,6 +54,12 @@ firewall:
     # Lets the kernel log the nft log statements of container network
     # namespaces (LXC guests) into the host kernel log.
     all_netns: false
+    # Shared by the Spamhaus drop (fw-spamhaus-in: ) and reject (fw-spamhaus-out: ) chains, one limit each.
+    spamhaus_rate: "6/minute burst 10 packets"
+  # Spamhaus DROP, DROPv6 and ASN-DROP: traffic from the listed networks drops
+  # (input, forward), traffic to them is rejected (output, forward). The sets
+  # are refreshed daily by nft-host-spamhaus.
+  spamhaus: true
   # Reject outbound TCP 25; the mail group turns it off.
   block_smtp_out: true
   # EU-27 plus CH, NO, GB, IS, LI (ipdeny country codes). Empty leaves the geo

host_firewall/files/nft-host-spamhaus

@@ -0,0 +1,171 @@
+#!/usr/bin/python3
+# Managed by Ansible (role host_firewall), do not edit.
+"""Rebuilds the Spamhaus sets spamhaus_v4 and spamhaus_v6 of table inet host.
+
+Sources are the Spamhaus DROP and DROPv6 prefix lists and the ASN-DROP list,
+whose ASNs are expanded to prefixes with the iptoasn bulk files (one download
+per address family). Any failed download or sanity check exits non-zero before
+anything changes, so the previous sets stay in place.
+"""
+import gzip
+import http.client
+import ipaddress
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.request
+
+PERSISTED = "/etc/nftables.d/host-spamhaus.nft"
+SPAMHAUS = "https://www.spamhaus.org/drop/"
+IPTOASN = "https://iptoasn.com/data/"
+USER_AGENT = "ruekov-host-firewall (nft-host-spamhaus)"
+
+# Lower bounds that catch an empty or truncated answer.
+MIN_RECORDS = {"drop_v4": 800, "drop_v6": 30, "asndrop": 200}
+MIN_TSV_LINES = {4: 200000, 6: 20000}
+MIN_ASN_PREFIXES_V4 = 100
+# A prefix this wide is a parse error, not a bad network.
+MIN_PREFIXLEN = {4: 8, 6: 16}
+
+
+def fail(message):
+    print(f"nft-host-spamhaus: {message}", file=sys.stderr)
+    sys.exit(1)
+
+
+def fetch(url, dest):
+    error = None
+    for attempt in range(3):
+        try:
+            request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+            with urllib.request.urlopen(request, timeout=120) as response, open(dest, "wb") as out:
+                shutil.copyfileobj(response, out)
+            return
+        except (OSError, http.client.HTTPException) as e:
+            error = e
+            time.sleep(5 * (attempt + 1))
+    fail(f"download of {url} failed: {error}")
+
+
+def spamhaus_records(name, tmp):
+    path = os.path.join(tmp, name + ".json")
+    fetch(f"{SPAMHAUS}{name}.json", path)
+    records, meta = [], None
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            if not line.strip():
+                continue
+            record = json.loads(line)
+            if record.get("type") == "metadata":
+                meta = record
+            else:
+                records.append(record)
+    if meta is None or meta.get("records") != len(records):
+        fail(f"{name}: record count does not match the metadata record")
+    if len(records) < MIN_RECORDS[name]:
+        fail(f"{name}: only {len(records)} records")
+    return records
+
+
+def asn_prefixes(asns, family, tmp):
+    path = os.path.join(tmp, f"ip2asn-v{family}.tsv.gz")
+    fetch(f"{IPTOASN}ip2asn-v{family}.tsv.gz", path)
+    networks, lines = [], 0
+    with gzip.open(path, "rt", encoding="utf-8") as f:
+        for line in f:
+            lines += 1
+            fields = line.split("\t", 3)
+            if len(fields) < 3 or fields[2] not in asns:
+                continue
+            networks.extend(ipaddress.summarize_address_range(
+                ipaddress.ip_address(fields[0]), ipaddress.ip_address(fields[1])))
+    if lines < MIN_TSV_LINES[family]:
+        fail(f"ip2asn-v{family}: only {lines} lines")
+    return networks
+
+
+def merge(family, *groups):
+    networks, skipped = [], 0
+    for group in groups:
+        for network in group:
+            network = ipaddress.ip_network(network)
+            if network.prefixlen < MIN_PREFIXLEN[family]:
+                fail(f"suspiciously wide prefix {network}")
+            if not network.is_global:
+                skipped += 1
+                continue
+            networks.append(network)
+    return list(ipaddress.collapse_addresses(networks)), skipped
+
+
+def nft(*args):
+    return subprocess.run(["nft", *args], capture_output=True, text=True)
+
+
+def write(path, text):
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(text)
+
+
+def main():
+    with tempfile.TemporaryDirectory() as tmp:
+        drop4 = [r["cidr"] for r in spamhaus_records("drop_v4", tmp)]
+        drop6 = [r["cidr"] for r in spamhaus_records("drop_v6", tmp)]
+        asns = {str(r["asn"]) for r in spamhaus_records("asndrop", tmp)}
+        asn4 = asn_prefixes(asns, 4, tmp)
+        asn6 = asn_prefixes(asns, 6, tmp)
+        if len(asn4) < MIN_ASN_PREFIXES_V4:
+            fail(f"only {len(asn4)} IPv4 prefixes for {len(asns)} ASNs")
+        sets = {}
+        for family, name, drop, asn in ((4, "spamhaus_v4", drop4, asn4), (6, "spamhaus_v6", drop6, asn6)):
+            sets[name], skipped = merge(family, drop, asn)
+            print(f"{name}: {len(drop)} DROP + {len(asn)} ASN-DROP prefixes -> "
+                  f"{len(sets[name])} merged, {skipped} non-global skipped")
+        print(f"ASN-DROP: {len(asns)} ASNs")
+
+        def elements(networks):
+            return ",\n\t\t".join(str(n) for n in networks) + ","
+
+        persisted = ["# Generated by nft-host-spamhaus, do not edit.", "table inet host {"]
+        update = []
+        for name, kind in (("spamhaus_v4", "ipv4_addr"), ("spamhaus_v6", "ipv6_addr")):
+            persisted.append(f"\tset {name} {{ type {kind}; flags interval; auto-merge; elements = {{\n"
+                             f"\t\t{elements(sets[name])}\n\t}} }}")
+            update.append(f"flush set inet host {name}\n"
+                          f"add element inet host {name} {{\n\t{elements(sets[name])}\n}}")
+        persisted.append("}\n")
+
+        persisted_path = os.path.join(tmp, "host-spamhaus.nft")
+        update_path = os.path.join(tmp, "update.nft")
+        write(persisted_path, "\n".join(persisted))
+        write(update_path, "\n".join(update) + "\n")
+
+        # The running table gets the new elements in one transaction; without
+        # the sets (boot, or a rule set rendered without them) the persisted
+        # file is the only copy.
+        loaded = all(nft("list", "set", "inet", "host", n).returncode == 0
+                     for n in ("spamhaus_v4", "spamhaus_v6"))
+        for path in (persisted_path,) + ((update_path,) if loaded else ()):
+            checked = nft("-c", "-f", path)
+            if checked.returncode != 0:
+                fail(f"nft rejects {os.path.basename(path)}: {checked.stderr.strip()}")
+        if loaded:
+            applied = nft("-f", update_path)
+            if applied.returncode != 0:
+                fail(f"loading the sets failed: {applied.stderr.strip()}")
+        shutil.copyfile(persisted_path, PERSISTED + ".new")
+        os.chmod(PERSISTED + ".new", 0o644)
+        os.replace(PERSISTED + ".new", PERSISTED)
+        print("loaded into the running table" if loaded else "persisted only")
+
+
+try:
+    main()
+except SystemExit:
+    raise
+except Exception as e:  # keep the previous sets on any surprise
+    fail(f"{type(e).__name__}: {e}")

host_firewall/files/nft-host-spamhaus.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Refresh the Spamhaus sets of the host firewall
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/nft-host-spamhaus

host_firewall/files/nft-host-spamhaus.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily refresh of the host firewall Spamhaus sets
+
+[Timer]
+OnCalendar=daily
+RandomizedDelaySec=6h
+Persistent=true
+
+[Install]
+WantedBy=timers.target

host_firewall/tasks/main.yml

@@ -43,7 +43,7 @@
     src: "{{ item }}"
     dest: "/etc/systemd/system/{{ item }}"
     mode: "0644"
-  loop: [nft-host.service, nft-host-geo.service, nft-host-geo.timer]
+  loop: [nft-host.service, nft-host-geo.service, nft-host-geo.timer, nft-host-spamhaus.service, nft-host-spamhaus.timer]
   register: firewall_units
 
 - name: Install the rollback script
@@ -105,6 +105,59 @@
         path: /etc/nftables.d/host-geo.nft
         state: absent
 
+- name: Spamhaus sets
+  when: firewall.spamhaus
+  block:
+    - name: Install the Spamhaus set updater
+      ansible.builtin.copy:
+        src: nft-host-spamhaus
+        dest: /usr/local/sbin/nft-host-spamhaus
+        mode: "0755"
+
+    # The rule set includes this file, so a missing one would fail the load at boot.
+    - name: Build the Spamhaus sets on first run
+      ansible.builtin.command: /usr/local/sbin/nft-host-spamhaus
+      args:
+        creates: /etc/nftables.d/host-spamhaus.nft
+
+    - name: Read the Spamhaus sets file
+      ansible.builtin.stat:
+        path: /etc/nftables.d/host-spamhaus.nft
+      register: firewall_spamhaus
+
+    - name: Require non-empty Spamhaus sets
+      ansible.builtin.assert:
+        that: firewall_spamhaus.stat.exists and firewall_spamhaus.stat.size > 0
+        fail_msg: /etc/nftables.d/host-spamhaus.nft is missing or empty, refusing to apply the firewall
+      # Check mode neither installs the updater nor builds the sets.
+      when: not ansible_check_mode
+
+    - name: Enable the Spamhaus set timer
+      ansible.builtin.systemd:
+        name: nft-host-spamhaus.timer
+        enabled: true
+        state: started
+        daemon_reload: "{{ firewall_units is changed }}"
+      # Check mode does not install the unit.
+      ignore_errors: "{{ ansible_check_mode }}"
+
+- name: Stop the Spamhaus set refresh
+  when: not firewall.spamhaus
+  block:
+    - name: Disable the Spamhaus set timer
+      ansible.builtin.systemd:
+        name: nft-host-spamhaus.timer
+        enabled: false
+        state: stopped
+        daemon_reload: "{{ firewall_units is changed }}"
+      # Check mode does not install the unit.
+      ignore_errors: "{{ ansible_check_mode }}"
+
+    - name: Remove the Spamhaus sets file
+      ansible.builtin.file:
+        path: /etc/nftables.d/host-spamhaus.nft
+        state: absent
+
 - name: Render the rule set
   ansible.builtin.template:
     src: host.nft.j2

host_firewall/templates/host.nft.j2

@@ -31,6 +31,9 @@ delete table ip host_nat
 {% if firewall.geo_countries %}
 include "/etc/nftables.d/host-geo.nft"
 {% endif %}
+{% if firewall.spamhaus %}
+include "/etc/nftables.d/host-spamhaus.nft"
+{% endif %}
 
 table inet host {
 	{{ addr_set('admin_v4', 'ipv4_addr', firewall.admin.v4) }}
@@ -48,7 +51,14 @@ table inet host {
 
 	chain input {
 		type filter hook input priority filter; policy drop;
+{% if firewall.spamhaus %}
 
+		# Before the established rule: outbound is blocked too, so listed
+		# addresses have no legitimate replies.
+		ip saddr @spamhaus_v4 jump spamhaus_in
+		ip6 saddr @spamhaus_v6 jump spamhaus_in
+
+{% endif %}
 		ct state established,related accept
 		ct state invalid jump drop_log
 		iif lo accept
@@ -98,15 +108,36 @@ table inet host {
 		limit rate {{ firewall.log.in_rate }} log prefix "fw-in-drop: "
 		drop
 	}
-{% if firewall.block_smtp_out %}
+{% if firewall.spamhaus %}
+
+	# Own limits, so a listed scanner cannot use up the budget of fw-in-drop.
+	# CrowdSec whitelists both prefixes: the lists already ban these networks.
+	chain spamhaus_in {
+		limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-spamhaus-in: "
+		drop
+	}
+
+	chain spamhaus_out {
+		limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-spamhaus-out: "
+		reject
+	}
+{% endif %}
+{% if firewall.block_smtp_out or firewall.spamhaus %}
 
 	# Outbound mail goes through the mail host only. Loopback stays open for a
 	# local MTA; the log lines carry OUT=, which CrowdSec does not count as a scan.
 	chain output {
 		type filter hook output priority filter; policy accept;
+{% if firewall.spamhaus %}
+
+		ip daddr @spamhaus_v4 jump spamhaus_out
+		ip6 daddr @spamhaus_v6 jump spamhaus_out
+{% endif %}
+{% if firewall.block_smtp_out %}
 
 		oifname != "lo" tcp dport 25 limit rate {{ firewall.log.out_rate }} log prefix "fw-out-reject: "
 		oifname != "lo" tcp dport 25 reject with tcp reset
+{% endif %}
 	}
 {% endif %}
 
@@ -164,10 +195,17 @@ table inet host {
 		ip6 saddr @geo_v6 accept
 {% endif %}
 	}
-{% if forward_guard %}
+{% if forward_guard or firewall.spamhaus %}
 
 	chain forward {
 		type filter hook forward priority filter; policy accept;
+{% if firewall.spamhaus %}
+
+		ip saddr @spamhaus_v4 jump spamhaus_in
+		ip6 saddr @spamhaus_v6 jump spamhaus_in
+		ip daddr @spamhaus_v4 jump spamhaus_out
+		ip6 daddr @spamhaus_v6 jump spamhaus_out
+{% endif %}
 {% for guard in forward_guard %}
 
 		# {{ guard.key }}