From c50c32059e7a0665b9ca070438be89c6babe6f14 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Sun, 25 Oct 2026 12:12:27 +0000 Subject: [PATCH] host_firewall: drop inbound traffic from internet-wide scanners (OpenFilters internet-scanners), v4 and v6, daily refresh --- host_firewall/defaults/main.yml | 12 +- host_firewall/files/nft-host-scanners | 182 ++++++++++++++++++ host_firewall/files/nft-host-scanners.service | 8 + host_firewall/files/nft-host-scanners.timer | 10 + host_firewall/tasks/main.yml | 62 +++++- host_firewall/templates/host.nft.j2 | 27 ++- .../templates/nft-host-scanners.allowlist.j2 | 11 ++ 7 files changed, 309 insertions(+), 3 deletions(-) create mode 100755 host_firewall/files/nft-host-scanners create mode 100644 host_firewall/files/nft-host-scanners.service create mode 100644 host_firewall/files/nft-host-scanners.timer create mode 100644 host_firewall/templates/nft-host-scanners.allowlist.j2 diff --git a/host_firewall/defaults/main.yml b/host_firewall/defaults/main.yml index 304a683..6dccf07 100644 --- a/host_firewall/defaults/main.yml +++ b/host_firewall/defaults/main.yml @@ -54,7 +54,7 @@ firewall: # Lets the kernel log the nft log statements of container network # namespaces (LXC guests) into the host kernel log. all_netns: false - # Shared by the Spamhaus (fw-spamhaus-in: , fw-spamhaus-out: ) and FireHOL (fw-firehol-in: , fw-firehol-out: ) chains, one limit each. + # Shared by the Spamhaus (fw-spamhaus-in: , fw-spamhaus-out: ), FireHOL (fw-firehol-in: , fw-firehol-out: ) and scanner (fw-scanners-in: ) chains, one limit each. spamhaus_rate: "6/minute burst 10 packets" # Spamhaus DROP, DROPv6 and ASN-DROP: traffic from the listed networks drops # (input, forward), traffic to them is rejected (output, forward). The sets @@ -65,6 +65,16 @@ firewall: # private or reserved space and ones holding an address of the host are left # out. Refreshed daily by nft-host-firehol. firehol: true + # Internet-wide scanners (Censys, Shodan, ONYPHE, Shadowserver, ...), IPv4 + # and IPv6: the cidr files of github.com/OpenFilters/internet-scanners + # (Unlicense). Inbound traffic from the listed networks drops (input, + # forward) once the established rule has passed; outbound traffic is never + # filtered. Log prefix fw-scanners-in: with the Spamhaus log limit, which + # CrowdSec whitelists. Entries in private or reserved space, overlapping the + # admin, scraper and cluster sets, the CrowdSec whitelist or the public_ips + # of any host, and ones holding an address of the host are left out + # (/etc/nft-host-scanners.allowlist). Refreshed daily by nft-host-scanners. + scanners: true # Reject outbound TCP 25; the mail group turns it off. block_smtp_out: true # EU-27 plus CH, NO, GB, IS, LI (ipdeny country codes). Empty leaves the geo diff --git a/host_firewall/files/nft-host-scanners b/host_firewall/files/nft-host-scanners new file mode 100755 index 0000000..6e97ee8 --- /dev/null +++ b/host_firewall/files/nft-host-scanners @@ -0,0 +1,182 @@ +#!/usr/bin/python3 +# Managed by Ansible (role host_firewall), do not edit. +"""Rebuilds the internet scanner sets scanners_v4 and scanners_v6 of table inet host. + +Source: the cidr directory of OpenFilters/internet-scanners (Unlicense), read +from the tarball of the default branch. A failed download or sanity check +exits non-zero before anything changes, so the previous sets stay in place. +""" +import io +import ipaddress +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.request + +PERSISTED = "/etc/nftables.d/host-scanners.nft" +# One CIDR or address per line: the public addresses of the hosts, the +# CrowdSec whitelist and the admin, scraper and cluster sets of the firewall. +ALLOWLIST = "/etc/nft-host-scanners.allowlist" +URL = "https://codeload.github.com/OpenFilters/internet-scanners/tar.gz/refs/heads/main" +USER_AGENT = "ruekov-host-firewall (nft-host-scanners)" +MEMBER = re.compile(r"^[^/]+/cidr/[a-z0-9]+_v([46])\.txt$") +MAX_BYTES = 50 * 1024 * 1024 + +# Lower bounds that catch an empty or truncated answer (the list holds about +# 21000 IPv4 and 3000 IPv6 entries). +MIN_ENTRIES = {4: 5000, 6: 500} +# A prefix this wide is a parse error, not a scanner network. +MIN_PREFIXLEN = {4: 12, 6: 24} +# Private, CGNAT (NetBird), loopback, link-local, ULA and multicast/reserved +# space: an entry overlapping one of these never lands in a set, so the own +# infrastructure cannot block itself. +EXCLUDED = [ipaddress.ip_network(n) for n in ( + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", + "169.254.0.0/16", "172.16.0.0/12", "192.168.0.0/16", "224.0.0.0/3", + "::/128", "::1/128", "::ffff:0:0/96", "fc00::/7", "fe80::/10", "ff00::/8")] + + +def fail(message): + print(f"nft-host-scanners: {message}", file=sys.stderr) + sys.exit(1) + + +def fetch(): + error = None + for attempt in range(3): + try: + request = urllib.request.Request(URL, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=120) as response: + data = response.read(MAX_BYTES + 1) + if len(data) > MAX_BYTES: + fail(f"answer from {URL} exceeds {MAX_BYTES} bytes") + return data + except OSError as e: + error = e + time.sleep(5 * (attempt + 1)) + fail(f"download of {URL} failed: {error}") + + +def parse(data): + """Returns {4: [networks], 6: [networks]} from the cidr/*_v4.txt and *_v6.txt files.""" + networks = {4: [], 6: []} + try: + archive = tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") + except (tarfile.TarError, OSError) as e: + fail(f"unreadable archive: {e}") + with archive: + for member in archive: + match = MEMBER.match(member.name) + if not match or not member.isfile(): + continue + version = int(match.group(1)) + for line in archive.extractfile(member).read().decode("utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + try: + # Host bits set (34.19.127.176/27) are masked, not rejected. + network = ipaddress.ip_network(line, strict=False) + except ValueError: + fail(f"unparsable line in {member.name}: {line!r}") + if network.version != version: + fail(f"IPv{network.version} entry in {member.name}: {line!r}") + networks[version].append(network) + return networks + + +def read_allowlist(): + entries = [] + with open(ALLOWLIST, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + try: + entries.append(ipaddress.ip_network(line, strict=False)) + except ValueError: + fail(f"unparsable line in {ALLOWLIST}: {line!r}") + return entries + + +def own_addresses(): + result = subprocess.run(["ip", "-j", "address", "show"], capture_output=True, text=True) + if result.returncode != 0: + fail(f"ip address show failed: {result.stderr.strip()}") + return [ipaddress.ip_address(a["local"]) for link in json.loads(result.stdout) + for a in link.get("addr_info", []) if a.get("family") in ("inet", "inet6")] + + +def main(): + listed = parse(fetch()) + for version, minimum in MIN_ENTRIES.items(): + if len(listed[version]) < minimum: + fail(f"only {len(listed[version])} IPv{version} entries") + + protected = EXCLUDED + read_allowlist() + own = own_addresses() + merged = {} + for version in (4, 6): + kept, protected_skipped, own_skipped = [], 0, 0 + for network in listed[version]: + if network.prefixlen < MIN_PREFIXLEN[version]: + fail(f"suspiciously wide prefix {network}") + if any(p.version == version and network.overlaps(p) for p in protected): + protected_skipped += 1 + elif any(a.version == version and a in network for a in own): + own_skipped += 1 + else: + kept.append(network) + merged[version] = list(ipaddress.collapse_addresses(kept)) + print(f"scanners_v{version}: {len(listed[version])} entries, {protected_skipped} in private, reserved or " + f"allowlisted space and {own_skipped} holding an own address skipped -> {len(merged[version])} merged") + + persisted = "# Generated by nft-host-scanners, do not edit.\ntable inet host {\n" + for version in (4, 6): + elements = ",\n\t\t".join(str(n) for n in merged[version]) + "," + persisted += (f"\tset scanners_v{version} {{ type ipv{version}_addr; flags interval; auto-merge; elements = {{\n" + f"\t\t{elements}\n\t}} }}\n") + persisted += "}\n" + + # Checked under another table name: a dry run against the live table + # name changes the live sets for its duration and unlisted addresses match. + check_rules = persisted.replace("table inet host {", "table inet host_scanners_check {") + checked = subprocess.run(["nft", "-c", "-f", "-"], input=check_rules, capture_output=True, text=True) + if checked.returncode != 0: + fail(f"nft rejects the sets: {checked.stderr.strip()}") + + with tempfile.TemporaryDirectory() as tmp: + previous = None + if os.path.exists(PERSISTED): + previous = os.path.join(tmp, "previous.nft") + shutil.copyfile(PERSISTED, previous) + with open(PERSISTED + ".new", "w", encoding="utf-8") as f: + f.write(persisted) + os.chmod(PERSISTED + ".new", 0o644) + os.replace(PERSISTED + ".new", PERSISTED) + + # A reload replaces the whole table, sets included, in one transaction. + active = subprocess.run(["systemctl", "-q", "is-active", "nft-host.service"]).returncode == 0 + if active: + reloaded = subprocess.run(["systemctl", "reload", "nft-host.service"], capture_output=True, text=True) + if reloaded.returncode != 0: + if previous: + shutil.copyfile(previous, PERSISTED) + else: + os.remove(PERSISTED) + fail(f"reloading the rule set failed, kept the previous sets: {reloaded.stderr.strip()}") + print("reloaded the rule set" if active else "persisted only") + + +try: + main() +except SystemExit: + raise +except Exception as e: # keep the previous sets on any surprise + fail(f"{type(e).__name__}: {e}") diff --git a/host_firewall/files/nft-host-scanners.service b/host_firewall/files/nft-host-scanners.service new file mode 100644 index 0000000..84f708d --- /dev/null +++ b/host_firewall/files/nft-host-scanners.service @@ -0,0 +1,8 @@ +[Unit] +Description=Refresh the internet scanner sets of the host firewall +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/nft-host-scanners diff --git a/host_firewall/files/nft-host-scanners.timer b/host_firewall/files/nft-host-scanners.timer new file mode 100644 index 0000000..bb0af6f --- /dev/null +++ b/host_firewall/files/nft-host-scanners.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily refresh of the host firewall internet scanner sets + +[Timer] +OnCalendar=daily +RandomizedDelaySec=6h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/host_firewall/tasks/main.yml b/host_firewall/tasks/main.yml index 28f4347..51cb7d7 100644 --- a/host_firewall/tasks/main.yml +++ b/host_firewall/tasks/main.yml @@ -43,7 +43,7 @@ src: "{{ item }}" dest: "/etc/systemd/system/{{ item }}" mode: "0644" - loop: [nft-host.service, nft-host-geo.service, nft-host-geo.timer, nft-host-spamhaus.service, nft-host-spamhaus.timer, nft-host-firehol.service, nft-host-firehol.timer] + loop: [nft-host.service, nft-host-geo.service, nft-host-geo.timer, nft-host-spamhaus.service, nft-host-spamhaus.timer, nft-host-firehol.service, nft-host-firehol.timer, nft-host-scanners.service, nft-host-scanners.timer] register: firewall_units - name: Install the rollback script @@ -211,6 +211,66 @@ path: /etc/nftables.d/host-firehol.nft state: absent +- name: Internet scanner sets + when: firewall.scanners + block: + - name: Install the internet scanner set updater + ansible.builtin.copy: + src: nft-host-scanners + dest: /usr/local/sbin/nft-host-scanners + mode: "0755" + + # The updater leaves out entries overlapping these addresses. + - name: Install the internet scanner allowlist + ansible.builtin.template: + src: nft-host-scanners.allowlist.j2 + dest: /etc/nft-host-scanners.allowlist + mode: "0644" + + # The rule set includes this file, so a missing one would fail the load at boot. + - name: Build the internet scanner sets on first run + ansible.builtin.command: /usr/local/sbin/nft-host-scanners + args: + creates: /etc/nftables.d/host-scanners.nft + + - name: Read the internet scanner sets file + ansible.builtin.stat: + path: /etc/nftables.d/host-scanners.nft + register: firewall_scanners + + - name: Require non-empty internet scanner sets + ansible.builtin.assert: + that: firewall_scanners.stat.exists and firewall_scanners.stat.size > 0 + fail_msg: /etc/nftables.d/host-scanners.nft is missing or empty, refusing to apply the firewall + # Check mode neither installs the updater nor builds the sets. + when: not ansible_check_mode + + - name: Enable the internet scanner set timer + ansible.builtin.systemd: + name: nft-host-scanners.timer + enabled: true + state: started + daemon_reload: "{{ firewall_units is changed }}" + # Check mode does not install the unit. + ignore_errors: "{{ ansible_check_mode }}" + +- name: Stop the internet scanner set refresh + when: not firewall.scanners + block: + - name: Disable the internet scanner set timer + ansible.builtin.systemd: + name: nft-host-scanners.timer + enabled: false + state: stopped + daemon_reload: "{{ firewall_units is changed }}" + # Check mode does not install the unit. + ignore_errors: "{{ ansible_check_mode }}" + + - name: Remove the internet scanner sets file + ansible.builtin.file: + path: /etc/nftables.d/host-scanners.nft + state: absent + - name: Render the rule set ansible.builtin.template: src: host.nft.j2 diff --git a/host_firewall/templates/host.nft.j2 b/host_firewall/templates/host.nft.j2 index 824ba06..103ab6c 100644 --- a/host_firewall/templates/host.nft.j2 +++ b/host_firewall/templates/host.nft.j2 @@ -38,6 +38,9 @@ include "/etc/nftables.d/host-spamhaus.nft" {% if firewall.firehol %} include "/etc/nftables.d/host-firehol.nft" {% endif %} +{% if firewall.scanners %} +include "/etc/nftables.d/host-scanners.nft" +{% endif %} table inet host { {{ addr_set('admin_v4', 'ipv4_addr', firewall.admin.v4) }} @@ -69,6 +72,13 @@ table inet host { {% endif %} ct state established,related accept +{% if firewall.scanners %} + + # After the established rule: only inbound connections of scanners drop. + ip saddr @scanners_v4 jump scanners_in + ip6 saddr @scanners_v6 jump scanners_in + +{% endif %} ct state invalid jump drop_log iif lo accept @@ -143,6 +153,16 @@ table inet host { reject } {% endif %} +{% if firewall.scanners %} + + # Own limit and prefix, so scanners use up neither the fw-in-drop budget + # nor the one of the blocklists. CrowdSec whitelists the prefix: scans of + # these networks are known and need no decision. + chain scanners_in { + limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-scanners-in: " + drop + } +{% endif %} {% if firewall.block_smtp_out or blocklists %} # Outbound mail goes through the mail host only. Loopback stays open for a @@ -220,7 +240,7 @@ table inet host { ip6 saddr @geo_v6 accept {% endif %} } -{% if forward_guard or blocklists %} +{% if forward_guard or blocklists or firewall.scanners %} chain forward { type filter hook forward priority filter; policy accept; @@ -236,6 +256,11 @@ table inet host { ip saddr @firehol_v4 jump firehol_in ip daddr @firehol_v4 jump firehol_out {% endif %} +{% if firewall.scanners %} + + ct state new ip saddr @scanners_v4 jump scanners_in + ct state new ip6 saddr @scanners_v6 jump scanners_in +{% endif %} {% for guard in forward_guard %} # {{ guard.key }} diff --git a/host_firewall/templates/nft-host-scanners.allowlist.j2 b/host_firewall/templates/nft-host-scanners.allowlist.j2 new file mode 100644 index 0000000..e21d286 --- /dev/null +++ b/host_firewall/templates/nft-host-scanners.allowlist.j2 @@ -0,0 +1,11 @@ +# {{ ansible_managed }} +{% set maps = [ + (crowdsec | default({}, true)).whitelist | default({}, true), + firewall.admin.v4 | default({}, true), firewall.admin.v6 | default({}, true), + firewall.scrapers.v4 | default({}, true), firewall.scrapers.v6 | default({}, true), + firewall.cluster.v4 | default({}, true), firewall.cluster.v6 | default({}, true)] %} +{% set entries = (maps | map('dict2items') | sum(start=[]) | rejectattr('value', 'none') | map(attribute='value') | map('string') | list + + groups['all'] | map('extract', hostvars, 'public_ips') | select('defined') | flatten) | unique %} +{% for entry in entries %} +{{ entry }} +{% endfor %}