host_firewall: drop inbound traffic from internet-wide scanners (OpenFilters internet-scanners), v4 and v6, daily refresh
This commit is contained in:
@@ -54,7 +54,7 @@ firewall:
|
||||
# Lets the kernel log the nft log statements of container network
|
||||
# namespaces (LXC guests) into the host kernel log.
|
||||
all_netns: false
|
||||
# Shared by the Spamhaus (fw-spamhaus-in: , fw-spamhaus-out: ) and FireHOL (fw-firehol-in: , fw-firehol-out: ) chains, one limit each.
|
||||
# Shared by the Spamhaus (fw-spamhaus-in: , fw-spamhaus-out: ), FireHOL (fw-firehol-in: , fw-firehol-out: ) and scanner (fw-scanners-in: ) chains, one limit each.
|
||||
spamhaus_rate: "6/minute burst 10 packets"
|
||||
# Spamhaus DROP, DROPv6 and ASN-DROP: traffic from the listed networks drops
|
||||
# (input, forward), traffic to them is rejected (output, forward). The sets
|
||||
@@ -65,6 +65,16 @@ firewall:
|
||||
# private or reserved space and ones holding an address of the host are left
|
||||
# out. Refreshed daily by nft-host-firehol.
|
||||
firehol: true
|
||||
# Internet-wide scanners (Censys, Shodan, ONYPHE, Shadowserver, ...), IPv4
|
||||
# and IPv6: the cidr files of github.com/OpenFilters/internet-scanners
|
||||
# (Unlicense). Inbound traffic from the listed networks drops (input,
|
||||
# forward) once the established rule has passed; outbound traffic is never
|
||||
# filtered. Log prefix fw-scanners-in: with the Spamhaus log limit, which
|
||||
# CrowdSec whitelists. Entries in private or reserved space, overlapping the
|
||||
# admin, scraper and cluster sets, the CrowdSec whitelist or the public_ips
|
||||
# of any host, and ones holding an address of the host are left out
|
||||
# (/etc/nft-host-scanners.allowlist). Refreshed daily by nft-host-scanners.
|
||||
scanners: true
|
||||
# Reject outbound TCP 25; the mail group turns it off.
|
||||
block_smtp_out: true
|
||||
# EU-27 plus CH, NO, GB, IS, LI (ipdeny country codes). Empty leaves the geo
|
||||
|
||||
Executable
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/python3
|
||||
# Managed by Ansible (role host_firewall), do not edit.
|
||||
"""Rebuilds the internet scanner sets scanners_v4 and scanners_v6 of table inet host.
|
||||
|
||||
Source: the cidr directory of OpenFilters/internet-scanners (Unlicense), read
|
||||
from the tarball of the default branch. A failed download or sanity check
|
||||
exits non-zero before anything changes, so the previous sets stay in place.
|
||||
"""
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
PERSISTED = "/etc/nftables.d/host-scanners.nft"
|
||||
# One CIDR or address per line: the public addresses of the hosts, the
|
||||
# CrowdSec whitelist and the admin, scraper and cluster sets of the firewall.
|
||||
ALLOWLIST = "/etc/nft-host-scanners.allowlist"
|
||||
URL = "https://codeload.github.com/OpenFilters/internet-scanners/tar.gz/refs/heads/main"
|
||||
USER_AGENT = "ruekov-host-firewall (nft-host-scanners)"
|
||||
MEMBER = re.compile(r"^[^/]+/cidr/[a-z0-9]+_v([46])\.txt$")
|
||||
MAX_BYTES = 50 * 1024 * 1024
|
||||
|
||||
# Lower bounds that catch an empty or truncated answer (the list holds about
|
||||
# 21000 IPv4 and 3000 IPv6 entries).
|
||||
MIN_ENTRIES = {4: 5000, 6: 500}
|
||||
# A prefix this wide is a parse error, not a scanner network.
|
||||
MIN_PREFIXLEN = {4: 12, 6: 24}
|
||||
# Private, CGNAT (NetBird), loopback, link-local, ULA and multicast/reserved
|
||||
# space: an entry overlapping one of these never lands in a set, so the own
|
||||
# infrastructure cannot block itself.
|
||||
EXCLUDED = [ipaddress.ip_network(n) for n in (
|
||||
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8",
|
||||
"169.254.0.0/16", "172.16.0.0/12", "192.168.0.0/16", "224.0.0.0/3",
|
||||
"::/128", "::1/128", "::ffff:0:0/96", "fc00::/7", "fe80::/10", "ff00::/8")]
|
||||
|
||||
|
||||
def fail(message):
|
||||
print(f"nft-host-scanners: {message}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def fetch():
|
||||
error = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
request = urllib.request.Request(URL, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
data = response.read(MAX_BYTES + 1)
|
||||
if len(data) > MAX_BYTES:
|
||||
fail(f"answer from {URL} exceeds {MAX_BYTES} bytes")
|
||||
return data
|
||||
except OSError as e:
|
||||
error = e
|
||||
time.sleep(5 * (attempt + 1))
|
||||
fail(f"download of {URL} failed: {error}")
|
||||
|
||||
|
||||
def parse(data):
|
||||
"""Returns {4: [networks], 6: [networks]} from the cidr/*_v4.txt and *_v6.txt files."""
|
||||
networks = {4: [], 6: []}
|
||||
try:
|
||||
archive = tarfile.open(fileobj=io.BytesIO(data), mode="r:gz")
|
||||
except (tarfile.TarError, OSError) as e:
|
||||
fail(f"unreadable archive: {e}")
|
||||
with archive:
|
||||
for member in archive:
|
||||
match = MEMBER.match(member.name)
|
||||
if not match or not member.isfile():
|
||||
continue
|
||||
version = int(match.group(1))
|
||||
for line in archive.extractfile(member).read().decode("utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
# Host bits set (34.19.127.176/27) are masked, not rejected.
|
||||
network = ipaddress.ip_network(line, strict=False)
|
||||
except ValueError:
|
||||
fail(f"unparsable line in {member.name}: {line!r}")
|
||||
if network.version != version:
|
||||
fail(f"IPv{network.version} entry in {member.name}: {line!r}")
|
||||
networks[version].append(network)
|
||||
return networks
|
||||
|
||||
|
||||
def read_allowlist():
|
||||
entries = []
|
||||
with open(ALLOWLIST, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
entries.append(ipaddress.ip_network(line, strict=False))
|
||||
except ValueError:
|
||||
fail(f"unparsable line in {ALLOWLIST}: {line!r}")
|
||||
return entries
|
||||
|
||||
|
||||
def own_addresses():
|
||||
result = subprocess.run(["ip", "-j", "address", "show"], capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
fail(f"ip address show failed: {result.stderr.strip()}")
|
||||
return [ipaddress.ip_address(a["local"]) for link in json.loads(result.stdout)
|
||||
for a in link.get("addr_info", []) if a.get("family") in ("inet", "inet6")]
|
||||
|
||||
|
||||
def main():
|
||||
listed = parse(fetch())
|
||||
for version, minimum in MIN_ENTRIES.items():
|
||||
if len(listed[version]) < minimum:
|
||||
fail(f"only {len(listed[version])} IPv{version} entries")
|
||||
|
||||
protected = EXCLUDED + read_allowlist()
|
||||
own = own_addresses()
|
||||
merged = {}
|
||||
for version in (4, 6):
|
||||
kept, protected_skipped, own_skipped = [], 0, 0
|
||||
for network in listed[version]:
|
||||
if network.prefixlen < MIN_PREFIXLEN[version]:
|
||||
fail(f"suspiciously wide prefix {network}")
|
||||
if any(p.version == version and network.overlaps(p) for p in protected):
|
||||
protected_skipped += 1
|
||||
elif any(a.version == version and a in network for a in own):
|
||||
own_skipped += 1
|
||||
else:
|
||||
kept.append(network)
|
||||
merged[version] = list(ipaddress.collapse_addresses(kept))
|
||||
print(f"scanners_v{version}: {len(listed[version])} entries, {protected_skipped} in private, reserved or "
|
||||
f"allowlisted space and {own_skipped} holding an own address skipped -> {len(merged[version])} merged")
|
||||
|
||||
persisted = "# Generated by nft-host-scanners, do not edit.\ntable inet host {\n"
|
||||
for version in (4, 6):
|
||||
elements = ",\n\t\t".join(str(n) for n in merged[version]) + ","
|
||||
persisted += (f"\tset scanners_v{version} {{ type ipv{version}_addr; flags interval; auto-merge; elements = {{\n"
|
||||
f"\t\t{elements}\n\t}} }}\n")
|
||||
persisted += "}\n"
|
||||
|
||||
# Checked under another table name: a dry run against the live table
|
||||
# name changes the live sets for its duration and unlisted addresses match.
|
||||
check_rules = persisted.replace("table inet host {", "table inet host_scanners_check {")
|
||||
checked = subprocess.run(["nft", "-c", "-f", "-"], input=check_rules, capture_output=True, text=True)
|
||||
if checked.returncode != 0:
|
||||
fail(f"nft rejects the sets: {checked.stderr.strip()}")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
previous = None
|
||||
if os.path.exists(PERSISTED):
|
||||
previous = os.path.join(tmp, "previous.nft")
|
||||
shutil.copyfile(PERSISTED, previous)
|
||||
with open(PERSISTED + ".new", "w", encoding="utf-8") as f:
|
||||
f.write(persisted)
|
||||
os.chmod(PERSISTED + ".new", 0o644)
|
||||
os.replace(PERSISTED + ".new", PERSISTED)
|
||||
|
||||
# A reload replaces the whole table, sets included, in one transaction.
|
||||
active = subprocess.run(["systemctl", "-q", "is-active", "nft-host.service"]).returncode == 0
|
||||
if active:
|
||||
reloaded = subprocess.run(["systemctl", "reload", "nft-host.service"], capture_output=True, text=True)
|
||||
if reloaded.returncode != 0:
|
||||
if previous:
|
||||
shutil.copyfile(previous, PERSISTED)
|
||||
else:
|
||||
os.remove(PERSISTED)
|
||||
fail(f"reloading the rule set failed, kept the previous sets: {reloaded.stderr.strip()}")
|
||||
print("reloaded the rule set" if active else "persisted only")
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e: # keep the previous sets on any surprise
|
||||
fail(f"{type(e).__name__}: {e}")
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Refresh the internet scanner sets of the host firewall
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/nft-host-scanners
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily refresh of the host firewall internet scanner sets
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
RandomizedDelaySec=6h
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -43,7 +43,7 @@
|
||||
src: "{{ item }}"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: "0644"
|
||||
loop: [nft-host.service, nft-host-geo.service, nft-host-geo.timer, nft-host-spamhaus.service, nft-host-spamhaus.timer, nft-host-firehol.service, nft-host-firehol.timer]
|
||||
loop: [nft-host.service, nft-host-geo.service, nft-host-geo.timer, nft-host-spamhaus.service, nft-host-spamhaus.timer, nft-host-firehol.service, nft-host-firehol.timer, nft-host-scanners.service, nft-host-scanners.timer]
|
||||
register: firewall_units
|
||||
|
||||
- name: Install the rollback script
|
||||
@@ -211,6 +211,66 @@
|
||||
path: /etc/nftables.d/host-firehol.nft
|
||||
state: absent
|
||||
|
||||
- name: Internet scanner sets
|
||||
when: firewall.scanners
|
||||
block:
|
||||
- name: Install the internet scanner set updater
|
||||
ansible.builtin.copy:
|
||||
src: nft-host-scanners
|
||||
dest: /usr/local/sbin/nft-host-scanners
|
||||
mode: "0755"
|
||||
|
||||
# The updater leaves out entries overlapping these addresses.
|
||||
- name: Install the internet scanner allowlist
|
||||
ansible.builtin.template:
|
||||
src: nft-host-scanners.allowlist.j2
|
||||
dest: /etc/nft-host-scanners.allowlist
|
||||
mode: "0644"
|
||||
|
||||
# The rule set includes this file, so a missing one would fail the load at boot.
|
||||
- name: Build the internet scanner sets on first run
|
||||
ansible.builtin.command: /usr/local/sbin/nft-host-scanners
|
||||
args:
|
||||
creates: /etc/nftables.d/host-scanners.nft
|
||||
|
||||
- name: Read the internet scanner sets file
|
||||
ansible.builtin.stat:
|
||||
path: /etc/nftables.d/host-scanners.nft
|
||||
register: firewall_scanners
|
||||
|
||||
- name: Require non-empty internet scanner sets
|
||||
ansible.builtin.assert:
|
||||
that: firewall_scanners.stat.exists and firewall_scanners.stat.size > 0
|
||||
fail_msg: /etc/nftables.d/host-scanners.nft is missing or empty, refusing to apply the firewall
|
||||
# Check mode neither installs the updater nor builds the sets.
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Enable the internet scanner set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-scanners.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Stop the internet scanner set refresh
|
||||
when: not firewall.scanners
|
||||
block:
|
||||
- name: Disable the internet scanner set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-scanners.timer
|
||||
enabled: false
|
||||
state: stopped
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Remove the internet scanner sets file
|
||||
ansible.builtin.file:
|
||||
path: /etc/nftables.d/host-scanners.nft
|
||||
state: absent
|
||||
|
||||
- name: Render the rule set
|
||||
ansible.builtin.template:
|
||||
src: host.nft.j2
|
||||
|
||||
@@ -38,6 +38,9 @@ include "/etc/nftables.d/host-spamhaus.nft"
|
||||
{% if firewall.firehol %}
|
||||
include "/etc/nftables.d/host-firehol.nft"
|
||||
{% endif %}
|
||||
{% if firewall.scanners %}
|
||||
include "/etc/nftables.d/host-scanners.nft"
|
||||
{% endif %}
|
||||
|
||||
table inet host {
|
||||
{{ addr_set('admin_v4', 'ipv4_addr', firewall.admin.v4) }}
|
||||
@@ -69,6 +72,13 @@ table inet host {
|
||||
|
||||
{% endif %}
|
||||
ct state established,related accept
|
||||
{% if firewall.scanners %}
|
||||
|
||||
# After the established rule: only inbound connections of scanners drop.
|
||||
ip saddr @scanners_v4 jump scanners_in
|
||||
ip6 saddr @scanners_v6 jump scanners_in
|
||||
|
||||
{% endif %}
|
||||
ct state invalid jump drop_log
|
||||
iif lo accept
|
||||
|
||||
@@ -143,6 +153,16 @@ table inet host {
|
||||
reject
|
||||
}
|
||||
{% endif %}
|
||||
{% if firewall.scanners %}
|
||||
|
||||
# Own limit and prefix, so scanners use up neither the fw-in-drop budget
|
||||
# nor the one of the blocklists. CrowdSec whitelists the prefix: scans of
|
||||
# these networks are known and need no decision.
|
||||
chain scanners_in {
|
||||
limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-scanners-in: "
|
||||
drop
|
||||
}
|
||||
{% endif %}
|
||||
{% if firewall.block_smtp_out or blocklists %}
|
||||
|
||||
# Outbound mail goes through the mail host only. Loopback stays open for a
|
||||
@@ -220,7 +240,7 @@ table inet host {
|
||||
ip6 saddr @geo_v6 accept
|
||||
{% endif %}
|
||||
}
|
||||
{% if forward_guard or blocklists %}
|
||||
{% if forward_guard or blocklists or firewall.scanners %}
|
||||
|
||||
chain forward {
|
||||
type filter hook forward priority filter; policy accept;
|
||||
@@ -236,6 +256,11 @@ table inet host {
|
||||
ip saddr @firehol_v4 jump firehol_in
|
||||
ip daddr @firehol_v4 jump firehol_out
|
||||
{% endif %}
|
||||
{% if firewall.scanners %}
|
||||
|
||||
ct state new ip saddr @scanners_v4 jump scanners_in
|
||||
ct state new ip6 saddr @scanners_v6 jump scanners_in
|
||||
{% endif %}
|
||||
{% for guard in forward_guard %}
|
||||
|
||||
# {{ guard.key }}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# {{ ansible_managed }}
|
||||
{% set maps = [
|
||||
(crowdsec | default({}, true)).whitelist | default({}, true),
|
||||
firewall.admin.v4 | default({}, true), firewall.admin.v6 | default({}, true),
|
||||
firewall.scrapers.v4 | default({}, true), firewall.scrapers.v6 | default({}, true),
|
||||
firewall.cluster.v4 | default({}, true), firewall.cluster.v6 | default({}, true)] %}
|
||||
{% set entries = (maps | map('dict2items') | sum(start=[]) | rejectattr('value', 'none') | map(attribute='value') | map('string') | list
|
||||
+ groups['all'] | map('extract', hostvars, 'public_ips') | select('defined') | flatten) | unique %}
|
||||
{% for entry in entries %}
|
||||
{{ entry }}
|
||||
{% endfor %}
|
||||
Reference in New Issue
Block a user