Compare commits
142
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
188183667a | ||
|
|
2b293e9835 | ||
|
|
fee8ec1ff6 | ||
|
|
d61403ee33 | ||
|
|
885e6f3338 | ||
|
|
c91c56b153 | ||
|
|
4dc7b23e06 | ||
|
|
89821b479b | ||
|
|
f91a9b8fc9 | ||
|
|
7bed387ece | ||
|
|
8a0c8fb31b | ||
|
|
4c39069241 | ||
|
|
c50c32059e | ||
|
|
38d1ac6ced | ||
|
|
5160e11dc2 | ||
|
|
c49fd28b76 | ||
|
|
c55a96c0ea | ||
|
|
312be161b9 | ||
|
|
49834b858b | ||
|
|
c36f269406 | ||
|
|
19a62a43ca | ||
|
|
fd58ee2c16 | ||
|
|
d5610a5b34 | ||
|
|
ebb1377470 | ||
|
|
5e544cdfb5 | ||
|
|
88c30eb252 | ||
|
|
41dbce3641 | ||
|
|
81e7a9dc6a | ||
|
|
5cac169224 | ||
|
|
60a55e6e49 | ||
|
|
c5b51777c6 | ||
|
|
a4648e0ea7 | ||
|
|
a64d082c0c | ||
|
|
381e1b285c | ||
|
|
bcdc460297 | ||
|
|
bd24f01110 | ||
|
|
1ede3d3329 | ||
|
|
deb3521bfa | ||
|
|
c8cac22d12 | ||
|
|
6fe03c5f70 | ||
|
|
15b4e54bd5 | ||
|
|
c016459922 | ||
|
|
4d1725f17f | ||
|
|
80658c632f | ||
|
|
60b67f748e | ||
|
|
4a76aa3a2e | ||
|
|
e9f1153482 | ||
|
|
ca1fcc16b6 | ||
|
|
fb94fd5997 | ||
|
|
713d38d77d | ||
|
|
bc71066de1 | ||
|
|
5eae9ca348 | ||
|
|
844e017f9e | ||
|
|
381f3ab7c9 | ||
|
|
6390e6288b | ||
|
|
13d60c4f9b | ||
|
|
cb09f77b68 | ||
|
|
84fd52b678 | ||
|
|
c7eceae087 | ||
|
|
23ed82b113 | ||
|
|
37e21ead36 | ||
|
|
2798853153 | ||
|
|
e9ba272d70 | ||
|
|
b62992f21e | ||
|
|
fd73949412 | ||
|
|
d345ccf585 | ||
|
|
b45a07da56 | ||
|
|
9ded6814c3 | ||
|
|
a9a60e3106 | ||
|
|
5da89fdf29 | ||
|
|
9e2e6cbb99 | ||
|
|
c9d2334906 | ||
|
|
c87af201aa | ||
|
|
cfe87002c5 | ||
|
|
27e9d4a1ab | ||
|
|
755f9b2e1a | ||
|
|
753f456ef3 | ||
|
|
093612f3a7 | ||
|
|
a92409c56f | ||
|
|
f50e3ac33c | ||
|
|
668ff23ee6 | ||
|
|
c2c6a2872f | ||
|
|
c1c7ec9e56 | ||
|
|
550f6868ff | ||
|
|
3fe288f6a5 | ||
|
|
ab9220d042 | ||
|
|
15ad7920d4 | ||
|
|
7012e1ea2b | ||
|
|
13ebd48c5d | ||
|
|
1a76b94a46 | ||
|
|
1b765689e6 | ||
|
|
5b607df2de | ||
|
|
521b76453a | ||
|
|
7f503d983a | ||
|
|
b143d9c848 | ||
|
|
f95bcbc38d | ||
|
|
248a3c08b8 | ||
|
|
f2c86dc22d | ||
|
|
a79f2cac8a | ||
|
|
c8f90f0f8d | ||
|
|
41570ea40d | ||
|
|
a3c887748a | ||
|
|
d113625fa8 | ||
|
|
dadd077723 | ||
|
|
0d43d07ad4 | ||
|
|
d6f8f975bb | ||
|
|
7c86a5d77d | ||
|
|
8c4e3c2401 | ||
|
|
b46d35c8a5 | ||
|
|
791ad96849 | ||
|
|
fc3d9845d6 | ||
|
|
590b75ac23 | ||
|
|
0c82504299 | ||
|
|
2fee9a1747 | ||
|
|
fb44c39969 | ||
|
|
5452303992 | ||
|
|
4321d78cf8 | ||
|
|
f9a859e95c | ||
|
|
fd302e4ebc | ||
|
|
b5729caa0e | ||
|
|
dca40ed835 | ||
|
|
95ddd04a86 | ||
|
|
911bc47acb | ||
|
|
e5920b3ddf | ||
|
|
f3e381aca3 | ||
|
|
233eadaf40 | ||
|
|
e5ebc2ad5f | ||
|
|
98c51c6fc1 | ||
|
|
6b59bf6c75 | ||
|
|
1b2af7cf6c | ||
|
|
d9cf3d2066 | ||
|
|
f42bce9b6b | ||
|
|
0fc5dbb791 | ||
|
|
975746e7d7 | ||
|
|
247fdec7ae | ||
|
|
9cb2e88193 | ||
|
|
25ceb0f456 | ||
|
|
fcc4f1ed18 | ||
|
|
de0e220004 | ||
|
|
f9cc97a8f2 | ||
|
|
811fc22eef | ||
|
|
4cb418e2b6 |
@@ -0,0 +1,4 @@
|
||||
netclient:
|
||||
# Token to join default-network
|
||||
# leave empty to ignore
|
||||
join_network_token:
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
dependencies: []
|
||||
#- role: docker
|
||||
@@ -0,0 +1,4 @@
|
||||
- name: Deploy CA Certificate
|
||||
ansible.builtin.copy:
|
||||
src: secret_files/netmaker_server/ca/ca.crt
|
||||
dest: /etc/ssl/certs/netmaker-ca.pem
|
||||
@@ -0,0 +1,25 @@
|
||||
- name: Install Packages
|
||||
# when: docker_file.stat.exists == False
|
||||
package:
|
||||
name:
|
||||
- gpg
|
||||
- gpg-agent
|
||||
|
||||
- name: Add netmaker-key
|
||||
apt_key:
|
||||
url: https://apt.netmaker.org/gpg.key
|
||||
state: present
|
||||
|
||||
- name: Add netmaker-repository
|
||||
apt_repository:
|
||||
repo: "deb https:apt.netmaker.org stable main"
|
||||
state: present
|
||||
filename: netmaker
|
||||
update_cache: yes
|
||||
|
||||
- name: Install wireguard & netclient
|
||||
package:
|
||||
name:
|
||||
- wireguard
|
||||
- netclient
|
||||
state: latest
|
||||
@@ -0,0 +1,7 @@
|
||||
- name: Join netmaker-network
|
||||
when: "netclient.join_network_token is defined"
|
||||
command: "netclient join -t {{ netclient.join_network_token }}"
|
||||
failed_when: command.rc != 0
|
||||
changed_when: "'starting wireguard' in command.stdout"
|
||||
register: command
|
||||
throttle: 1
|
||||
@@ -0,0 +1,8 @@
|
||||
- import_tasks: ./certs.yml
|
||||
|
||||
- import_tasks: ./install.yml
|
||||
|
||||
- import_tasks: ./join-network.yml
|
||||
|
||||
- name: Gather facts to get changes
|
||||
ansible.builtin.gather_facts:
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
ipv6_stable_secret_seed: "@JW7rQ^P3*f^KRkX3744DZQJb3DfMgZJJMtGc!Jmy!ECBqb^QzAMYLHD8dLvf8m%2bRSRFhgY#SXzAgnxWTvHv#ti6b5PngR^k2gHbQW*%dJ9sdeKM$H77Gymh&xamNN"
|
||||
|
||||
# Bridges and other interfaces that appear after sysctl runs at boot and so miss the accept_ra=2 of
|
||||
# "all"; without it a forwarding host ignores the router advertisements on them.
|
||||
ipv6_accept_ra_interfaces: []
|
||||
|
||||
# Times of the heavy disk jobs, for hosts on a shared HDD pool that keep them out of the backup window.
|
||||
disk_maintenance:
|
||||
# systemd OnCalendar for fstrim.timer; unset keeps the distribution schedule.
|
||||
fstrim_on_calendar:
|
||||
# Cron "minute hour" of the monthly ZFS trim and scrub from zfsutils-linux; unset keeps 00:24.
|
||||
zfs_cron_time:
|
||||
|
||||
# journald.conf [Journal] settings of a drop-in, in this order; null leaves journald untouched.
|
||||
journald_size_cap:
|
||||
SystemMaxUse: 512M
|
||||
RuntimeMaxUse: 128M
|
||||
MaxRetentionSec: 2week
|
||||
|
||||
# Remove fail2ban from the host (only while the CrowdSec agent and bouncer run); set with
|
||||
# -e fail2ban_remove=true once CrowdSec covers the jails of the host.
|
||||
fail2ban_remove: false
|
||||
@@ -1,2 +1,7 @@
|
||||
- name: reload_sysctl
|
||||
command: sysctl --system
|
||||
|
||||
- name: restart_journald
|
||||
systemd:
|
||||
name: systemd-journald
|
||||
state: restarted
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
- name: Ensure .bashrc exists
|
||||
file:
|
||||
path: "{{ ansible_facts.env.HOME }}/.bashrc"
|
||||
state: touch
|
||||
mode: "0644"
|
||||
|
||||
- name: General aliases
|
||||
blockinfile:
|
||||
path: "{{ ansible_facts.env.HOME }}/.bashrc"
|
||||
@@ -8,11 +14,26 @@
|
||||
alias syslog="tail -f --lines=100 /var/log/syslog"
|
||||
alias cp="rsync -hlAXEptgoDS --numeric-ids --info=progress2"
|
||||
|
||||
# We can tell bash to check the next word after the alias (i.e sudo) by adding a space to the end of the alias value. https://askubuntu.com/questions/22037/aliases-not-available-when-using-sudo
|
||||
alias sudo='sudo '
|
||||
|
||||
- name: ls aliases and colors
|
||||
blockinfile:
|
||||
path: "{{ ansible_facts.env.HOME }}/.bashrc"
|
||||
marker: "# {mark} ANSIBLE MANAGED BLOCK | ls aliases and colors"
|
||||
block: |
|
||||
# enable color support of ls and also add handy aliases
|
||||
if [ -x /usr/bin/dircolors ]; then
|
||||
test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
|
||||
alias ls='ls --color=auto'
|
||||
#alias dir='dir --color=auto'
|
||||
#alias vdir='vdir --color=auto'
|
||||
|
||||
alias grep='grep --color=auto'
|
||||
alias fgrep='fgrep --color=auto'
|
||||
alias egrep='egrep --color=auto'
|
||||
fi
|
||||
|
||||
export LS_OPTIONS='--color=auto'
|
||||
eval "`dircolors`"
|
||||
alias ls='ls $LS_OPTIONS'
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
- name: Pin the fstrim timer
|
||||
when: disk_maintenance.fstrim_on_calendar | default(none) is not none
|
||||
block:
|
||||
- name: Create the fstrim timer drop-in directory
|
||||
file:
|
||||
path: /etc/systemd/system/fstrim.timer.d
|
||||
state: directory
|
||||
|
||||
- name: Set the fstrim timer schedule
|
||||
copy:
|
||||
dest: /etc/systemd/system/fstrim.timer.d/schedule.conf
|
||||
content: |
|
||||
[Timer]
|
||||
OnCalendar=
|
||||
OnCalendar={{ disk_maintenance.fstrim_on_calendar }}
|
||||
RandomizedDelaySec=0
|
||||
# A catch-up run after a boot or a timezone change would land at any time.
|
||||
Persistent=false
|
||||
register: fstrim_timer
|
||||
|
||||
- name: Restart the fstrim timer
|
||||
when: fstrim_timer.changed
|
||||
systemd:
|
||||
name: fstrim.timer
|
||||
daemon_reload: yes
|
||||
state: restarted
|
||||
|
||||
- name: Schedule the monthly ZFS trim and scrub
|
||||
when: disk_maintenance.zfs_cron_time | default(none) is not none
|
||||
lineinfile:
|
||||
path: /etc/cron.d/zfsutils-linux
|
||||
regexp: '^\S+ \S+ (\S+ \* \* root .*/usr/lib/zfs-linux/{{ item }};.*)$'
|
||||
line: '{{ disk_maintenance.zfs_cron_time }} \1'
|
||||
backrefs: yes
|
||||
loop: [trim, scrub]
|
||||
@@ -0,0 +1,27 @@
|
||||
# CrowdSec reads the same logs and bans through its bouncers. fail2ban is
|
||||
# removed only on a host switched over on purpose (fail2ban_remove) where both
|
||||
# are running, so a host never ends up without a ban mechanism.
|
||||
- name: Check the CrowdSec services
|
||||
command: systemctl is-active crowdsec crowdsec-firewall-bouncer
|
||||
register: crowdsec_active
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
check_mode: false
|
||||
tags: fail2ban
|
||||
|
||||
- name: Stop and disable fail2ban
|
||||
systemd:
|
||||
name: fail2ban
|
||||
state: stopped
|
||||
enabled: false
|
||||
when: fail2ban_remove and crowdsec_active.rc == 0
|
||||
failed_when: false
|
||||
tags: fail2ban
|
||||
|
||||
- name: Remove fail2ban
|
||||
apt:
|
||||
name: fail2ban
|
||||
state: absent
|
||||
purge: true
|
||||
when: fail2ban_remove and crowdsec_active.rc == 0
|
||||
tags: fail2ban
|
||||
@@ -0,0 +1,17 @@
|
||||
- name: Cap the journal size
|
||||
when: journald_size_cap | default(none) is not none
|
||||
block:
|
||||
- name: Create the journald drop-in directory
|
||||
file:
|
||||
path: /etc/systemd/journald.conf.d
|
||||
state: directory
|
||||
|
||||
- name: Set the journal size cap
|
||||
copy:
|
||||
dest: /etc/systemd/journald.conf.d/10-size-cap.conf
|
||||
content: |
|
||||
[Journal]
|
||||
{% for key, value in journald_size_cap.items() %}
|
||||
{{ key }}={{ value }}
|
||||
{% endfor %}
|
||||
notify: restart_journald
|
||||
Regular → Executable
+11
@@ -1,7 +1,18 @@
|
||||
- import_tasks: ./packages.yml
|
||||
|
||||
- import_tasks: ./fail2ban.yml
|
||||
|
||||
- import_tasks: ./ssh.yml
|
||||
|
||||
- import_tasks: ./packages.yml
|
||||
|
||||
- import_tasks: ./timesync.yml
|
||||
|
||||
- import_tasks: ./aliases.yml
|
||||
|
||||
- import_tasks: ./networking.yml
|
||||
|
||||
- import_tasks: ./disk_maintenance.yml
|
||||
|
||||
- import_tasks: ./journald.yml
|
||||
tags: journald
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
- name: Set sysctl settings for ip-forwarding
|
||||
copy:
|
||||
dest: "/etc/sysctl.d/ip-forwarding.conf"
|
||||
content: |
|
||||
net.ipv4.ip_forward = 1
|
||||
net.ipv6.conf.all.forwarding = 1
|
||||
notify: reload_sysctl
|
||||
|
||||
- name: Check if the sysctl configuration file exists
|
||||
stat:
|
||||
path: "/etc/sysctl.d/ipv6-slaac-address-generation.conf"
|
||||
register: sysctl_conf
|
||||
|
||||
- block:
|
||||
# - name: Generate IPv6 stable-secret segments
|
||||
# set_fact:
|
||||
# stable_secret_segments: "{{ stable_secret_segments | default() + '%0x' | format(range(0, 2**(16*2)) | random(seed=ipv6_stable_secret_seed+inventory_hostname+(stable_secret_segments | default()))) }}"
|
||||
# loop: "{{ range(4) }}"
|
||||
|
||||
# - name: Format IPv6 stable-secret segments
|
||||
# set_fact:
|
||||
# ipv6_stable_secret: "{{ stable_secret_segments | regex_replace('(.{4})(?=.)', '\\1:') }}"
|
||||
|
||||
- name: Generate random IPv6 stable-secret
|
||||
set_fact:
|
||||
ipv6_stable_secret: "{{ 'fd' + (lookup('password', '/dev/null length=30 chars=hex')).split('.')[0][:7] + '::' + (lookup('password', '/dev/null length=4 chars=hex')) }}"
|
||||
|
||||
- name: Set sysctl settings for ipv6-address-generation
|
||||
copy:
|
||||
dest: "/etc/sysctl.d/ipv6-slaac-address-generation.conf"
|
||||
content: |
|
||||
{% if ansible_facts.default_ipv6.interface is defined %}
|
||||
net.ipv6.conf.{{ ansible_facts.default_ipv6.interface }}.addr_gen_mode = 2
|
||||
net.ipv6.conf.{{ ansible_facts.default_ipv6.interface }}.stable_secret = {{ ipv6_stable_secret }}
|
||||
{% else %}
|
||||
net.ipv6.conf.{{ ansible_facts.default_ipv4.interface }}.addr_gen_mode = 2
|
||||
net.ipv6.conf.{{ ansible_facts.default_ipv4.interface }}.stable_secret = {{ ipv6_stable_secret }}
|
||||
{% endif %}
|
||||
notify: reload_sysctl
|
||||
when:
|
||||
- ansible_facts.default_ipv6 is defined
|
||||
- not sysctl_conf.stat.exists
|
||||
|
||||
- name: Set sysctl settings to override ipv6-slaac with enabled forwarding
|
||||
copy:
|
||||
dest: "/etc/sysctl.d/ipv6-slaac-override.conf"
|
||||
content: |
|
||||
net.ipv6.conf.all.accept_ra = 2
|
||||
{% for iface in ipv6_accept_ra_interfaces %}
|
||||
net.ipv6.conf.{{ iface }}.accept_ra = 2
|
||||
{% endfor %}
|
||||
notify: reload_sysctl
|
||||
tags: ipv6_accept_ra
|
||||
|
||||
- name: Set sysctl settings for TCP BBR
|
||||
copy:
|
||||
dest: "/etc/sysctl.d/tcp-bbr.conf"
|
||||
content: |
|
||||
net.core.default_qdisc = fq
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
when:
|
||||
- ansible_kernel is version('4.9', '>=')
|
||||
notify: reload_sysctl
|
||||
@@ -2,7 +2,6 @@
|
||||
apt:
|
||||
update_cache: yes
|
||||
upgrade: yes
|
||||
when: ansible_facts.distribution == "Debian"
|
||||
|
||||
- name: Install Packages
|
||||
package:
|
||||
@@ -11,10 +10,10 @@
|
||||
- htop
|
||||
- iotop
|
||||
- slurm
|
||||
- nmon
|
||||
- sudo
|
||||
- screen
|
||||
- curl
|
||||
- rsync
|
||||
- zstd
|
||||
state: latest
|
||||
when: ansible_facts.distribution == "Debian"
|
||||
|
||||
+10
-2
@@ -1,12 +1,20 @@
|
||||
- name: Disable SSH password auth
|
||||
- name: Check if SSH daemon is active
|
||||
systemd:
|
||||
name: sshd
|
||||
state: started
|
||||
register: sshd_status
|
||||
|
||||
- block:
|
||||
- name: Disable SSH password auth
|
||||
lineinfile:
|
||||
dest: /etc/ssh/sshd_config
|
||||
regexp: '^PasswordAuthentication\s*yes'
|
||||
line: "PasswordAuthentication no"
|
||||
register: sshd_config
|
||||
|
||||
- name: Restart SSH daemon
|
||||
- name: Restart SSH daemon
|
||||
service:
|
||||
name: sshd
|
||||
state: restarted
|
||||
when: sshd_config.changed
|
||||
when: sshd_status.status.ActiveState == 'active'
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Containers share the host kernel clock and cannot discipline it themselves.
|
||||
- name: Configure time synchronisation
|
||||
when: ansible_facts['virtualization_role'] != 'guest' or ansible_facts['virtualization_type'] not in ['lxc', 'docker', 'podman', 'container']
|
||||
block:
|
||||
- name: Install systemd-timesyncd
|
||||
package:
|
||||
name: systemd-timesyncd
|
||||
state: present
|
||||
|
||||
- name: Enable systemd-timesyncd
|
||||
systemd:
|
||||
name: systemd-timesyncd
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
# Logs and local-time timers line up across hosts.
|
||||
- name: Set the timezone to UTC
|
||||
tags: timezone
|
||||
community.general.timezone:
|
||||
name: Etc/UTC
|
||||
@@ -0,0 +1,32 @@
|
||||
---
|
||||
crowdsec:
|
||||
# Central LAPI; hosts run only the agent and the bouncer.
|
||||
lapi_url: ""
|
||||
registration_token: ""
|
||||
bouncer_api_key: ""
|
||||
# Machine name at the LAPI; empty is the short hostname.
|
||||
machine_name: ""
|
||||
# name -> hub collection, merged across group_vars/host_vars (null drops one).
|
||||
collections: {}
|
||||
# name -> hub scenario outside the collections.
|
||||
scenarios: {}
|
||||
# path below /etc/crowdsec -> content, for local parsers and scenarios.
|
||||
local_files: {}
|
||||
# name -> installed hub scenario removed from the agent (null drops one); needed where a local scenario replaces it.
|
||||
disabled_scenarios: {}
|
||||
# name -> path below /etc/crowdsec of a retired local file, removed when present (null drops one).
|
||||
absent_files: {}
|
||||
# name -> CIDR (single addresses as /32 or /128) never acted upon. Joined by
|
||||
# the public_ips (same format) of every inventory host, then whitelisted by
|
||||
# this agent and, when lapi_allowlist is set, allowlisted on the LAPI.
|
||||
whitelist: {}
|
||||
# LAPI allowlist kept equal to the whitelist; it also drops alerts on ranges
|
||||
# holding a whitelisted address, which a parser whitelist cannot.
|
||||
lapi_allowlist: ""
|
||||
# Host whose k3s kubectl reaches the LAPI pod (crowdsec/crowdsec-lapi).
|
||||
lapi_kubectl_host: ""
|
||||
# name -> acquisition datasource, written to /etc/crowdsec/acquis.d/<name>.yaml.
|
||||
acquisition: {}
|
||||
# packagecloud has no trixie suite yet; the bookworm build is static Go.
|
||||
apt_suite_map:
|
||||
trixie: bookworm
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
- name: Restart crowdsec
|
||||
ansible.builtin.systemd:
|
||||
name: crowdsec.service
|
||||
state: restarted
|
||||
|
||||
- name: Restart crowdsec-firewall-bouncer
|
||||
ansible.builtin.systemd:
|
||||
name: crowdsec-firewall-bouncer.service
|
||||
state: restarted
|
||||
@@ -0,0 +1,250 @@
|
||||
---
|
||||
- name: Require LAPI settings
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- crowdsec.lapi_url | length > 0
|
||||
- crowdsec.registration_token | length > 0
|
||||
- crowdsec.bouncer_api_key | length > 0
|
||||
fail_msg: crowdsec.lapi_url, crowdsec.registration_token and crowdsec.bouncer_api_key must be set
|
||||
|
||||
- name: Create /etc/apt/keyrings
|
||||
ansible.builtin.file:
|
||||
path: /etc/apt/keyrings
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Add the CrowdSec apt key
|
||||
ansible.builtin.get_url:
|
||||
url: https://packagecloud.io/crowdsec/crowdsec/gpgkey
|
||||
dest: /etc/apt/keyrings/crowdsec.asc
|
||||
mode: "0644"
|
||||
|
||||
- name: Add the CrowdSec apt repository
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/apt/sources.list.d/crowdsec.list
|
||||
mode: "0644"
|
||||
content: >
|
||||
deb [signed-by=/etc/apt/keyrings/crowdsec.asc]
|
||||
https://packagecloud.io/crowdsec/crowdsec/{{ ansible_distribution | lower }}/
|
||||
{{ crowdsec.apt_suite_map[ansible_distribution_release] | default(ansible_distribution_release) }} main
|
||||
register: crowdsec_repo
|
||||
|
||||
# The postinst asks these; without them it registers a local machine for a
|
||||
# local LAPI and enrols in the central CAPI.
|
||||
- name: Preseed the package for a remote LAPI
|
||||
ansible.builtin.debconf:
|
||||
name: crowdsec
|
||||
question: "{{ item.question }}"
|
||||
vtype: "{{ item.vtype }}"
|
||||
value: "{{ item.value }}"
|
||||
loop:
|
||||
- { question: crowdsec/lapi, vtype: boolean, value: "false" }
|
||||
- { question: crowdsec/capi, vtype: boolean, value: "false" }
|
||||
- { question: crowdsec/lapi_host, vtype: string, value: "{{ crowdsec.lapi_url | urlsplit('netloc') }}" }
|
||||
|
||||
- name: Create the CrowdSec config directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- /etc/crowdsec
|
||||
- /etc/crowdsec/acquis.d
|
||||
- /etc/crowdsec/bouncers
|
||||
- /etc/crowdsec/parsers/s02-enrich
|
||||
- /etc/systemd/system/crowdsec.service.d
|
||||
- /etc/systemd/system/crowdsec-firewall-bouncer.service.d
|
||||
- /etc/systemd/system/nftables.service.d
|
||||
|
||||
- name: Disable the local API server
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/crowdsec/config.yaml.local
|
||||
mode: "0644"
|
||||
content: |
|
||||
api:
|
||||
server:
|
||||
enable: false
|
||||
notify: Restart crowdsec
|
||||
|
||||
# The postinst starts the agent before it is registered; an empty credentials
|
||||
# file then makes the start a no-op instead of a failed package install.
|
||||
- name: Start the agent only once registered
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/crowdsec.service.d/registered.conf
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Unit]
|
||||
ConditionFileNotEmpty=/etc/crowdsec/local_api_credentials.yaml
|
||||
register: crowdsec_dropin
|
||||
|
||||
# Start, reload and stop of nftables.service flush the whole ruleset with the
|
||||
# bouncer tables; a restart recreates them with all decisions. Ordering after
|
||||
# nftables.service makes the queued restarts wait for its job to finish.
|
||||
- name: Restart the bouncer after nftables.service flushes the ruleset
|
||||
ansible.builtin.copy:
|
||||
dest: "{{ item.dest }}"
|
||||
mode: "0644"
|
||||
content: "{{ item.content }}"
|
||||
loop:
|
||||
- dest: /etc/systemd/system/nftables.service.d/crowdsec-firewall-bouncer.conf
|
||||
content: |
|
||||
[Service]
|
||||
ExecStartPost=-/bin/systemctl --no-block try-restart crowdsec-firewall-bouncer.service
|
||||
ExecReload=-/bin/systemctl --no-block try-restart crowdsec-firewall-bouncer.service
|
||||
ExecStopPost=-/bin/systemctl --no-block try-restart crowdsec-firewall-bouncer.service
|
||||
- dest: /etc/systemd/system/crowdsec-firewall-bouncer.service.d/nftables.conf
|
||||
content: |
|
||||
[Unit]
|
||||
After=nftables.service
|
||||
loop_control:
|
||||
label: "{{ item.dest }}"
|
||||
register: crowdsec_bouncer_dropins
|
||||
|
||||
# Present before the package install, so the postinst finds an API key and
|
||||
# does not try to create one on a local LAPI.
|
||||
- name: Configure the firewall bouncer
|
||||
ansible.builtin.template:
|
||||
src: crowdsec-firewall-bouncer.yaml.local.j2
|
||||
dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml.local
|
||||
mode: "0600"
|
||||
notify: Restart crowdsec-firewall-bouncer
|
||||
|
||||
- name: Install the parser whitelist
|
||||
ansible.builtin.template:
|
||||
src: whitelist.yaml.j2
|
||||
dest: /etc/crowdsec/parsers/s02-enrich/host-whitelist.yaml
|
||||
mode: "0644"
|
||||
notify: Restart crowdsec
|
||||
|
||||
- name: Install the acquisition files
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/crowdsec/acquis.d/{{ item.key }}.yaml"
|
||||
mode: "0644"
|
||||
content: "{{ item.value | to_nice_yaml }}"
|
||||
loop: "{{ crowdsec.acquisition | dict2items | rejectattr('value', 'none') }}"
|
||||
notify: Restart crowdsec
|
||||
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
when: crowdsec_dropin is changed or crowdsec_bouncer_dropins is changed
|
||||
|
||||
- name: Install the agent and the nftables bouncer
|
||||
ansible.builtin.apt:
|
||||
name: [crowdsec, crowdsec-firewall-bouncer-nftables]
|
||||
state: present
|
||||
update_cache: "{{ crowdsec_repo is changed }}"
|
||||
# The postinst otherwise runs service detection, which needs credentials that only the registration below creates
|
||||
environment:
|
||||
CROWDSEC_SETUP_UNATTENDED_DISABLE: "1"
|
||||
|
||||
- name: Read the agent credentials
|
||||
ansible.builtin.command: grep -q '^login:' /etc/crowdsec/local_api_credentials.yaml
|
||||
register: crowdsec_registered
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Register the agent at the LAPI
|
||||
ansible.builtin.command: >-
|
||||
cscli lapi register --url {{ crowdsec.lapi_url }}
|
||||
--token {{ crowdsec.registration_token }} --machine {{ crowdsec.machine_name | default(ansible_hostname, true) }}
|
||||
when: crowdsec_registered.rc != 0
|
||||
no_log: true
|
||||
notify: Restart crowdsec
|
||||
|
||||
- name: List the installed hub items
|
||||
ansible.builtin.command: cscli hub list -o json
|
||||
register: crowdsec_hub_json
|
||||
changed_when: false
|
||||
# cscli is missing in check mode before the first install.
|
||||
failed_when: crowdsec_hub_json.rc != 0 and not ansible_check_mode
|
||||
check_mode: false
|
||||
|
||||
- name: Install the hub items
|
||||
ansible.builtin.command: "cscli {{ item }} install {{ crowdsec_missing | join(' ') }}"
|
||||
vars:
|
||||
crowdsec_wanted: "{{ crowdsec[item] | dict2items | rejectattr('value', 'none') | map(attribute='value') | unique }}"
|
||||
crowdsec_present: "{{ (crowdsec_hub_json.stdout | default('{}', true) | from_json)[item] | default([], true) | map(attribute='name') }}"
|
||||
crowdsec_missing: "{{ crowdsec_wanted | difference(crowdsec_present) }}"
|
||||
loop: [collections, scenarios]
|
||||
when: crowdsec_missing | length > 0
|
||||
notify: Restart crowdsec
|
||||
|
||||
# Read again: the collections installed above bring their scenarios along.
|
||||
- name: List the installed hub scenarios
|
||||
ansible.builtin.command: cscli hub list -o json
|
||||
register: crowdsec_scenarios_json
|
||||
changed_when: false
|
||||
failed_when: crowdsec_scenarios_json.rc != 0 and not ansible_check_mode
|
||||
check_mode: false
|
||||
|
||||
# --force also removes items that belong to an installed collection. A plain "cscli hub upgrade" keeps them removed;
|
||||
# "cscli hub upgrade --force" would bring them back.
|
||||
- name: Remove the hub scenarios replaced by local ones
|
||||
ansible.builtin.command: "cscli scenarios remove {{ item.value }} --force"
|
||||
loop: "{{ crowdsec.disabled_scenarios | dict2items | rejectattr('value', 'none') }}"
|
||||
loop_control:
|
||||
label: "{{ item.value }}"
|
||||
when: item.value in ((crowdsec_scenarios_json.stdout | default('{}', true) | from_json).scenarios | default([], true) | map(attribute='name'))
|
||||
notify: Restart crowdsec
|
||||
|
||||
- name: Install the local parsers and scenarios
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/crowdsec/{{ item.key }}"
|
||||
mode: "0644"
|
||||
content: "{{ item.value }}"
|
||||
loop: "{{ crowdsec.local_files | dict2items | rejectattr('value', 'none') }}"
|
||||
loop_control:
|
||||
label: "{{ item.key }}"
|
||||
notify: Restart crowdsec
|
||||
|
||||
- name: Remove retired local files
|
||||
ansible.builtin.file:
|
||||
path: "/etc/crowdsec/{{ item.value }}"
|
||||
state: absent
|
||||
loop: "{{ crowdsec.absent_files | dict2items | rejectattr('value', 'none') }}"
|
||||
loop_control:
|
||||
label: "{{ item.value }}"
|
||||
notify: Restart crowdsec
|
||||
|
||||
- name: Enable the agent and the bouncer
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ item }}"
|
||||
enabled: true
|
||||
state: started
|
||||
loop: [crowdsec.service, crowdsec-firewall-bouncer.service]
|
||||
|
||||
- name: Sync the LAPI allowlist
|
||||
when: crowdsec.lapi_allowlist | length > 0
|
||||
run_once: true
|
||||
delegate_to: "{{ crowdsec.lapi_kubectl_host }}"
|
||||
vars:
|
||||
crowdsec_lapi_cscli: k3s kubectl -n crowdsec exec deploy/crowdsec-lapi -c crowdsec-lapi -- cscli
|
||||
block:
|
||||
- name: Read the LAPI allowlist
|
||||
ansible.builtin.command: "{{ crowdsec_lapi_cscli }} allowlists inspect {{ crowdsec.lapi_allowlist }} -o json"
|
||||
register: crowdsec_allowlist_json
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Create the LAPI allowlist
|
||||
ansible.builtin.command: >-
|
||||
{{ crowdsec_lapi_cscli }} allowlists create {{ crowdsec.lapi_allowlist }}
|
||||
-d "Internal ranges and the public addresses of all hosts, managed by Ansible"
|
||||
when: crowdsec_allowlist_json.rc != 0
|
||||
|
||||
# cscli stores single addresses without their /32 or /128.
|
||||
- name: Update the LAPI allowlist
|
||||
ansible.builtin.command: "{{ crowdsec_lapi_cscli }} allowlists {{ item.action }} {{ crowdsec.lapi_allowlist }} {{ item.cidrs | join(' ') }}"
|
||||
vars:
|
||||
crowdsec_allowlist_wanted: "{{ crowdsec_whitelist | map('regex_replace', '/(32|128)$', '') | list }}"
|
||||
crowdsec_allowlist_present: >-
|
||||
{{ ((crowdsec_allowlist_json.stdout | default('{}', true) | from_json).get('items') or []) | map(attribute='value') | list }}
|
||||
loop:
|
||||
- { action: add, cidrs: "{{ crowdsec_allowlist_wanted | difference(crowdsec_allowlist_present) }}" }
|
||||
- { action: remove, cidrs: "{{ crowdsec_allowlist_present | difference(crowdsec_allowlist_wanted) }}" }
|
||||
loop_control:
|
||||
label: "{{ item.action }} {{ item.cidrs | join(' ') }}"
|
||||
when: item.cidrs | length > 0
|
||||
@@ -0,0 +1,8 @@
|
||||
# {{ ansible_managed }}
|
||||
api_url: {{ crowdsec.lapi_url }}/
|
||||
api_key: {{ crowdsec.bouncer_api_key }}
|
||||
mode: nftables
|
||||
# forward covers Docker-published ports and Proxmox guests.
|
||||
nftables_hooks:
|
||||
- input
|
||||
- forward
|
||||
@@ -0,0 +1,9 @@
|
||||
# {{ ansible_managed }}
|
||||
name: local/host-whitelist
|
||||
description: Internal ranges and the public addresses of all hosts
|
||||
whitelist:
|
||||
reason: internal or admin source
|
||||
cidr:
|
||||
{% for cidr in crowdsec_whitelist %}
|
||||
- "{{ cidr }}"
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
crowdsec_whitelist: >-
|
||||
{{ (crowdsec.whitelist | dict2items | rejectattr('value', 'none') | map(attribute='value') | list
|
||||
+ groups['all'] | map('extract', hostvars, 'public_ips') | select('defined') | flatten) | unique }}
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Mirror the CrowdSec range ban decisions into the nftables table inet crowdsec_ranges
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/crowdsec-range-bouncer
|
||||
TimeoutStartSec=60
|
||||
StateDirectory=crowdsec-range-bouncer
|
||||
StateDirectoryMode=0700
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
NoNewPrivileges=true
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Mirror the CrowdSec range ban decisions into nftables every minute
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
- name: Run the range bouncer
|
||||
ansible.builtin.systemd:
|
||||
name: crowdsec-range-bouncer.service
|
||||
state: started
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
@@ -0,0 +1,53 @@
|
||||
---
|
||||
# crowdsec-firewall-bouncer in nftables mode stores only the network address of
|
||||
# a range decision (CIDR scope "Range") in a plain address set. This role keeps
|
||||
# the ranges in interval sets of its own table, next to the firewall bouncer,
|
||||
# which keeps handling the single addresses.
|
||||
- name: Require LAPI settings
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- crowdsec.lapi_url | length > 0
|
||||
- crowdsec.range_bouncer_api_key | default('') | length > 0
|
||||
fail_msg: crowdsec.lapi_url and crowdsec.range_bouncer_api_key must be set
|
||||
|
||||
- name: Install python3 and nftables
|
||||
ansible.builtin.apt:
|
||||
name: [python3, nftables]
|
||||
state: present
|
||||
|
||||
- name: Configure the range bouncer
|
||||
ansible.builtin.template:
|
||||
src: crowdsec-range-bouncer.json.j2
|
||||
dest: /etc/crowdsec-range-bouncer.json
|
||||
mode: "0600"
|
||||
no_log: true
|
||||
notify: Run the range bouncer
|
||||
|
||||
- name: Install the range bouncer
|
||||
ansible.builtin.template:
|
||||
src: crowdsec-range-bouncer.py.j2
|
||||
dest: /usr/local/sbin/crowdsec-range-bouncer
|
||||
mode: "0755"
|
||||
notify: Run the range bouncer
|
||||
|
||||
- name: Install the systemd units
|
||||
ansible.builtin.copy:
|
||||
src: "{{ item }}"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: "0644"
|
||||
loop: [crowdsec-range-bouncer.service, crowdsec-range-bouncer.timer]
|
||||
register: crowdsec_range_bouncer_units
|
||||
notify: Run the range bouncer
|
||||
|
||||
- name: Enable the range bouncer timer
|
||||
ansible.builtin.systemd:
|
||||
name: crowdsec-range-bouncer.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: "{{ crowdsec_range_bouncer_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
# The first run has to succeed before the play moves on to the next host.
|
||||
- name: Apply the changes now
|
||||
ansible.builtin.meta: flush_handlers
|
||||
@@ -0,0 +1 @@
|
||||
{{ {'lapi_url': crowdsec.lapi_url, 'api_key': crowdsec.range_bouncer_api_key, 'allow': crowdsec_range_bouncer_allow} | to_nice_json }}
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/python3
|
||||
# {{ ansible_managed }}
|
||||
"""Mirror the CrowdSec range ban decisions into the nftables table inet crowdsec_ranges.
|
||||
|
||||
Stateless: every run fetches all range decisions from the LAPI, builds the
|
||||
whole table and loads it in one transaction. The table is only touched when
|
||||
its text changed or the table is gone (nftables.service flushes the whole
|
||||
ruleset on reload). Any error leaves the current table in place.
|
||||
"""
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
CONFIG = "/etc/crowdsec-range-bouncer.json"
|
||||
STATE_DIR = os.environ.get("STATE_DIRECTORY", "/var/lib/crowdsec-range-bouncer")
|
||||
HASH_FILE = os.path.join(STATE_DIR, "last.sha256")
|
||||
NFT = "/usr/sbin/nft"
|
||||
TABLE = "crowdsec_ranges"
|
||||
|
||||
|
||||
def fetch_ranges(lapi_url, api_key):
|
||||
"""Return the networks of all ban decisions with scope Range."""
|
||||
request = urllib.request.Request(
|
||||
lapi_url.rstrip("/") + "/v1/decisions?scopes=range",
|
||||
headers={"X-Api-Key": api_key, "User-Agent": "crowdsec-range-bouncer"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=20) as response:
|
||||
decisions = json.load(response)
|
||||
# The LAPI answers a literal null when nothing matches.
|
||||
if decisions is None:
|
||||
return []
|
||||
if not isinstance(decisions, list):
|
||||
raise ValueError("unexpected LAPI answer: %s" % type(decisions).__name__)
|
||||
return [
|
||||
ipaddress.ip_network(d["value"], strict=False)
|
||||
for d in decisions
|
||||
if d["type"] == "ban" and d["scope"].lower() == "range"
|
||||
]
|
||||
|
||||
|
||||
def collapse(networks, version):
|
||||
"""Merge overlapping and adjacent networks: interval set elements must not overlap."""
|
||||
return [str(n) for n in ipaddress.collapse_addresses(n for n in networks if n.version == version)]
|
||||
|
||||
|
||||
def set_text(name, address_type, elements):
|
||||
text = "\tset %s {\n\t\ttype %s\n\t\tflags interval\n" % (name, address_type)
|
||||
if elements:
|
||||
text += "\t\telements = {\n\t\t\t%s\n\t\t}\n" % ",\n\t\t\t".join(elements)
|
||||
return text + "\t}\n"
|
||||
|
||||
|
||||
def table_text(allow, block):
|
||||
# The accept of an allowed address only ends the chain of this table:
|
||||
# host_firewall, Docker, Cilium and the CrowdSec firewall bouncer still decide.
|
||||
# Same hooks and priority as the firewall bouncer: input, and forward for
|
||||
# published Docker ports and guests.
|
||||
rules = (
|
||||
"\t\tip saddr @allow4 accept\n"
|
||||
"\t\tip6 saddr @allow6 accept\n"
|
||||
"\t\tip saddr @block4 counter drop\n"
|
||||
"\t\tip6 saddr @block6 counter drop\n"
|
||||
)
|
||||
return (
|
||||
"table inet %(t)s {}\n"
|
||||
"delete table inet %(t)s\n"
|
||||
"table inet %(t)s {\n"
|
||||
% {"t": TABLE}
|
||||
+ set_text("allow4", "ipv4_addr", collapse(allow, 4))
|
||||
+ set_text("allow6", "ipv6_addr", collapse(allow, 6))
|
||||
+ set_text("block4", "ipv4_addr", collapse(block, 4))
|
||||
+ set_text("block6", "ipv6_addr", collapse(block, 6))
|
||||
+ "\tchain input {\n\t\ttype filter hook input priority -10; policy accept;\n"
|
||||
+ "\t\tiifname \"lo\" accept\n"
|
||||
+ rules
|
||||
+ "\t}\n"
|
||||
+ "\tchain forward {\n\t\ttype filter hook forward priority -10; policy accept;\n"
|
||||
+ rules
|
||||
+ "\t}\n}\n"
|
||||
)
|
||||
|
||||
|
||||
def table_exists():
|
||||
result = subprocess.run(
|
||||
[NFT, "list", "table", "inet", TABLE], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def main():
|
||||
with open(CONFIG) as f:
|
||||
config = json.load(f)
|
||||
allow = [ipaddress.ip_network(v, strict=False) for v in config["allow"]]
|
||||
block = fetch_ranges(config["lapi_url"], config["api_key"])
|
||||
text = table_text(allow, block)
|
||||
digest = hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
try:
|
||||
with open(HASH_FILE) as f:
|
||||
last = f.read().strip()
|
||||
except FileNotFoundError:
|
||||
last = ""
|
||||
if digest == last and table_exists():
|
||||
return
|
||||
|
||||
subprocess.run([NFT, "-f", "-"], input=text.encode(), check=True)
|
||||
tmp = HASH_FILE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
f.write(digest + "\n")
|
||||
os.replace(tmp, HASH_FILE)
|
||||
print("loaded %d IPv4 and %d IPv6 ranges from %d decisions" % (
|
||||
len(collapse(block, 4)), len(collapse(block, 6)), len(block)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as e:
|
||||
print("error, keeping the current table: %s: %s" % (type(e).__name__, e), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
# Same sources as the crowdsec role (roles/crowdsec/vars/main.yml), plus loopback.
|
||||
crowdsec_range_bouncer_allow: >-
|
||||
{{ (crowdsec.whitelist | default({}) | dict2items | rejectattr('value', 'none') | map(attribute='value') | list
|
||||
+ groups['all'] | map('extract', hostvars, 'public_ips') | select('defined') | flatten
|
||||
+ ['127.0.0.0/8', '::1/128']) | unique }}
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
# Silent on success: cron mails all output to root, so mail arrives only when the prune fails.
|
||||
set -u
|
||||
|
||||
output=$(docker system prune -af --filter "until=168h" 2>&1)
|
||||
rc=$?
|
||||
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
printf '%s\n' "$output"
|
||||
echo "docker system prune exited with rc=$rc"
|
||||
exit "$rc"
|
||||
fi
|
||||
+67
-7
@@ -9,18 +9,45 @@
|
||||
name:
|
||||
- gpg
|
||||
- gpg-agent
|
||||
- python3-debian
|
||||
tags: docker_repo
|
||||
|
||||
- name: Create apt keyrings directory
|
||||
file:
|
||||
path: /etc/apt/keyrings
|
||||
state: directory
|
||||
mode: "0755"
|
||||
tags: docker_repo
|
||||
|
||||
- name: Add docker-key
|
||||
apt_key:
|
||||
url: https://download.docker.com/linux/debian/gpg
|
||||
state: present
|
||||
get_url:
|
||||
url: "https://download.docker.com/linux/{{ ansible_facts.distribution | lower }}/gpg"
|
||||
dest: /etc/apt/keyrings/docker.asc
|
||||
mode: "0644"
|
||||
tags: docker_repo
|
||||
|
||||
- name: Remove one-line docker-repository
|
||||
file:
|
||||
path: /etc/apt/sources.list.d/docker.list
|
||||
state: absent
|
||||
tags: docker_repo
|
||||
|
||||
- name: Add docker-repository
|
||||
apt_repository:
|
||||
repo: "deb https://download.docker.com/linux/{{ ansible_facts.distribution | lower }} {{ ansible_facts.distribution_release }} stable"
|
||||
state: present
|
||||
filename: docker
|
||||
deb822_repository:
|
||||
name: docker
|
||||
types: deb
|
||||
uris: "https://download.docker.com/linux/{{ ansible_facts.distribution | lower }}"
|
||||
suites: "{{ ansible_facts.distribution_release }}"
|
||||
components: stable
|
||||
signed_by: /etc/apt/keyrings/docker.asc
|
||||
register: docker_repo
|
||||
tags: docker_repo
|
||||
|
||||
- name: Update apt cache after docker-repository change
|
||||
apt:
|
||||
update_cache: yes
|
||||
when: docker_repo.changed
|
||||
tags: docker_repo
|
||||
|
||||
- name: Install docker
|
||||
package:
|
||||
@@ -38,3 +65,36 @@
|
||||
dest: "/usr/local/bin/docker-compose"
|
||||
mode: preserve
|
||||
|
||||
- name: Check for docker binary
|
||||
ansible.builtin.stat:
|
||||
path: /usr/bin/docker
|
||||
register: docker_binary
|
||||
tags: docker_prune
|
||||
|
||||
- name: Deploy docker-prune script
|
||||
ansible.builtin.copy:
|
||||
src: docker-prune
|
||||
dest: /usr/local/sbin/docker-prune
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
when: docker_binary.stat.exists
|
||||
tags: docker_prune
|
||||
|
||||
- name: Schedule hourly docker-prune
|
||||
ansible.builtin.cron:
|
||||
name: docker prune
|
||||
cron_file: docker-prune
|
||||
user: root
|
||||
minute: "0"
|
||||
job: /usr/local/sbin/docker-prune
|
||||
when: docker_binary.stat.exists
|
||||
tags: docker_prune
|
||||
|
||||
- name: Remove hand-written docker system prune lines from root crontab
|
||||
ansible.builtin.lineinfile:
|
||||
path: /var/spool/cron/crontabs/root
|
||||
regexp: '^\S+ \S+ \S+ \S+ \S+ docker system prune -af'
|
||||
state: absent
|
||||
tags: docker_prune
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
---
|
||||
# Adds prometheus-smartctl-exporter (trixie-backports) next to the node exporter
|
||||
# on hosts with physical disks.
|
||||
host_exporters_smart: false
|
||||
|
||||
# Adds ipmitool and prometheus-ipmi-exporter (port 9290) on hosts with a BMC reachable through /dev/ipmi0.
|
||||
host_exporters_ipmi: false
|
||||
host_exporters_ipmi_collectors:
|
||||
- ipmi
|
||||
- bmc
|
||||
- sel
|
||||
- chassis
|
||||
# The tools run as root and parse the FreeIPMI configuration file named by --config-file. The exporter
|
||||
# writes that file itself (a named pipe in its /tmp, empty for in-band local mode), so a compromised
|
||||
# exporter could set options such as sdr-cache-directory and make root write anywhere. The wrappers
|
||||
# therefore replace the path with this static root-owned file, and sudoers admits only that path.
|
||||
host_exporters_ipmi_freeipmi_conf: /etc/ipmi-exporter/freeipmi.conf
|
||||
|
||||
# FreeIPMI tool behind each collector and the fixed arguments the exporter (1.10) passes before
|
||||
# --config-file. sudo admits exactly these, so a compromised exporter cannot run chassis control or
|
||||
# clear the SEL. A different exporter version needs matching arguments (ipmi_up stays 0 until then).
|
||||
# dcmi is not listed: the S1200RP BMC has no power monitoring.
|
||||
host_exporters_ipmi_tools:
|
||||
ipmi:
|
||||
tool: ipmimonitoring
|
||||
args: --quiet-cache --ignore-unrecognized-events --comma-separated-output --no-header-output --sdr-cache-recreate --output-event-bitmask --output-sensor-state
|
||||
bmc:
|
||||
tool: bmc-info
|
||||
args: ""
|
||||
sel:
|
||||
tool: ipmi-sel
|
||||
args: --info
|
||||
chassis:
|
||||
tool: ipmi-chassis
|
||||
args: --get-chassis-status
|
||||
@@ -0,0 +1,190 @@
|
||||
---
|
||||
# Both exporters listen on all addresses; host_firewall limits who reaches them.
|
||||
# Its recommends (smartmontools, nvme-cli, ipmitool) start units that fail on hosts without such hardware.
|
||||
- name: Install the node exporter
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-node-exporter
|
||||
install_recommends: false
|
||||
state: present
|
||||
|
||||
- name: Enable the node exporter
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus-node-exporter
|
||||
enabled: true
|
||||
state: started
|
||||
# Check mode does not install the package.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Install the smartctl exporter
|
||||
when: host_exporters_smart
|
||||
block:
|
||||
- name: Look for an existing backports source
|
||||
ansible.builtin.shell: >-
|
||||
grep -rEqs '^[^#]*{{ ansible_distribution_release }}-backports'
|
||||
--include='*.list' --include='*.sources' --exclude=host-exporters-backports.sources /etc/apt
|
||||
register: host_exporters_backports
|
||||
changed_when: false
|
||||
failed_when: host_exporters_backports.rc > 1
|
||||
check_mode: false
|
||||
|
||||
# Backports stay at priority 100, so only packages named with default_release come from there.
|
||||
- name: Add the backports source
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/apt/sources.list.d/host-exporters-backports.sources
|
||||
mode: "0644"
|
||||
content: |
|
||||
Types: deb
|
||||
URIs: http://deb.debian.org/debian
|
||||
Suites: {{ ansible_distribution_release }}-backports
|
||||
Components: main
|
||||
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
|
||||
register: host_exporters_source
|
||||
when: host_exporters_backports.rc == 1
|
||||
|
||||
# The module rejects default_release while the release is missing from its package cache.
|
||||
- name: Look for the backports release in the package lists
|
||||
ansible.builtin.shell: apt-cache policy | grep -q 'n={{ ansible_distribution_release }}-backports'
|
||||
register: host_exporters_lists
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Refresh the package lists for the backports source
|
||||
ansible.builtin.apt:
|
||||
update_cache: true
|
||||
when: host_exporters_lists.rc != 0
|
||||
|
||||
- name: Install the smartctl exporter from backports
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-smartctl-exporter
|
||||
default_release: "{{ ansible_distribution_release }}-backports"
|
||||
state: present
|
||||
# Check mode does not write the backports source.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
# The packaged unit runs as root, which smartctl needs for the disks.
|
||||
- name: Enable the smartctl exporter
|
||||
ansible.builtin.systemd:
|
||||
name: smartctl_exporter
|
||||
enabled: true
|
||||
state: started
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
# Reads the BMC through /dev/ipmi0, read-only: no raw commands, no configuration.
|
||||
# FreeIPMI refuses in-band access for any uid but 0, so the exporter runs as the packaged
|
||||
# unprivileged user prometheus and starts the FreeIPMI tools it needs through sudo.
|
||||
- name: Install the IPMI exporter
|
||||
when: host_exporters_ipmi
|
||||
block:
|
||||
- name: Install ipmitool and the IPMI exporter
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- ipmitool
|
||||
- sudo
|
||||
- prometheus-ipmi-exporter
|
||||
install_recommends: false
|
||||
state: present
|
||||
|
||||
- name: Allow the exporter user to run the FreeIPMI tools
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/sudoers.d/prometheus-ipmi-exporter
|
||||
mode: "0440"
|
||||
validate: visudo -cf %s
|
||||
content: |
|
||||
Defaults:prometheus !pam_session
|
||||
{% for collector in host_exporters_ipmi_collectors %}
|
||||
prometheus ALL = (root) NOPASSWD: /usr/sbin/{{ host_exporters_ipmi_tools[collector].tool }} {{ host_exporters_ipmi_tools[collector].args }} --config-file {{ host_exporters_ipmi_freeipmi_conf }}
|
||||
{% endfor %}
|
||||
register: host_exporters_ipmi_sudo
|
||||
|
||||
- name: Create the FreeIPMI configuration directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ host_exporters_ipmi_freeipmi_conf | dirname }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
# See the defaults for why the file is static.
|
||||
- name: Write the FreeIPMI configuration
|
||||
ansible.builtin.copy:
|
||||
dest: "{{ host_exporters_ipmi_freeipmi_conf }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
# In-band access needs no settings. Not even sdr-cache-directory: the tools without an SDR cache
|
||||
# reject it, and the cache of the ipmi tool stays in the home of root.
|
||||
|
||||
# collector_cmd replaces the tool itself, so each collector gets a wrapper that adds sudo. It swaps
|
||||
# the exporter's --config-file for the static one. The exporter fills a named pipe at the path it
|
||||
# passes and waits for a reader; the wrapper drains it so that writer does not hang.
|
||||
- name: Create the sudo wrappers directory
|
||||
ansible.builtin.file:
|
||||
path: /usr/local/libexec/ipmi-exporter
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Write the sudo wrappers
|
||||
ansible.builtin.copy:
|
||||
dest: /usr/local/libexec/ipmi-exporter/{{ item.key }}
|
||||
mode: "0755"
|
||||
content: |
|
||||
#!/bin/sh
|
||||
n=$#
|
||||
while [ "$n" -gt 0 ]; do
|
||||
a=$1; shift; n=$((n - 1))
|
||||
if [ "$a" = --config-file ] && [ "$n" -gt 0 ]; then
|
||||
timeout 5 cat "$1" > /dev/null 2>&1
|
||||
shift; n=$((n - 1))
|
||||
else
|
||||
set -- "$@" "$a"
|
||||
fi
|
||||
done
|
||||
exec sudo -n /usr/sbin/{{ item.value.tool }} "$@" --config-file {{ host_exporters_ipmi_freeipmi_conf }}
|
||||
loop: "{{ host_exporters_ipmi_tools | dict2items | selectattr('key', 'in', host_exporters_ipmi_collectors) | list }}"
|
||||
loop_control:
|
||||
label: "{{ item.key }}"
|
||||
register: host_exporters_ipmi_wrappers
|
||||
|
||||
- name: Write the IPMI exporter configuration
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/prometheus/ipmi-local.yml
|
||||
mode: "0644"
|
||||
content: |
|
||||
modules:
|
||||
default:
|
||||
collectors:
|
||||
{% for collector in host_exporters_ipmi_collectors %}
|
||||
- {{ collector }}
|
||||
{% endfor %}
|
||||
collector_cmd:
|
||||
{% for collector in host_exporters_ipmi_collectors %}
|
||||
{{ collector }}: /usr/local/libexec/ipmi-exporter/{{ collector }}
|
||||
{% endfor %}
|
||||
register: host_exporters_ipmi_config
|
||||
|
||||
- name: Create the IPMI exporter unit drop-in directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/systemd/system/prometheus-ipmi-exporter.service.d
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
# sudo needs setuid, so NoNewPrivileges stays off.
|
||||
- name: Run the IPMI exporter with that configuration
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/prometheus-ipmi-exporter.service.d/local.conf
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/prometheus-ipmi-exporter --config.file=/etc/prometheus/ipmi-local.yml
|
||||
register: host_exporters_ipmi_unit
|
||||
|
||||
- name: Enable the IPMI exporter
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus-ipmi-exporter
|
||||
enabled: true
|
||||
state: "{{ 'restarted' if (host_exporters_ipmi_config.changed or host_exporters_ipmi_unit.changed or host_exporters_ipmi_wrappers.changed) else 'started' }}"
|
||||
daemon_reload: true
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
@@ -0,0 +1,86 @@
|
||||
---
|
||||
# Port and address maps are name -> value so group_vars and host_vars levels
|
||||
# merge into one map (hash_behaviour = merge); a null value drops an entry.
|
||||
# Ports are numbers or "first-last" ranges.
|
||||
firewall:
|
||||
public:
|
||||
tcp: {}
|
||||
udp: {}
|
||||
# Accepted only from the geo sets or the admin set.
|
||||
geo:
|
||||
tcp: {}
|
||||
# Accepted on wt0 before the public zone is evaluated.
|
||||
vpn:
|
||||
tcp: {}
|
||||
udp: {}
|
||||
# tcp: ports accepted from the admin set alone, never from the geo sets.
|
||||
admin:
|
||||
tcp: {}
|
||||
v4: {}
|
||||
v6: {}
|
||||
# Metrics scrapers: on wt0 only these source addresses reach the listed tcp
|
||||
# ports (the vpn zone applies to every peer). Rendered only when both are set.
|
||||
scrapers:
|
||||
tcp: {}
|
||||
v4: {}
|
||||
v6: {}
|
||||
# Interface names (nft wildcards allowed) whose traffic is trusted.
|
||||
internal_ifaces: {}
|
||||
# Guest bridges (Proxmox vmbr*): DHCP, router solicitations and the public
|
||||
# zone only, never the geo ports.
|
||||
guest_ifaces: {}
|
||||
# Kubernetes ring: pod interfaces, and node NetBird IPs / pod CIDRs on wt0.
|
||||
cluster:
|
||||
ifaces: {}
|
||||
v4: {}
|
||||
v6: {}
|
||||
# IPv4 port forwards: name -> {daddr, to, tcp: ports, udp: ports}, ports as in
|
||||
# the maps above. Forwarded traffic passes the forward hook, not input.
|
||||
dnat: {}
|
||||
# IPv4 source NAT: name -> nft match, e.g. 'oifname "vmbr0"'.
|
||||
masquerade: {}
|
||||
# Forwarded traffic to guarded guests: name -> {ifaces, v4, v6}, where v4 and
|
||||
# v6 are {daddr, tcp: ports, icmp: icmp or icmpv6 types}. Replies and traffic
|
||||
# in on ifaces pass, the listed ports and types pass, the rest to daddr drops;
|
||||
# traffic to other addresses is not filtered.
|
||||
forward_guard: {}
|
||||
ssh_rate_limit: "10/minute burst 5 packets"
|
||||
# Kernel log of dropped inbound packets (fw-in-drop: ) and of rejected
|
||||
# outbound mail (fw-out-reject: ), read from the journal by CrowdSec. Each
|
||||
# rate is one limit shared by all matching packets.
|
||||
log:
|
||||
in_rate: "5/second burst 30 packets"
|
||||
out_rate: "6/minute burst 5 packets"
|
||||
# Lets the kernel log the nft log statements of container network
|
||||
# namespaces (LXC guests) into the host kernel log.
|
||||
all_netns: false
|
||||
# Shared by the Spamhaus (fw-spamhaus-in: , fw-spamhaus-out: ), FireHOL (fw-firehol-in: , fw-firehol-out: ) and scanner (fw-scanners-in: ) chains, one limit each.
|
||||
spamhaus_rate: "6/minute burst 10 packets"
|
||||
# Spamhaus DROP, DROPv6 and ASN-DROP: traffic from the listed networks drops
|
||||
# (input, forward), traffic to them is rejected (output, forward). The sets
|
||||
# are refreshed daily by nft-host-spamhaus.
|
||||
spamhaus: true
|
||||
# FireHOL level1 (IPv4): treated like the Spamhaus lists, with the prefixes
|
||||
# fw-firehol-in: and fw-firehol-out: and the same log limit. Entries in
|
||||
# private or reserved space and ones holding an address of the host are left
|
||||
# out. Refreshed daily by nft-host-firehol.
|
||||
firehol: true
|
||||
# Internet-wide scanners (Censys, Shodan, ONYPHE, Shadowserver, ...), IPv4
|
||||
# and IPv6: the cidr files of github.com/OpenFilters/internet-scanners
|
||||
# (Unlicense). Inbound traffic from the listed networks drops (input,
|
||||
# forward) once the established rule has passed; outbound traffic is never
|
||||
# filtered. Log prefix fw-scanners-in: with the Spamhaus log limit, which
|
||||
# CrowdSec whitelists. Entries in private or reserved space, overlapping the
|
||||
# admin, scraper and cluster sets, the CrowdSec whitelist or the public_ips
|
||||
# of any host, and ones holding an address of the host are left out
|
||||
# (/etc/nft-host-scanners.allowlist). Refreshed daily by nft-host-scanners.
|
||||
scanners: true
|
||||
# Reject outbound TCP 25; the mail group turns it off.
|
||||
block_smtp_out: true
|
||||
# EU-27 plus CH, NO, GB, IS, LI (ipdeny country codes). Empty leaves the geo
|
||||
# ports to the admin set; a load peaks at about 60 MB of kernel memory, too much for a 1 GB host.
|
||||
geo_countries: [at, be, bg, hr, cy, cz, dk, ee, fi, fr, de, gr, hu, ie, it, lv, lt, lu, mt, nl, pl, pt, ro, sk, si, es, se, ch, "no", gb, is, li]
|
||||
rollback_seconds: 120
|
||||
# name -> shell command that has to exit 0 through the new rule set before the
|
||||
# rollback is disarmed; retried for a quarter of the rollback window.
|
||||
confirm: {}
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/python3
|
||||
# Managed by Ansible (role host_firewall), do not edit.
|
||||
"""Rebuilds the FireHOL level1 set firehol_v4 of table inet host.
|
||||
|
||||
A failed download or sanity check exits non-zero before anything changes, so
|
||||
the previous set stays in place.
|
||||
"""
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
PERSISTED = "/etc/nftables.d/host-firehol.nft"
|
||||
URL = "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"
|
||||
USER_AGENT = "ruekov-host-firewall (nft-host-firehol)"
|
||||
|
||||
# Lower bound that catches an empty or truncated answer.
|
||||
MIN_ENTRIES = 1000
|
||||
# A prefix this wide is a parse error, not a bad network.
|
||||
MIN_PREFIXLEN = 8
|
||||
# Private, CGNAT (NetBird), loopback, link-local and multicast/reserved space:
|
||||
# an entry overlapping one of these never lands in the set, so the own
|
||||
# infrastructure cannot block itself.
|
||||
EXCLUDED = [ipaddress.ip_network(n) for n in (
|
||||
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8",
|
||||
"169.254.0.0/16", "172.16.0.0/12", "192.168.0.0/16", "224.0.0.0/3")]
|
||||
|
||||
|
||||
def fail(message):
|
||||
print(f"nft-host-firehol: {message}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def fetch():
|
||||
error = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
request = urllib.request.Request(URL, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
return response.read().decode("utf-8")
|
||||
except OSError as e:
|
||||
error = e
|
||||
time.sleep(5 * (attempt + 1))
|
||||
fail(f"download of {URL} failed: {error}")
|
||||
|
||||
|
||||
def own_addresses():
|
||||
result = subprocess.run(["ip", "-j", "-4", "address", "show"], capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
fail(f"ip address show failed: {result.stderr.strip()}")
|
||||
return [ipaddress.ip_address(a["local"]) for link in json.loads(result.stdout)
|
||||
for a in link.get("addr_info", []) if a.get("family") == "inet"]
|
||||
|
||||
|
||||
def main():
|
||||
networks = []
|
||||
for line in fetch().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
networks.append(ipaddress.IPv4Network(line))
|
||||
except ValueError:
|
||||
fail(f"unparsable line: {line!r}")
|
||||
if len(networks) < MIN_ENTRIES:
|
||||
fail(f"only {len(networks)} entries")
|
||||
|
||||
own = own_addresses()
|
||||
kept, excluded, own_skipped = [], 0, 0
|
||||
for network in networks:
|
||||
if any(network.overlaps(e) for e in EXCLUDED):
|
||||
excluded += 1
|
||||
elif any(address in network for address in own):
|
||||
own_skipped += 1
|
||||
else:
|
||||
if network.prefixlen < MIN_PREFIXLEN:
|
||||
fail(f"suspiciously wide prefix {network}")
|
||||
kept.append(network)
|
||||
merged = list(ipaddress.collapse_addresses(kept))
|
||||
print(f"firehol_v4: {len(networks)} entries, {excluded} in private or reserved space and "
|
||||
f"{own_skipped} holding an own address skipped -> {len(merged)} merged")
|
||||
|
||||
elements = ",\n\t\t".join(str(n) for n in merged) + ","
|
||||
persisted = ("# Generated by nft-host-firehol, do not edit.\n"
|
||||
"table inet host {\n"
|
||||
"\tset firehol_v4 { type ipv4_addr; flags interval; auto-merge; elements = {\n"
|
||||
f"\t\t{elements}\n\t}} }}\n}}\n")
|
||||
|
||||
# Checked under another table name: a dry run against the live table
|
||||
# name changes the live sets for its duration and unlisted addresses match.
|
||||
check_rules = persisted.replace("table inet host {", "table inet host_firehol_check {")
|
||||
checked = subprocess.run(["nft", "-c", "-f", "-"], input=check_rules, capture_output=True, text=True)
|
||||
if checked.returncode != 0:
|
||||
fail(f"nft rejects the set: {checked.stderr.strip()}")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
previous = None
|
||||
if os.path.exists(PERSISTED):
|
||||
previous = os.path.join(tmp, "previous.nft")
|
||||
shutil.copyfile(PERSISTED, previous)
|
||||
with open(PERSISTED + ".new", "w", encoding="utf-8") as f:
|
||||
f.write(persisted)
|
||||
os.chmod(PERSISTED + ".new", 0o644)
|
||||
os.replace(PERSISTED + ".new", PERSISTED)
|
||||
|
||||
# A reload replaces the whole table, sets included, in one transaction.
|
||||
active = subprocess.run(["systemctl", "-q", "is-active", "nft-host.service"]).returncode == 0
|
||||
if active:
|
||||
reloaded = subprocess.run(["systemctl", "reload", "nft-host.service"], capture_output=True, text=True)
|
||||
if reloaded.returncode != 0:
|
||||
if previous:
|
||||
shutil.copyfile(previous, PERSISTED)
|
||||
fail(f"reloading the rule set failed, kept the previous set: {reloaded.stderr.strip()}")
|
||||
print("reloaded the rule set" if active else "persisted only")
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e: # keep the previous set on any surprise
|
||||
fail(f"{type(e).__name__}: {e}")
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Refresh the FireHOL set of the host firewall
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/nft-host-firehol
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily refresh of the host firewall FireHOL set
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
RandomizedDelaySec=6h
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Refresh the geo sets of the host firewall
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/nft-host-geo
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily refresh of the host firewall geo sets
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
RandomizedDelaySec=6h
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Started by a transient timer before a rule set load; Ansible stops the timer
|
||||
# once the new rule set is confirmed.
|
||||
cd /etc/nftables.d || exit 1
|
||||
mv -f host.nft host.nft.rolled-back
|
||||
if [ -s host.nft.good ]; then
|
||||
cp host.nft.good host.nft && nft -f host.nft && exit 0
|
||||
fi
|
||||
nft delete table inet host
|
||||
Executable
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/python3
|
||||
# Managed by Ansible (role host_firewall), do not edit.
|
||||
"""Rebuilds the internet scanner sets scanners_v4 and scanners_v6 of table inet host.
|
||||
|
||||
Source: the cidr directory of OpenFilters/internet-scanners (Unlicense), read
|
||||
from the tarball of the default branch. A failed download or sanity check
|
||||
exits non-zero before anything changes, so the previous sets stay in place.
|
||||
"""
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
PERSISTED = "/etc/nftables.d/host-scanners.nft"
|
||||
# One CIDR or address per line: the public addresses of the hosts, the
|
||||
# CrowdSec whitelist and the admin, scraper and cluster sets of the firewall.
|
||||
ALLOWLIST = "/etc/nft-host-scanners.allowlist"
|
||||
URL = "https://codeload.github.com/OpenFilters/internet-scanners/tar.gz/refs/heads/main"
|
||||
USER_AGENT = "ruekov-host-firewall (nft-host-scanners)"
|
||||
MEMBER = re.compile(r"^[^/]+/cidr/[a-z0-9]+_v([46])\.txt$")
|
||||
MAX_BYTES = 50 * 1024 * 1024
|
||||
|
||||
# Lower bounds that catch an empty or truncated answer (the list holds about
|
||||
# 21000 IPv4 and 3000 IPv6 entries).
|
||||
MIN_ENTRIES = {4: 5000, 6: 500}
|
||||
# A prefix this wide is a parse error, not a scanner network.
|
||||
MIN_PREFIXLEN = {4: 12, 6: 24}
|
||||
# Private, CGNAT (NetBird), loopback, link-local, ULA and multicast/reserved
|
||||
# space: an entry overlapping one of these never lands in a set, so the own
|
||||
# infrastructure cannot block itself.
|
||||
EXCLUDED = [ipaddress.ip_network(n) for n in (
|
||||
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8",
|
||||
"169.254.0.0/16", "172.16.0.0/12", "192.168.0.0/16", "224.0.0.0/3",
|
||||
"::/128", "::1/128", "::ffff:0:0/96", "fc00::/7", "fe80::/10", "ff00::/8")]
|
||||
|
||||
|
||||
def fail(message):
|
||||
print(f"nft-host-scanners: {message}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def fetch():
|
||||
error = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
request = urllib.request.Request(URL, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
data = response.read(MAX_BYTES + 1)
|
||||
if len(data) > MAX_BYTES:
|
||||
fail(f"answer from {URL} exceeds {MAX_BYTES} bytes")
|
||||
return data
|
||||
except OSError as e:
|
||||
error = e
|
||||
time.sleep(5 * (attempt + 1))
|
||||
fail(f"download of {URL} failed: {error}")
|
||||
|
||||
|
||||
def parse(data):
|
||||
"""Returns {4: [networks], 6: [networks]} from the cidr/*_v4.txt and *_v6.txt files."""
|
||||
networks = {4: [], 6: []}
|
||||
try:
|
||||
archive = tarfile.open(fileobj=io.BytesIO(data), mode="r:gz")
|
||||
except (tarfile.TarError, OSError) as e:
|
||||
fail(f"unreadable archive: {e}")
|
||||
with archive:
|
||||
for member in archive:
|
||||
match = MEMBER.match(member.name)
|
||||
if not match or not member.isfile():
|
||||
continue
|
||||
version = int(match.group(1))
|
||||
for line in archive.extractfile(member).read().decode("utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
# Host bits set (34.19.127.176/27) are masked, not rejected.
|
||||
network = ipaddress.ip_network(line, strict=False)
|
||||
except ValueError:
|
||||
fail(f"unparsable line in {member.name}: {line!r}")
|
||||
if network.version != version:
|
||||
fail(f"IPv{network.version} entry in {member.name}: {line!r}")
|
||||
networks[version].append(network)
|
||||
return networks
|
||||
|
||||
|
||||
def read_allowlist():
|
||||
entries = []
|
||||
with open(ALLOWLIST, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
entries.append(ipaddress.ip_network(line, strict=False))
|
||||
except ValueError:
|
||||
fail(f"unparsable line in {ALLOWLIST}: {line!r}")
|
||||
return entries
|
||||
|
||||
|
||||
def own_addresses():
|
||||
result = subprocess.run(["ip", "-j", "address", "show"], capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
fail(f"ip address show failed: {result.stderr.strip()}")
|
||||
return [ipaddress.ip_address(a["local"]) for link in json.loads(result.stdout)
|
||||
for a in link.get("addr_info", []) if a.get("family") in ("inet", "inet6")]
|
||||
|
||||
|
||||
def main():
|
||||
listed = parse(fetch())
|
||||
for version, minimum in MIN_ENTRIES.items():
|
||||
if len(listed[version]) < minimum:
|
||||
fail(f"only {len(listed[version])} IPv{version} entries")
|
||||
|
||||
protected = EXCLUDED + read_allowlist()
|
||||
own = own_addresses()
|
||||
merged = {}
|
||||
for version in (4, 6):
|
||||
kept, protected_skipped, own_skipped = [], 0, 0
|
||||
for network in listed[version]:
|
||||
if network.prefixlen < MIN_PREFIXLEN[version]:
|
||||
fail(f"suspiciously wide prefix {network}")
|
||||
if any(p.version == version and network.overlaps(p) for p in protected):
|
||||
protected_skipped += 1
|
||||
elif any(a.version == version and a in network for a in own):
|
||||
own_skipped += 1
|
||||
else:
|
||||
kept.append(network)
|
||||
merged[version] = list(ipaddress.collapse_addresses(kept))
|
||||
print(f"scanners_v{version}: {len(listed[version])} entries, {protected_skipped} in private, reserved or "
|
||||
f"allowlisted space and {own_skipped} holding an own address skipped -> {len(merged[version])} merged")
|
||||
|
||||
persisted = "# Generated by nft-host-scanners, do not edit.\ntable inet host {\n"
|
||||
for version in (4, 6):
|
||||
elements = ",\n\t\t".join(str(n) for n in merged[version]) + ","
|
||||
persisted += (f"\tset scanners_v{version} {{ type ipv{version}_addr; flags interval; auto-merge; elements = {{\n"
|
||||
f"\t\t{elements}\n\t}} }}\n")
|
||||
persisted += "}\n"
|
||||
|
||||
# Checked under another table name: a dry run against the live table
|
||||
# name changes the live sets for its duration and unlisted addresses match.
|
||||
check_rules = persisted.replace("table inet host {", "table inet host_scanners_check {")
|
||||
checked = subprocess.run(["nft", "-c", "-f", "-"], input=check_rules, capture_output=True, text=True)
|
||||
if checked.returncode != 0:
|
||||
fail(f"nft rejects the sets: {checked.stderr.strip()}")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
previous = None
|
||||
if os.path.exists(PERSISTED):
|
||||
previous = os.path.join(tmp, "previous.nft")
|
||||
shutil.copyfile(PERSISTED, previous)
|
||||
with open(PERSISTED + ".new", "w", encoding="utf-8") as f:
|
||||
f.write(persisted)
|
||||
os.chmod(PERSISTED + ".new", 0o644)
|
||||
os.replace(PERSISTED + ".new", PERSISTED)
|
||||
|
||||
# A reload replaces the whole table, sets included, in one transaction.
|
||||
active = subprocess.run(["systemctl", "-q", "is-active", "nft-host.service"]).returncode == 0
|
||||
if active:
|
||||
reloaded = subprocess.run(["systemctl", "reload", "nft-host.service"], capture_output=True, text=True)
|
||||
if reloaded.returncode != 0:
|
||||
if previous:
|
||||
shutil.copyfile(previous, PERSISTED)
|
||||
else:
|
||||
os.remove(PERSISTED)
|
||||
fail(f"reloading the rule set failed, kept the previous sets: {reloaded.stderr.strip()}")
|
||||
print("reloaded the rule set" if active else "persisted only")
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e: # keep the previous sets on any surprise
|
||||
fail(f"{type(e).__name__}: {e}")
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Refresh the internet scanner sets of the host firewall
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/nft-host-scanners
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily refresh of the host firewall internet scanner sets
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
RandomizedDelaySec=6h
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/python3
|
||||
# Managed by Ansible (role host_firewall), do not edit.
|
||||
"""Rebuilds the Spamhaus sets spamhaus_v4 and spamhaus_v6 of table inet host.
|
||||
|
||||
Sources are the Spamhaus DROP and DROPv6 prefix lists and the ASN-DROP list,
|
||||
whose ASNs are expanded to prefixes with the iptoasn bulk files (one download
|
||||
per address family). Any failed download or sanity check exits non-zero before
|
||||
anything changes, so the previous sets stay in place.
|
||||
"""
|
||||
import gzip
|
||||
import http.client
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
PERSISTED = "/etc/nftables.d/host-spamhaus.nft"
|
||||
SPAMHAUS = "https://www.spamhaus.org/drop/"
|
||||
IPTOASN = "https://iptoasn.com/data/"
|
||||
USER_AGENT = "ruekov-host-firewall (nft-host-spamhaus)"
|
||||
|
||||
# Lower bounds that catch an empty or truncated answer.
|
||||
MIN_RECORDS = {"drop_v4": 800, "drop_v6": 30, "asndrop": 200}
|
||||
MIN_TSV_LINES = {4: 200000, 6: 20000}
|
||||
MIN_ASN_PREFIXES_V4 = 100
|
||||
# A prefix this wide is a parse error, not a bad network.
|
||||
MIN_PREFIXLEN = {4: 8, 6: 16}
|
||||
|
||||
|
||||
def fail(message):
|
||||
print(f"nft-host-spamhaus: {message}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def fetch(url, dest):
|
||||
error = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(request, timeout=120) as response, open(dest, "wb") as out:
|
||||
shutil.copyfileobj(response, out)
|
||||
return
|
||||
except (OSError, http.client.HTTPException) as e:
|
||||
error = e
|
||||
time.sleep(5 * (attempt + 1))
|
||||
fail(f"download of {url} failed: {error}")
|
||||
|
||||
|
||||
def spamhaus_records(name, tmp):
|
||||
path = os.path.join(tmp, name + ".json")
|
||||
fetch(f"{SPAMHAUS}{name}.json", path)
|
||||
records, meta = [], None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
record = json.loads(line)
|
||||
if record.get("type") == "metadata":
|
||||
meta = record
|
||||
else:
|
||||
records.append(record)
|
||||
if meta is None or meta.get("records") != len(records):
|
||||
fail(f"{name}: record count does not match the metadata record")
|
||||
if len(records) < MIN_RECORDS[name]:
|
||||
fail(f"{name}: only {len(records)} records")
|
||||
return records
|
||||
|
||||
|
||||
def asn_prefixes(asns, family, tmp):
|
||||
path = os.path.join(tmp, f"ip2asn-v{family}.tsv.gz")
|
||||
fetch(f"{IPTOASN}ip2asn-v{family}.tsv.gz", path)
|
||||
networks, lines = [], 0
|
||||
with gzip.open(path, "rt", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
lines += 1
|
||||
fields = line.split("\t", 3)
|
||||
if len(fields) < 3 or fields[2] not in asns:
|
||||
continue
|
||||
networks.extend(ipaddress.summarize_address_range(
|
||||
ipaddress.ip_address(fields[0]), ipaddress.ip_address(fields[1])))
|
||||
if lines < MIN_TSV_LINES[family]:
|
||||
fail(f"ip2asn-v{family}: only {lines} lines")
|
||||
return networks
|
||||
|
||||
|
||||
def merge(family, *groups):
|
||||
networks, skipped = [], 0
|
||||
for group in groups:
|
||||
for network in group:
|
||||
network = ipaddress.ip_network(network)
|
||||
if network.prefixlen < MIN_PREFIXLEN[family]:
|
||||
fail(f"suspiciously wide prefix {network}")
|
||||
if not network.is_global:
|
||||
skipped += 1
|
||||
continue
|
||||
networks.append(network)
|
||||
return list(ipaddress.collapse_addresses(networks)), skipped
|
||||
|
||||
|
||||
def main():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
drop4 = [r["cidr"] for r in spamhaus_records("drop_v4", tmp)]
|
||||
drop6 = [r["cidr"] for r in spamhaus_records("drop_v6", tmp)]
|
||||
asns = {str(r["asn"]) for r in spamhaus_records("asndrop", tmp)}
|
||||
asn4 = asn_prefixes(asns, 4, tmp)
|
||||
asn6 = asn_prefixes(asns, 6, tmp)
|
||||
if len(asn4) < MIN_ASN_PREFIXES_V4:
|
||||
fail(f"only {len(asn4)} IPv4 prefixes for {len(asns)} ASNs")
|
||||
sets = {}
|
||||
for family, name, drop, asn in ((4, "spamhaus_v4", drop4, asn4), (6, "spamhaus_v6", drop6, asn6)):
|
||||
sets[name], skipped = merge(family, drop, asn)
|
||||
print(f"{name}: {len(drop)} DROP + {len(asn)} ASN-DROP prefixes -> "
|
||||
f"{len(sets[name])} merged, {skipped} non-global skipped")
|
||||
print(f"ASN-DROP: {len(asns)} ASNs")
|
||||
|
||||
def elements(networks):
|
||||
return ",\n\t\t".join(str(n) for n in networks) + ","
|
||||
|
||||
persisted = ["# Generated by nft-host-spamhaus, do not edit.", "table inet host {"]
|
||||
for name, kind in (("spamhaus_v4", "ipv4_addr"), ("spamhaus_v6", "ipv6_addr")):
|
||||
persisted.append(f"\tset {name} {{ type {kind}; flags interval; auto-merge; elements = {{\n"
|
||||
f"\t\t{elements(sets[name])}\n\t}} }}")
|
||||
persisted.append("}\n")
|
||||
|
||||
candidate = os.path.join(tmp, "host-spamhaus.nft")
|
||||
with open(candidate, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(persisted))
|
||||
# Checked under another table name: a dry run against the live table
|
||||
# name changes the live sets for its duration and unlisted addresses match.
|
||||
with open(candidate, encoding="utf-8") as f:
|
||||
check_rules = f.read().replace("table inet host {", "table inet host_spamhaus_check {")
|
||||
checked = subprocess.run(["nft", "-c", "-f", "-"], input=check_rules, capture_output=True, text=True)
|
||||
if checked.returncode != 0:
|
||||
fail(f"nft rejects the sets: {checked.stderr.strip()}")
|
||||
|
||||
previous = None
|
||||
if os.path.exists(PERSISTED):
|
||||
previous = os.path.join(tmp, "previous.nft")
|
||||
shutil.copyfile(PERSISTED, previous)
|
||||
shutil.copyfile(candidate, PERSISTED + ".new")
|
||||
os.chmod(PERSISTED + ".new", 0o644)
|
||||
os.replace(PERSISTED + ".new", PERSISTED)
|
||||
|
||||
# A reload replaces the whole table, sets included, in one transaction.
|
||||
# Flushing and refilling a running interval set instead lets packets of
|
||||
# unlisted addresses match while the transaction commits.
|
||||
active = subprocess.run(["systemctl", "-q", "is-active", "nft-host.service"]).returncode == 0
|
||||
if active:
|
||||
reloaded = subprocess.run(["systemctl", "reload", "nft-host.service"], capture_output=True, text=True)
|
||||
if reloaded.returncode != 0:
|
||||
if previous:
|
||||
shutil.copyfile(previous, PERSISTED)
|
||||
fail(f"reloading the rule set failed, kept the previous sets: {reloaded.stderr.strip()}")
|
||||
print("reloaded the rule set" if active else "persisted only")
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e: # keep the previous sets on any surprise
|
||||
fail(f"{type(e).__name__}: {e}")
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Refresh the Spamhaus sets of the host firewall
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/nft-host-spamhaus
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily refresh of the host firewall Spamhaus sets
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
RandomizedDelaySec=6h
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,22 @@
|
||||
[Unit]
|
||||
Description=Host input firewall and NAT (nftables tables inet host and ip host_nat)
|
||||
# Same early-boot placement as nftables.service. Its "flush ruleset" on start,
|
||||
# reload and stop would wipe this table, so load after it and follow all three.
|
||||
DefaultDependencies=no
|
||||
Wants=network-pre.target
|
||||
Before=network-pre.target shutdown.target
|
||||
After=nftables.service
|
||||
PartOf=nftables.service
|
||||
ReloadPropagatedFrom=nftables.service
|
||||
Conflicts=shutdown.target
|
||||
ConditionPathExists=/etc/nftables.d/host.nft
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/sbin/nft -f /etc/nftables.d/host.nft
|
||||
ExecReload=/usr/sbin/nft -f /etc/nftables.d/host.nft
|
||||
ExecStop=-/usr/sbin/nft delete table inet host
|
||||
|
||||
[Install]
|
||||
WantedBy=sysinit.target
|
||||
@@ -0,0 +1,347 @@
|
||||
---
|
||||
- name: Install nftables and curl
|
||||
ansible.builtin.apt:
|
||||
name: [nftables, curl]
|
||||
state: present
|
||||
|
||||
- name: Check that this node is part of its own cluster ring
|
||||
ansible.builtin.assert:
|
||||
that: ansible_wt0.ipv4.address in (firewall.cluster.v4 | dict2items | map(attribute='value') | list)
|
||||
fail_msg: "wt0 address {{ ansible_wt0.ipv4.address }} is missing from firewall.cluster.v4"
|
||||
when: firewall.cluster.v4 | default({}, true) | length > 0 and ansible_wt0.ipv4.address is defined
|
||||
|
||||
- name: Remove the include block of the old nftables role
|
||||
ansible.builtin.blockinfile:
|
||||
path: /etc/nftables.conf
|
||||
marker: "# {mark} ANSIBLE MANAGED BLOCK - nftables"
|
||||
state: absent
|
||||
|
||||
- name: Remove the rule files of the old nftables role
|
||||
ansible.builtin.file:
|
||||
path: /etc/nftables/ansible-managed
|
||||
state: absent
|
||||
|
||||
- name: Create /etc/nftables.d
|
||||
ansible.builtin.file:
|
||||
path: /etc/nftables.d
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
# The kernel logs nft log statements of other network namespaces (LXC guests)
|
||||
# only with this sysctl; the lines land in the host kernel log.
|
||||
- name: Log the firewalls of container network namespaces
|
||||
ansible.posix.sysctl:
|
||||
name: net.netfilter.nf_log_all_netns
|
||||
value: "1"
|
||||
sysctl_file: /etc/sysctl.d/90-nf-log-all-netns.conf
|
||||
sysctl_set: true
|
||||
reload: false
|
||||
when: firewall.log.all_netns
|
||||
|
||||
- name: Install the systemd units
|
||||
ansible.builtin.copy:
|
||||
src: "{{ item }}"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: "0644"
|
||||
loop: [nft-host.service, nft-host-geo.service, nft-host-geo.timer, nft-host-spamhaus.service, nft-host-spamhaus.timer, nft-host-firehol.service, nft-host-firehol.timer, nft-host-scanners.service, nft-host-scanners.timer]
|
||||
register: firewall_units
|
||||
|
||||
- name: Install the rollback script
|
||||
ansible.builtin.copy:
|
||||
src: nft-host-rollback
|
||||
dest: /usr/local/sbin/nft-host-rollback
|
||||
mode: "0755"
|
||||
|
||||
- name: Geo sets
|
||||
when: firewall.geo_countries | length > 0
|
||||
block:
|
||||
- name: Install the geo set updater
|
||||
ansible.builtin.template:
|
||||
src: nft-host-geo.j2
|
||||
dest: /usr/local/sbin/nft-host-geo
|
||||
mode: "0755"
|
||||
|
||||
- name: Build the geo sets on first run
|
||||
ansible.builtin.command: /usr/local/sbin/nft-host-geo
|
||||
args:
|
||||
creates: /etc/nftables.d/host-geo.nft
|
||||
|
||||
- name: Read the geo sets file
|
||||
ansible.builtin.stat:
|
||||
path: /etc/nftables.d/host-geo.nft
|
||||
register: firewall_geo
|
||||
|
||||
# Without geo sets the public SSH port would only answer NetBird and the admin set.
|
||||
- name: Require non-empty geo sets
|
||||
ansible.builtin.assert:
|
||||
that: firewall_geo.stat.exists and firewall_geo.stat.size > 0
|
||||
fail_msg: /etc/nftables.d/host-geo.nft is missing or empty, refusing to apply the firewall
|
||||
# Check mode neither installs the updater nor builds the sets.
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Enable the geo set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-geo.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Stop the geo set refresh
|
||||
when: firewall.geo_countries | length == 0
|
||||
block:
|
||||
- name: Disable the geo set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-geo.timer
|
||||
enabled: false
|
||||
state: stopped
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Remove the geo sets file
|
||||
ansible.builtin.file:
|
||||
path: /etc/nftables.d/host-geo.nft
|
||||
state: absent
|
||||
|
||||
- name: Spamhaus sets
|
||||
when: firewall.spamhaus
|
||||
block:
|
||||
- name: Install the Spamhaus set updater
|
||||
ansible.builtin.copy:
|
||||
src: nft-host-spamhaus
|
||||
dest: /usr/local/sbin/nft-host-spamhaus
|
||||
mode: "0755"
|
||||
|
||||
# The rule set includes this file, so a missing one would fail the load at boot.
|
||||
- name: Build the Spamhaus sets on first run
|
||||
ansible.builtin.command: /usr/local/sbin/nft-host-spamhaus
|
||||
args:
|
||||
creates: /etc/nftables.d/host-spamhaus.nft
|
||||
|
||||
- name: Read the Spamhaus sets file
|
||||
ansible.builtin.stat:
|
||||
path: /etc/nftables.d/host-spamhaus.nft
|
||||
register: firewall_spamhaus
|
||||
|
||||
- name: Require non-empty Spamhaus sets
|
||||
ansible.builtin.assert:
|
||||
that: firewall_spamhaus.stat.exists and firewall_spamhaus.stat.size > 0
|
||||
fail_msg: /etc/nftables.d/host-spamhaus.nft is missing or empty, refusing to apply the firewall
|
||||
# Check mode neither installs the updater nor builds the sets.
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Enable the Spamhaus set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-spamhaus.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Stop the Spamhaus set refresh
|
||||
when: not firewall.spamhaus
|
||||
block:
|
||||
- name: Disable the Spamhaus set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-spamhaus.timer
|
||||
enabled: false
|
||||
state: stopped
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Remove the Spamhaus sets file
|
||||
ansible.builtin.file:
|
||||
path: /etc/nftables.d/host-spamhaus.nft
|
||||
state: absent
|
||||
|
||||
- name: FireHOL set
|
||||
when: firewall.firehol
|
||||
block:
|
||||
- name: Install the FireHOL set updater
|
||||
ansible.builtin.copy:
|
||||
src: nft-host-firehol
|
||||
dest: /usr/local/sbin/nft-host-firehol
|
||||
mode: "0755"
|
||||
|
||||
# The rule set includes this file, so a missing one would fail the load at boot.
|
||||
- name: Build the FireHOL set on first run
|
||||
ansible.builtin.command: /usr/local/sbin/nft-host-firehol
|
||||
args:
|
||||
creates: /etc/nftables.d/host-firehol.nft
|
||||
|
||||
- name: Read the FireHOL set file
|
||||
ansible.builtin.stat:
|
||||
path: /etc/nftables.d/host-firehol.nft
|
||||
register: firewall_firehol
|
||||
|
||||
- name: Require non-empty FireHOL set
|
||||
ansible.builtin.assert:
|
||||
that: firewall_firehol.stat.exists and firewall_firehol.stat.size > 0
|
||||
fail_msg: /etc/nftables.d/host-firehol.nft is missing or empty, refusing to apply the firewall
|
||||
# Check mode neither installs the updater nor builds the sets.
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Enable the FireHOL set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-firehol.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Stop the FireHOL set refresh
|
||||
when: not firewall.firehol
|
||||
block:
|
||||
- name: Disable the FireHOL set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-firehol.timer
|
||||
enabled: false
|
||||
state: stopped
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Remove the FireHOL set file
|
||||
ansible.builtin.file:
|
||||
path: /etc/nftables.d/host-firehol.nft
|
||||
state: absent
|
||||
|
||||
- name: Internet scanner sets
|
||||
when: firewall.scanners
|
||||
block:
|
||||
- name: Install the internet scanner set updater
|
||||
ansible.builtin.copy:
|
||||
src: nft-host-scanners
|
||||
dest: /usr/local/sbin/nft-host-scanners
|
||||
mode: "0755"
|
||||
|
||||
# The updater leaves out entries overlapping these addresses.
|
||||
- name: Install the internet scanner allowlist
|
||||
ansible.builtin.template:
|
||||
src: nft-host-scanners.allowlist.j2
|
||||
dest: /etc/nft-host-scanners.allowlist
|
||||
mode: "0644"
|
||||
|
||||
# The rule set includes this file, so a missing one would fail the load at boot.
|
||||
- name: Build the internet scanner sets on first run
|
||||
ansible.builtin.command: /usr/local/sbin/nft-host-scanners
|
||||
args:
|
||||
creates: /etc/nftables.d/host-scanners.nft
|
||||
|
||||
- name: Read the internet scanner sets file
|
||||
ansible.builtin.stat:
|
||||
path: /etc/nftables.d/host-scanners.nft
|
||||
register: firewall_scanners
|
||||
|
||||
- name: Require non-empty internet scanner sets
|
||||
ansible.builtin.assert:
|
||||
that: firewall_scanners.stat.exists and firewall_scanners.stat.size > 0
|
||||
fail_msg: /etc/nftables.d/host-scanners.nft is missing or empty, refusing to apply the firewall
|
||||
# Check mode neither installs the updater nor builds the sets.
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Enable the internet scanner set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-scanners.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Stop the internet scanner set refresh
|
||||
when: not firewall.scanners
|
||||
block:
|
||||
- name: Disable the internet scanner set timer
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host-scanners.timer
|
||||
enabled: false
|
||||
state: stopped
|
||||
daemon_reload: "{{ firewall_units is changed }}"
|
||||
# Check mode does not install the unit.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
|
||||
- name: Remove the internet scanner sets file
|
||||
ansible.builtin.file:
|
||||
path: /etc/nftables.d/host-scanners.nft
|
||||
state: absent
|
||||
|
||||
- name: Render the rule set
|
||||
ansible.builtin.template:
|
||||
src: host.nft.j2
|
||||
dest: /etc/nftables.d/host.nft
|
||||
mode: "0644"
|
||||
validate: nft -c -f %s
|
||||
register: firewall_rules
|
||||
|
||||
- name: Read the firewall unit state
|
||||
ansible.builtin.command: systemctl is-active nft-host.service
|
||||
register: firewall_active
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Apply the rule set with a rollback timer
|
||||
when:
|
||||
- firewall_rules is changed or firewall_units is changed or firewall_active.stdout != 'active'
|
||||
- not ansible_check_mode
|
||||
block:
|
||||
# A failed run of the transient unit keeps its name taken for systemd-run.
|
||||
- name: Clear a leftover rollback timer
|
||||
ansible.builtin.shell: >-
|
||||
systemctl stop nft-host-rollback.timer;
|
||||
systemctl reset-failed nft-host-rollback.timer nft-host-rollback.service;
|
||||
true
|
||||
changed_when: false
|
||||
|
||||
# Unless disarmed below, this moves the rejected rule file aside so a reboot
|
||||
# does not load it again, and reloads the last confirmed one; without one
|
||||
# it drops the table.
|
||||
- name: Arm the rollback
|
||||
ansible.builtin.command: systemd-run --on-active={{ firewall.rollback_seconds }} --unit=nft-host-rollback /usr/local/sbin/nft-host-rollback
|
||||
changed_when: false
|
||||
|
||||
- name: Load the rule set
|
||||
ansible.builtin.systemd:
|
||||
name: nft-host.service
|
||||
enabled: true
|
||||
state: reloaded
|
||||
daemon_reload: true
|
||||
|
||||
# A reused SSH control socket would hide a lockout, so test a fresh connection.
|
||||
- name: Drop the SSH connection
|
||||
ansible.builtin.meta: reset_connection
|
||||
|
||||
- name: Reconnect through the new rule set
|
||||
ansible.builtin.wait_for_connection:
|
||||
timeout: "{{ (firewall.rollback_seconds / 2) | int }}"
|
||||
|
||||
- name: Check the services behind the new rule set
|
||||
ansible.builtin.shell: "{{ item.value }}"
|
||||
loop: "{{ firewall.confirm | default({}, true) | dict2items | rejectattr('value', 'none') }}"
|
||||
loop_control:
|
||||
label: "{{ item.key }}"
|
||||
register: firewall_confirm
|
||||
until: firewall_confirm.rc == 0
|
||||
retries: "{{ (firewall.rollback_seconds / 20) | int }}"
|
||||
delay: 5
|
||||
changed_when: false
|
||||
|
||||
# Fails when the timer already fired, so a slow check cannot pass for a rolled-back host.
|
||||
- name: Disarm the rollback
|
||||
ansible.builtin.shell: >-
|
||||
test "$(systemctl show --property=SubState --value nft-host-rollback.timer)" = waiting &&
|
||||
systemctl stop nft-host-rollback.timer
|
||||
changed_when: true
|
||||
|
||||
- name: Keep the rule set as the last confirmed one
|
||||
ansible.builtin.copy:
|
||||
src: /etc/nftables.d/host.nft
|
||||
dest: /etc/nftables.d/host.nft.good
|
||||
remote_src: true
|
||||
mode: "0644"
|
||||
@@ -0,0 +1,306 @@
|
||||
#!/usr/sbin/nft -f
|
||||
# {{ ansible_managed }}
|
||||
{% macro list(map) -%}
|
||||
{{ map | default({}, true) | dict2items | rejectattr('value', 'none') | map(attribute='value') | map('string') | unique | join(', ') }}
|
||||
{%- endmacro %}
|
||||
{% macro addr_set(name, type, map) -%}
|
||||
set {{ name }} {
|
||||
type {{ type }}; flags interval; auto-merge;
|
||||
{% if list(map) %}
|
||||
elements = { {{ list(map) }} }
|
||||
{% endif %}
|
||||
}
|
||||
{%- endmacro %}
|
||||
{% set cluster = firewall.cluster | default({}, true) %}
|
||||
{% set cluster_ifaces = cluster.ifaces | default({}, true) | dict2items | rejectattr('value', 'none') | map(attribute='value') | unique | list %}
|
||||
{% set internal_ifaces = firewall.internal_ifaces | default({}, true) | dict2items | rejectattr('value', 'none') | map(attribute='value') | unique | list %}
|
||||
{% set guest_ifaces = firewall.guest_ifaces | default({}, true) | dict2items | rejectattr('value', 'none') | map(attribute='value') | unique | list %}
|
||||
{% set guest_match = 'iifname { "' ~ (guest_ifaces | join('", "')) ~ '" }' %}
|
||||
{% set dnat = firewall.dnat | default({}, true) | dict2items | rejectattr('value', 'none') | map(attribute='value') | list %}
|
||||
{% set masquerade = firewall.masquerade | default({}, true) | dict2items | rejectattr('value', 'none') | map(attribute='value') | unique | list %}
|
||||
{% set scrapers = firewall.scrapers | default({}, true) %}
|
||||
{% set forward_guard = firewall.forward_guard | default({}, true) | dict2items | rejectattr('value', 'none') | list %}
|
||||
{% set blocklists = firewall.spamhaus or firewall.firehol %}
|
||||
|
||||
# Declare-then-delete makes every load one atomic transaction that replaces
|
||||
# only these tables; Cilium, Docker, NetBird and CrowdSec tables stay untouched.
|
||||
table inet host {}
|
||||
delete table inet host
|
||||
table ip host_nat {}
|
||||
delete table ip host_nat
|
||||
|
||||
{% if firewall.geo_countries %}
|
||||
include "/etc/nftables.d/host-geo.nft"
|
||||
{% endif %}
|
||||
{% if firewall.spamhaus %}
|
||||
include "/etc/nftables.d/host-spamhaus.nft"
|
||||
{% endif %}
|
||||
{% if firewall.firehol %}
|
||||
include "/etc/nftables.d/host-firehol.nft"
|
||||
{% endif %}
|
||||
{% if firewall.scanners %}
|
||||
include "/etc/nftables.d/host-scanners.nft"
|
||||
{% endif %}
|
||||
|
||||
table inet host {
|
||||
{{ addr_set('admin_v4', 'ipv4_addr', firewall.admin.v4) }}
|
||||
{{ addr_set('admin_v6', 'ipv6_addr', firewall.admin.v6) }}
|
||||
{{ addr_set('cluster_v4', 'ipv4_addr', cluster.v4) }}
|
||||
{{ addr_set('cluster_v6', 'ipv6_addr', cluster.v6) }}
|
||||
{{ addr_set('scrapers_v4', 'ipv4_addr', scrapers.v4) }}
|
||||
{{ addr_set('scrapers_v6', 'ipv6_addr', scrapers.v6) }}
|
||||
set ssh_meter_v4 {
|
||||
type ipv4_addr; flags dynamic; timeout 10m; size 65535;
|
||||
}
|
||||
set ssh_meter_v6 {
|
||||
type ipv6_addr; flags dynamic; timeout 10m; size 65535;
|
||||
}
|
||||
|
||||
chain input {
|
||||
type filter hook input priority filter; policy drop;
|
||||
{% if blocklists %}
|
||||
|
||||
# Before the established rule: outbound is blocked too, so listed
|
||||
# addresses have no legitimate replies.
|
||||
{% if firewall.spamhaus %}
|
||||
ip saddr @spamhaus_v4 jump spamhaus_in
|
||||
ip6 saddr @spamhaus_v6 jump spamhaus_in
|
||||
{% endif %}
|
||||
{% if firewall.firehol %}
|
||||
ip saddr @firehol_v4 jump firehol_in
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
ct state established,related accept
|
||||
{% if firewall.scanners %}
|
||||
|
||||
# After the established rule: only inbound connections of scanners drop.
|
||||
ip saddr @scanners_v4 jump scanners_in
|
||||
ip6 saddr @scanners_v6 jump scanners_in
|
||||
|
||||
{% endif %}
|
||||
ct state invalid jump drop_log
|
||||
iif lo accept
|
||||
|
||||
icmp type { echo-request, destination-unreachable, time-exceeded, parameter-problem } accept
|
||||
icmpv6 type { echo-request, destination-unreachable, packet-too-big, time-exceeded, parameter-problem, nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert, mld-listener-query } accept
|
||||
udp sport 67 udp dport 68 accept
|
||||
meta nfproto ipv6 udp sport 547 udp dport 546 accept
|
||||
{% if cluster_ifaces or list(cluster.v4) or list(cluster.v6) %}
|
||||
|
||||
# ponytail: the cluster ring trusts every port between nodes and pods;
|
||||
# per-port lists (6443, 10250, 2379-2380, 4240, 8472, ...) are the upgrade path.
|
||||
{% for iface in cluster_ifaces %}
|
||||
iifname "{{ iface }}" accept
|
||||
{% endfor %}
|
||||
iifname "wt0" ip saddr @cluster_v4 accept
|
||||
iifname "wt0" ip6 saddr @cluster_v6 accept
|
||||
{% endif %}
|
||||
{% if internal_ifaces %}
|
||||
|
||||
# ponytail: internal bridges are trusted on every port; per-port lists are the upgrade path.
|
||||
{% for iface in internal_ifaces %}
|
||||
iifname "{{ iface }}" accept
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if guest_ifaces %}
|
||||
|
||||
# Pods on k3s guests egress through the node IP, so guest bridges get
|
||||
# only DHCP and router solicitations beyond the public zone.
|
||||
{{ guest_match }} udp dport 67 accept
|
||||
{{ guest_match }} icmpv6 type nd-router-solicit accept
|
||||
{% endif %}
|
||||
|
||||
# VPN peers get the vpn ports, then everything public.
|
||||
iifname "wt0" jump vpn
|
||||
jump public
|
||||
|
||||
# Broadcasts and multicasts are never scans; they stay out of the log.
|
||||
meta pkttype { broadcast, multicast } drop
|
||||
jump drop_log
|
||||
}
|
||||
|
||||
# A single rule makes the limit global: a flood logs at the limit and the
|
||||
# rest drops silently. The kernel log line has the iptables LOG fields that
|
||||
# the CrowdSec iptables-logs parser reads.
|
||||
chain drop_log {
|
||||
limit rate {{ firewall.log.in_rate }} log prefix "fw-in-drop: "
|
||||
drop
|
||||
}
|
||||
{% if firewall.spamhaus %}
|
||||
|
||||
# Own limits, so a listed scanner cannot use up the budget of fw-in-drop.
|
||||
# CrowdSec whitelists both prefixes: the lists already ban these networks.
|
||||
chain spamhaus_in {
|
||||
limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-spamhaus-in: "
|
||||
drop
|
||||
}
|
||||
|
||||
chain spamhaus_out {
|
||||
limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-spamhaus-out: "
|
||||
reject
|
||||
}
|
||||
{% endif %}
|
||||
{% if firewall.firehol %}
|
||||
|
||||
chain firehol_in {
|
||||
limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-firehol-in: "
|
||||
drop
|
||||
}
|
||||
|
||||
chain firehol_out {
|
||||
limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-firehol-out: "
|
||||
reject
|
||||
}
|
||||
{% endif %}
|
||||
{% if firewall.scanners %}
|
||||
|
||||
# Own limit and prefix, so scanners use up neither the fw-in-drop budget
|
||||
# nor the one of the blocklists. CrowdSec whitelists the prefix: scans of
|
||||
# these networks are known and need no decision.
|
||||
chain scanners_in {
|
||||
limit rate {{ firewall.log.spamhaus_rate }} log prefix "fw-scanners-in: "
|
||||
drop
|
||||
}
|
||||
{% endif %}
|
||||
{% if firewall.block_smtp_out or blocklists %}
|
||||
|
||||
# Outbound mail goes through the mail host only. Loopback stays open for a
|
||||
# local MTA; the log lines carry OUT=, which CrowdSec does not count as a scan.
|
||||
chain output {
|
||||
type filter hook output priority filter; policy accept;
|
||||
{% if firewall.spamhaus %}
|
||||
|
||||
ip daddr @spamhaus_v4 jump spamhaus_out
|
||||
ip6 daddr @spamhaus_v6 jump spamhaus_out
|
||||
{% endif %}
|
||||
{% if firewall.firehol %}
|
||||
|
||||
ip daddr @firehol_v4 jump firehol_out
|
||||
{% endif %}
|
||||
{% if firewall.block_smtp_out %}
|
||||
|
||||
oifname != "lo" tcp dport 25 limit rate {{ firewall.log.out_rate }} log prefix "fw-out-reject: "
|
||||
oifname != "lo" tcp dport 25 reject with tcp reset
|
||||
{% endif %}
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
chain vpn {
|
||||
{% if list(scrapers.tcp) %}
|
||||
{% if list(scrapers.v4) %}
|
||||
ip saddr @scrapers_v4 tcp dport { {{ list(scrapers.tcp) }} } accept
|
||||
{% endif %}
|
||||
{% if list(scrapers.v6) %}
|
||||
ip6 saddr @scrapers_v6 tcp dport { {{ list(scrapers.tcp) }} } accept
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if list(firewall.vpn.tcp) %}
|
||||
tcp dport { {{ list(firewall.vpn.tcp) }} } accept
|
||||
{% endif %}
|
||||
{% if list(firewall.vpn.udp) %}
|
||||
udp dport { {{ list(firewall.vpn.udp) }} } accept
|
||||
{% endif %}
|
||||
}
|
||||
|
||||
chain public {
|
||||
{% if list(firewall.public.tcp) %}
|
||||
tcp dport { {{ list(firewall.public.tcp) }} } accept
|
||||
{% endif %}
|
||||
{% if list(firewall.public.udp) %}
|
||||
udp dport { {{ list(firewall.public.udp) }} } accept
|
||||
{% endif %}
|
||||
{% if list(firewall.geo.tcp) %}
|
||||
tcp dport { {{ list(firewall.geo.tcp) }} } jump geo
|
||||
{% endif %}
|
||||
{% if list(firewall.admin.tcp) %}
|
||||
tcp dport { {{ list(firewall.admin.tcp) }} } jump admin
|
||||
{% endif %}
|
||||
}
|
||||
{% if list(firewall.admin.tcp) %}
|
||||
|
||||
chain admin {
|
||||
ip saddr @admin_v4 accept
|
||||
ip6 saddr @admin_v6 accept
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
chain geo {
|
||||
{% if guest_ifaces %}
|
||||
# Guest IPv6 addresses sit inside the geo sets.
|
||||
{{ guest_match }} return
|
||||
{% endif %}
|
||||
ip saddr @admin_v4 accept
|
||||
ip6 saddr @admin_v6 accept
|
||||
tcp dport 22 ct state new update @ssh_meter_v4 { ip saddr limit rate over {{ firewall.ssh_rate_limit }} } jump drop_log
|
||||
# One IPv6 client usually holds a whole /64.
|
||||
tcp dport 22 ct state new update @ssh_meter_v6 { ip6 saddr & ffff:ffff:ffff:ffff:: limit rate over {{ firewall.ssh_rate_limit }} } jump drop_log
|
||||
{% if firewall.geo_countries %}
|
||||
ip saddr @geo_v4 accept
|
||||
ip6 saddr @geo_v6 accept
|
||||
{% endif %}
|
||||
}
|
||||
{% if forward_guard or blocklists or firewall.scanners %}
|
||||
|
||||
chain forward {
|
||||
type filter hook forward priority filter; policy accept;
|
||||
{% if firewall.spamhaus %}
|
||||
|
||||
ip saddr @spamhaus_v4 jump spamhaus_in
|
||||
ip6 saddr @spamhaus_v6 jump spamhaus_in
|
||||
ip daddr @spamhaus_v4 jump spamhaus_out
|
||||
ip6 daddr @spamhaus_v6 jump spamhaus_out
|
||||
{% endif %}
|
||||
{% if firewall.firehol %}
|
||||
|
||||
ip saddr @firehol_v4 jump firehol_in
|
||||
ip daddr @firehol_v4 jump firehol_out
|
||||
{% endif %}
|
||||
{% if firewall.scanners %}
|
||||
|
||||
ct state new ip saddr @scanners_v4 jump scanners_in
|
||||
ct state new ip6 saddr @scanners_v6 jump scanners_in
|
||||
{% endif %}
|
||||
{% for guard in forward_guard %}
|
||||
|
||||
# {{ guard.key }}
|
||||
{% set ifaces = guard.value.ifaces | default({}, true) | dict2items | rejectattr('value', 'none') | map(attribute='value') | unique | list %}
|
||||
{% for fam, l3, icmp in [('v4', 'ip', 'icmp'), ('v6', 'ip6', 'icmpv6')] if list((guard.value[fam] | default({}, true)).daddr) %}
|
||||
{% set zone = guard.value[fam] %}
|
||||
{% set match = l3 ~ ' daddr { ' ~ list(zone.daddr) ~ ' }' %}
|
||||
{{ match }} ct state established,related accept
|
||||
{% if ifaces %}
|
||||
{{ match }} iifname { "{{ ifaces | join('", "') }}" } accept
|
||||
{% endif %}
|
||||
{% if list(zone.tcp) %}
|
||||
{{ match }} tcp dport { {{ list(zone.tcp) }} } accept
|
||||
{% endif %}
|
||||
{% if list(zone.icmp) %}
|
||||
{{ match }} {{ icmp }} type { {{ list(zone.icmp) }} } accept
|
||||
{% endif %}
|
||||
{{ match }} jump drop_log
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
}
|
||||
{% endif %}
|
||||
}
|
||||
{% if dnat or masquerade %}
|
||||
|
||||
table ip host_nat {
|
||||
chain prerouting {
|
||||
type nat hook prerouting priority dstnat; policy accept;
|
||||
{% for fwd in dnat %}
|
||||
{% for proto in ['tcp', 'udp'] if list(fwd[proto]) %}
|
||||
ip daddr {{ fwd.daddr }} {{ proto }} dport { {{ list(fwd[proto]) }} } dnat to {{ fwd.to }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
}
|
||||
|
||||
chain postrouting {
|
||||
type nat hook postrouting priority srcnat; policy accept;
|
||||
{% for match in masquerade %}
|
||||
{{ match }} masquerade
|
||||
{% endfor %}
|
||||
}
|
||||
}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
# {{ ansible_managed }}
|
||||
# Rebuilds the geo sets of table inet host from ipdeny. Any failed download or
|
||||
# check exits before the swap, so the previous file stays in place.
|
||||
set -eu
|
||||
|
||||
out=/etc/nftables.d/host-geo.nft
|
||||
countries="{{ firewall.geo_countries | join(' ') }}"
|
||||
prefix='^[0-9a-f.:]+/[0-9]+$'
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
fetch() {
|
||||
curl -fsS --retry 3 --max-time 120 -o "$2" "$1"
|
||||
grep -Eq "$prefix" "$2" || { echo "no prefixes in $1" >&2; exit 1; }
|
||||
}
|
||||
|
||||
for cc in $countries; do
|
||||
fetch "https://www.ipdeny.com/ipblocks/data/aggregated/$cc-aggregated.zone" "$tmp/$cc.v4"
|
||||
fetch "https://www.ipdeny.com/ipv6/ipaddresses/aggregated/$cc-aggregated.zone" "$tmp/$cc.v6"
|
||||
done
|
||||
|
||||
elements() {
|
||||
cat "$@" | grep -E "$prefix" | sed 's/$/,/'
|
||||
}
|
||||
|
||||
{
|
||||
echo "# Generated by $0, do not edit."
|
||||
echo 'table inet host {'
|
||||
echo ' set geo_v4 { type ipv4_addr; flags interval; auto-merge; elements = {'
|
||||
elements "$tmp"/*.v4
|
||||
echo ' } }'
|
||||
echo ' set geo_v6 { type ipv6_addr; flags interval; auto-merge; elements = {'
|
||||
elements "$tmp"/*.v6
|
||||
echo ' } }'
|
||||
echo '}'
|
||||
} >"$tmp/host-geo.nft"
|
||||
|
||||
# Checked under another table name: a dry run against the live table name
|
||||
# changes the live sets for its duration and unlisted addresses match.
|
||||
sed 's/^table inet host {/table inet host_geo_check {/' "$tmp/host-geo.nft" | nft -c -f -
|
||||
install -m 0644 "$tmp/host-geo.nft" "$out.new"
|
||||
mv -f "$out.new" "$out"
|
||||
|
||||
if systemctl -q is-active nft-host.service; then
|
||||
systemctl reload nft-host.service
|
||||
fi
|
||||
@@ -0,0 +1,11 @@
|
||||
# {{ ansible_managed }}
|
||||
{% set maps = [
|
||||
(crowdsec | default({}, true)).whitelist | default({}, true),
|
||||
firewall.admin.v4 | default({}, true), firewall.admin.v6 | default({}, true),
|
||||
firewall.scrapers.v4 | default({}, true), firewall.scrapers.v6 | default({}, true),
|
||||
firewall.cluster.v4 | default({}, true), firewall.cluster.v6 | default({}, true)] %}
|
||||
{% set entries = (maps | map('dict2items') | sum(start=[]) | rejectattr('value', 'none') | map(attribute='value') | map('string') | list
|
||||
+ groups['all'] | map('extract', hostvars, 'public_ips') | select('defined') | flatten) | unique %}
|
||||
{% for entry in entries %}
|
||||
{{ entry }}
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
journal_upload:
|
||||
# Ingest route of VictoriaLogs (insert path only, NetBird sources, basic auth per host).
|
||||
# The public DNS answers with the cluster's public address, where the route rejects
|
||||
# every source outside NetBird; /etc/hosts pins the name to the NetBird addresses below.
|
||||
host: ""
|
||||
path: /insert/journald
|
||||
# NetBird addresses of the Traefik nodes. The client tries them in this order when one is down.
|
||||
addresses: []
|
||||
# Basic auth credential; the user is the host's entry in the Secret vl-ingest-users.
|
||||
user: "{{ inventory_hostname }}"
|
||||
password: ""
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
- name: Reload systemd manager
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Restart systemd-journal-upload
|
||||
ansible.builtin.systemd:
|
||||
name: systemd-journal-upload
|
||||
state: restarted
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
@@ -0,0 +1,83 @@
|
||||
---
|
||||
# Ships the whole journal to VictoriaLogs. The uploader keeps the cursor of the last uploaded
|
||||
# entry in its state directory, so a restart resumes where it stopped.
|
||||
- name: Require the endpoint and the credential
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- journal_upload.host | length > 0
|
||||
- journal_upload.addresses | length > 0
|
||||
- journal_upload.user | length > 0
|
||||
- journal_upload.password | length > 0
|
||||
fail_msg: journal_upload.host, addresses, user and password must be set
|
||||
|
||||
- name: Install the journal uploader
|
||||
ansible.builtin.apt:
|
||||
name: systemd-journal-remote
|
||||
install_recommends: false
|
||||
state: present
|
||||
|
||||
# A node reaching its own address arrives at Traefik from its cilium_host address, which the
|
||||
# NetBird allowlist rejects; it takes the other nodes.
|
||||
- name: Pin the ingest name to the NetBird addresses
|
||||
ansible.builtin.blockinfile:
|
||||
path: /etc/hosts
|
||||
marker: "# {mark} journal_upload"
|
||||
block: |
|
||||
{% for address in journal_upload.addresses | reject('in', ansible_facts.all_ipv4_addresses) %}
|
||||
{{ address }} {{ journal_upload.host }}
|
||||
{% endfor %}
|
||||
notify: Restart systemd-journal-upload
|
||||
|
||||
- name: Create the uploader config directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/systemd/journal-upload.conf.d
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
# The service runs as a dynamic user, which reads the file after the manager dropped privileges.
|
||||
- name: Configure the upload URL
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/journal-upload.conf.d/10-victorialogs.conf
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Upload]
|
||||
URL=https://{{ journal_upload.user }}:{{ journal_upload.password }}@{{ journal_upload.host }}{{ journal_upload.path }}
|
||||
# No client certificate; the server certificate is checked against the system CAs.
|
||||
ServerKeyFile=-
|
||||
ServerCertificateFile=-
|
||||
TrustedCertificateFile=/etc/ssl/certs/ca-certificates.crt
|
||||
notify: Restart systemd-journal-upload
|
||||
no_log: true
|
||||
|
||||
# The default start limit (5 starts within 10 seconds) gives up during an ingest outage. The
|
||||
# uploader resumes from its cursor, so it keeps retrying.
|
||||
- name: Create the uploader unit drop-in directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/systemd/system/systemd-journal-upload.service.d
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Keep the uploader retrying through ingest outages
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/systemd-journal-upload.service.d/restart.conf
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Unit]
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=30s
|
||||
notify: Reload systemd manager
|
||||
|
||||
# The reload has to precede the start below, which otherwise hits the default start limit.
|
||||
- name: Load the uploader drop-in
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable the journal uploader
|
||||
ansible.builtin.systemd:
|
||||
name: systemd-journal-upload
|
||||
enabled: true
|
||||
state: started
|
||||
# Check mode does not install the package.
|
||||
ignore_errors: "{{ ansible_check_mode }}"
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
kopia_backup:
|
||||
# S3 repository; it is created, and its policies and maintenance are set, outside this role.
|
||||
endpoint: tier1.backups.ruekov.eu
|
||||
bucket: ""
|
||||
access_key: ""
|
||||
secret_key: ""
|
||||
password: ""
|
||||
# Snapshot owner is root@<hostname>; several hosts can share one repository.
|
||||
hostname: "{{ inventory_hostname_short }}"
|
||||
# Kopia defaults when unset.
|
||||
content_cache_size_mb:
|
||||
metadata_cache_size_mb:
|
||||
# Paths to snapshot; missing ones are skipped at run time.
|
||||
sources: []
|
||||
# source path -> ignore rules, replacing the policy rules on every run (null drops one).
|
||||
ignore: {}
|
||||
# Command run first, e.g. a host script writing dumps into a source; a failure fails the run, the snapshot still runs.
|
||||
pre_command: ""
|
||||
# Database dumps written to /var/backups/dumps before the snapshot, which then includes that directory.
|
||||
# Each item: file, container, command (run with sh -c inside the container, output is the dump).
|
||||
dumps: []
|
||||
# Files copied to the host as they are, each { src: local path, dest, mode }; owned by root.
|
||||
files: []
|
||||
# Snapshot the newest k3s etcd snapshot, staged in /var/backups/etcd.
|
||||
etcd_snapshot: false
|
||||
# Explicit UTC: the hosts run in different timezones.
|
||||
on_calendar: "*-*-* 03:30 UTC"
|
||||
randomized_delay: ""
|
||||
# Extra units the service starts after, e.g. docker.service for dumps.
|
||||
after: []
|
||||
# Nice=10 and idle IO scheduling.
|
||||
idle_priority: true
|
||||
@@ -0,0 +1,104 @@
|
||||
---
|
||||
- name: Require the repository settings
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- kopia_backup.bucket | length > 0
|
||||
- kopia_backup.access_key | length > 0
|
||||
- kopia_backup.secret_key | length > 0
|
||||
- kopia_backup.password | length > 0
|
||||
- kopia_backup.sources | length > 0
|
||||
fail_msg: kopia_backup.bucket, access_key, secret_key, password and sources must be set
|
||||
|
||||
- name: Create /etc/apt/keyrings
|
||||
ansible.builtin.file:
|
||||
path: /etc/apt/keyrings
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Install gpg for the apt key
|
||||
ansible.builtin.apt:
|
||||
name: gpg
|
||||
state: present
|
||||
|
||||
- name: Add the Kopia apt key
|
||||
ansible.builtin.shell: curl -fsSL https://kopia.io/signing-key | gpg --dearmor -o /etc/apt/keyrings/kopia-keyring.gpg
|
||||
args:
|
||||
creates: /etc/apt/keyrings/kopia-keyring.gpg
|
||||
|
||||
- name: Add the Kopia apt repository
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/apt/sources.list.d/kopia.list
|
||||
mode: "0644"
|
||||
content: |
|
||||
deb [signed-by=/etc/apt/keyrings/kopia-keyring.gpg] http://packages.kopia.io/apt/ stable main
|
||||
register: kopia_backup_repo
|
||||
|
||||
- name: Install Kopia
|
||||
ansible.builtin.apt:
|
||||
name: kopia
|
||||
state: present
|
||||
update_cache: "{{ kopia_backup_repo is changed }}"
|
||||
|
||||
- name: Read the repository connection
|
||||
ansible.builtin.command: kopia repository status
|
||||
environment:
|
||||
KOPIA_CHECK_FOR_UPDATES: "false"
|
||||
register: kopia_backup_status
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Connect the repository
|
||||
ansible.builtin.command:
|
||||
argv: "{{ argv + (cache if kopia_backup.content_cache_size_mb else []) }}"
|
||||
vars:
|
||||
argv:
|
||||
- kopia
|
||||
- repository
|
||||
- connect
|
||||
- s3
|
||||
- --endpoint={{ kopia_backup.endpoint }}
|
||||
- --bucket={{ kopia_backup.bucket }}
|
||||
- --override-hostname={{ kopia_backup.hostname }}
|
||||
- --override-username=root
|
||||
cache:
|
||||
- --content-cache-size-mb={{ kopia_backup.content_cache_size_mb }}
|
||||
- --metadata-cache-size-mb={{ kopia_backup.metadata_cache_size_mb }}
|
||||
# Secrets through the environment, flags would show them in the process list
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID: "{{ kopia_backup.access_key }}"
|
||||
AWS_SECRET_ACCESS_KEY: "{{ kopia_backup.secret_key }}"
|
||||
KOPIA_PASSWORD: "{{ kopia_backup.password }}"
|
||||
KOPIA_CHECK_FOR_UPDATES: "false"
|
||||
when: kopia_backup_status.rc != 0
|
||||
no_log: true
|
||||
|
||||
- name: Install the files of the host
|
||||
ansible.builtin.copy:
|
||||
src: "{{ item.src }}"
|
||||
dest: "{{ item.dest }}"
|
||||
mode: "{{ item.mode }}"
|
||||
owner: root
|
||||
group: root
|
||||
loop: "{{ kopia_backup.files }}"
|
||||
|
||||
- name: Install the backup script
|
||||
ansible.builtin.template:
|
||||
src: kopia-backup.j2
|
||||
dest: /usr/local/sbin/kopia-backup
|
||||
mode: "0755"
|
||||
|
||||
- name: Install the systemd units
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: "0644"
|
||||
loop: [kopia-backup.service, kopia-backup.timer]
|
||||
register: kopia_backup_units
|
||||
|
||||
- name: Enable the timer
|
||||
ansible.builtin.systemd:
|
||||
name: kopia-backup.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: "{{ kopia_backup_units is changed }}"
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# {{ ansible_managed }}
|
||||
# Daily Kopia backup of {{ kopia_backup.hostname }} into the bucket {{ kopia_backup.bucket }} on {{ kopia_backup.endpoint }}.
|
||||
set -uo pipefail
|
||||
export KOPIA_CHECK_FOR_UPDATES=false
|
||||
failed=0
|
||||
sources=()
|
||||
{% if kopia_backup.pre_command %}
|
||||
|
||||
{{ kopia_backup.pre_command }} || { echo "pre_command failed" >&2; failed=1; }
|
||||
{% endif %}
|
||||
{% if kopia_backup.etcd_snapshot %}
|
||||
|
||||
# Only the latest etcd snapshot; the live etcd dir is not consistent on disk and older snapshots only add churn
|
||||
etcd=/var/backups/etcd
|
||||
latest=$(ls -t /var/lib/rancher/k3s/server/db/snapshots/etcd-snapshot-* 2>/dev/null | head -1)
|
||||
if [ -n "$latest" ]; then
|
||||
install -d -m700 "$etcd"
|
||||
rm -f "$etcd/etcd-snapshot"
|
||||
ln "$latest" "$etcd/etcd-snapshot" 2>/dev/null || cp -p "$latest" "$etcd/etcd-snapshot" || failed=1
|
||||
fi
|
||||
{% endif %}
|
||||
{% if kopia_backup.dumps %}
|
||||
|
||||
dumps=/var/backups/dumps
|
||||
install -d -m700 "$dumps"
|
||||
dump() { # file, container, command run inside the container
|
||||
if docker exec "$2" sh -c "$3" > "$dumps/$1.tmp"; then
|
||||
mv "$dumps/$1.tmp" "$dumps/$1"
|
||||
else
|
||||
echo "dump of $1 failed" >&2; rm -f "$dumps/$1.tmp"; failed=1
|
||||
fi
|
||||
}
|
||||
{% for d in kopia_backup.dumps %}
|
||||
dump {{ d.file | quote }} {{ d.container | quote }} {{ d.command | quote }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% set ignores = kopia_backup.ignore | dict2items | rejectattr('value', 'none') | list %}
|
||||
{% if ignores %}
|
||||
|
||||
# Ignore rules of a source, replaced as a whole (--clear-ignore runs after --add-ignore within one call).
|
||||
# Kopia sorts the rules, so negations (!) cannot be relied on to come after the rule they override.
|
||||
ignore() {
|
||||
local path=$1; shift
|
||||
kopia policy set "$path" --clear-ignore >/dev/null 2>&1 &&
|
||||
kopia policy set "$path" "${@/#/--add-ignore=}" >/dev/null 2>&1 ||
|
||||
{ echo "setting the ignore rules of $path failed" >&2; failed=1; }
|
||||
}
|
||||
{% for i in ignores %}
|
||||
ignore {{ i.key | quote }} {{ i.value | map('quote') | join(' ') }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
for p in {{ kopia_backup.sources | map('quote') | join(' ') }}{{ ' "$etcd"' if kopia_backup.etcd_snapshot }}{{ ' "$dumps"' if kopia_backup.dumps }}; do
|
||||
[ -d "$p" ] && sources+=("$p")
|
||||
done
|
||||
kopia snapshot create "${sources[@]}" || failed=1
|
||||
exit $failed
|
||||
@@ -0,0 +1,14 @@
|
||||
# {{ ansible_managed }}
|
||||
[Unit]
|
||||
Description=Kopia backup to {{ kopia_backup.endpoint }}
|
||||
After={{ (['network-online.target'] + kopia_backup.after) | join(' ') }}
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
Environment=HOME=/root
|
||||
{% if kopia_backup.idle_priority %}
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
{% endif %}
|
||||
ExecStart=/usr/local/sbin/kopia-backup
|
||||
@@ -0,0 +1,13 @@
|
||||
# {{ ansible_managed }}
|
||||
[Unit]
|
||||
Description=Daily Kopia backup to {{ kopia_backup.endpoint }}
|
||||
|
||||
[Timer]
|
||||
OnCalendar={{ kopia_backup.on_calendar }}
|
||||
{% if kopia_backup.randomized_delay %}
|
||||
RandomizedDelaySec={{ kopia_backup.randomized_delay }}
|
||||
{% endif %}
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,77 @@
|
||||
---
|
||||
kubernetes:
|
||||
ipPool:
|
||||
ipv4:
|
||||
# Minimum: /24
|
||||
cluster_cidr: 10.42.0.0/16
|
||||
service_cidr: 10.43.0.0/16
|
||||
ipv6:
|
||||
# Minimum: /120
|
||||
cluster_cidr: fd42::/112
|
||||
service_cidr: fd43::/112
|
||||
|
||||
# Interface to grab node-IPv4/v6 from
|
||||
nodeIp_interface: <interface to grab nodeIp from>
|
||||
|
||||
control_plane:
|
||||
dns_name: <control-plane dns-reachable-name>
|
||||
|
||||
token: <shared token for nodes to join>
|
||||
|
||||
network:
|
||||
# Copies the current /etc/resolv.conf whilst only keeping "nameserver" lines
|
||||
# can be used to remove filter
|
||||
clean_resolv_conf: false
|
||||
|
||||
# One of [none, flannel, calico, cilium]
|
||||
plugin: calico
|
||||
|
||||
# Helper for networking
|
||||
helper:
|
||||
# https://github.com/Ruakij/RoutingTableToWg
|
||||
# Translates received-routes from e.g. BGP to wireguard-allowedips
|
||||
# Helpful, when nodeIp_interface is a wireguard-interface
|
||||
routingtabletowg: false
|
||||
|
||||
# One of [none, traefik-ingress, nginx-ingress]
|
||||
ingress_controller: traefik-ingress
|
||||
|
||||
# k3s version to install, e.g. v1.37.1+k3s1. An installed k3s is replaced only
|
||||
# when this is set and differs; empty installs the latest when k3s is missing.
|
||||
version: ""
|
||||
|
||||
# KubeletConfiguration fields (evictionHard, evictionSoft, ...) written to
|
||||
# /etc/rancher/k3s/kubelet.conf.d/20-memory.conf; empty writes no file
|
||||
kubelet_memory: {}
|
||||
|
||||
# KubeletConfiguration fields systemReserved and kubeReserved written to
|
||||
# /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf; empty writes no file.
|
||||
# Both only shrink the allocatable of the node (enforceNodeAllocatable stays
|
||||
# pods). Sized in group_vars per node type from the measured memory and CPU
|
||||
# of k3s.service (kubeReserved) and of everything else outside kubepods.slice
|
||||
# (systemReserved).
|
||||
kubelet_reserved: {}
|
||||
|
||||
# Extra config of the nodes running etcd, left out with disable-etcd
|
||||
etcd_config_extra:
|
||||
# Without it etcd serves metrics on 127.0.0.1 only, out of reach of vmagent
|
||||
etcd-expose-metrics: true
|
||||
|
||||
config_extra:
|
||||
# etcd-tuning
|
||||
# heartbeat: 0.5-1.5x of rtt
|
||||
# election: 10x- of heartbeat
|
||||
etcd-arg:
|
||||
heartbeat-interval: 500
|
||||
election-timeout: 5000
|
||||
|
||||
registries_config:
|
||||
# Defines the names and endpoints of registries
|
||||
# When embedded-registry is enabled, every registry needs to be defined here to allow this node to distribute those images.
|
||||
#mirrors:
|
||||
# Enable distribution for all registries
|
||||
#"*": {}
|
||||
|
||||
# Defines authentication and extra settings for the registries
|
||||
#config:
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
@startuml
|
||||
|
||||
rectangle "Control-Plane" as control_plane {
|
||||
rectangle "Node" as sn1 {
|
||||
component "netclient" as sn1_netclient
|
||||
|
||||
component etcd as sn1_etcd
|
||||
component "k3s-server" as sn1_k3s_server
|
||||
sn1_k3s_server - sn1_etcd
|
||||
}
|
||||
|
||||
rectangle "Node" as sn2 {
|
||||
component "netclient" as sn2_netclient
|
||||
|
||||
component etcd as sn2_etcd
|
||||
component "k3s-server" as sn2_k3s_server
|
||||
sn2_k3s_server - sn2_etcd
|
||||
}
|
||||
|
||||
sn1_netclient -- sn2_netclient
|
||||
sn1_etcd -- sn2_etcd
|
||||
}
|
||||
|
||||
rectangle "Workers" {
|
||||
rectangle "Node" as an1 {
|
||||
component "netclient" as an1_netclient
|
||||
|
||||
component "k3s-agent" as sn1_k3s_agent
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@enduml
|
||||
@@ -0,0 +1,35 @@
|
||||
# Copyright 2018-2022 Docker Inc.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
disabled_plugins = []
|
||||
|
||||
#root = "/var/lib/containerd"
|
||||
#state = "/run/containerd"
|
||||
#subreaper = true
|
||||
#oom_score = 0
|
||||
|
||||
#[grpc]
|
||||
# address = "/run/containerd/containerd.sock"
|
||||
# uid = 0
|
||||
# gid = 0
|
||||
|
||||
#[debug]
|
||||
# address = "/run/containerd/debug.sock"
|
||||
# uid = 0
|
||||
# gid = 0
|
||||
# level = "info"
|
||||
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
||||
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
||||
SystemdCgroup = true
|
||||
@@ -0,0 +1,39 @@
|
||||
- name: reload_sysctl
|
||||
command: sysctl --system
|
||||
|
||||
- name: restart_containerd
|
||||
ansible.builtin.service:
|
||||
name: containerd
|
||||
state: restarted
|
||||
|
||||
- name: reload_networking
|
||||
service:
|
||||
name: networking
|
||||
state: restarted
|
||||
async: 5
|
||||
poll: 0
|
||||
notify: wait_for_connection
|
||||
|
||||
- name: wait_for_connection
|
||||
wait_for_connection:
|
||||
delay: 5
|
||||
|
||||
# Restarts the kubelet to read changed drop-ins. Runs only with
|
||||
# -e k3s_restart_allowed=true; node.yml refuses it for more than one host.
|
||||
- name: restart_k3s
|
||||
ansible.builtin.service:
|
||||
name: k3s
|
||||
state: restarted
|
||||
when:
|
||||
- k3s_restart_allowed | default(false) | bool
|
||||
- not ansible_check_mode
|
||||
notify: wait_for_k3s
|
||||
|
||||
- name: wait_for_k3s
|
||||
command: k3s kubectl get --raw /readyz
|
||||
register: k3s_readyz
|
||||
until: k3s_readyz.rc == 0 and k3s_readyz.stdout == 'ok'
|
||||
retries: 30
|
||||
delay: 10
|
||||
changed_when: false
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
#dependencies:
|
||||
# - role: docker
|
||||
@@ -0,0 +1,36 @@
|
||||
# kubelet gives kubepods.slice a cpu.weight from its allocatable CPUs (313 on
|
||||
# 8 cores), while system.slice and k3s.service keep the default 100. Under load
|
||||
# the pods then outweigh etcd, apiserver and kubelet. Weight 1000 lets the
|
||||
# control plane win that contention.
|
||||
- name: Set the CPUWeight of system.slice and the k3s unit
|
||||
vars:
|
||||
k3s_unit: "{{ 'k3s' if type == 'server' else 'k3s-agent' }}.service"
|
||||
block:
|
||||
- name: Create the drop-in directories
|
||||
ansible.builtin.file:
|
||||
path: "/etc/systemd/system/{{ item }}.d"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- system.slice
|
||||
- "{{ k3s_unit }}"
|
||||
|
||||
- name: Write the CPUWeight drop-ins
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/{{ item.unit }}.d/50-cpu-weight.conf"
|
||||
mode: "0644"
|
||||
content: |
|
||||
[{{ item.section }}]
|
||||
CPUWeight=1000
|
||||
loop:
|
||||
- unit: system.slice
|
||||
section: Slice
|
||||
- unit: "{{ k3s_unit }}"
|
||||
section: Service
|
||||
register: cpu_weight
|
||||
|
||||
# Only daemon-reload: the weight applies to the running k3s unit without a restart
|
||||
- name: Reload systemd for the CPUWeight drop-ins
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
when: cpu_weight is changed
|
||||
@@ -0,0 +1,68 @@
|
||||
- name: Create k3s-folder
|
||||
ansible.builtin.file:
|
||||
path: /etc/rancher/k3s/
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Read the live k3s config
|
||||
slurp:
|
||||
src: /etc/rancher/k3s/config.yaml
|
||||
register: live_config
|
||||
failed_when: false
|
||||
|
||||
# A rewrite restarts k3s, so the file is written only when k3s would read other
|
||||
# settings: the live files differ from the rendering in comments, order and
|
||||
# whitespace.
|
||||
- name: Deploy k3s config
|
||||
ansible.builtin.template:
|
||||
src: k3s/{{ type }}/config.yaml.jinja2
|
||||
dest: /etc/rancher/k3s/config.yaml
|
||||
register: config
|
||||
when: >-
|
||||
live_config.content is not defined or
|
||||
(live_config.content | b64decode | from_yaml) != (lookup('template', 'k3s/' ~ type ~ '/config.yaml.jinja2') | from_yaml)
|
||||
|
||||
- name: Read the installed k3s version
|
||||
command: k3s --version
|
||||
register: k3s_installed
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Download install-script
|
||||
get_url:
|
||||
url: https://get.k3s.io
|
||||
dest: /root/k3s_install.sh
|
||||
mode: "744"
|
||||
# todo: update when file changed?
|
||||
|
||||
- name: Read the live registries.yaml
|
||||
slurp:
|
||||
src: /etc/rancher/k3s/registries.yaml
|
||||
register: live_registries
|
||||
failed_when: false
|
||||
when: "type == 'server'"
|
||||
|
||||
# Written only on a change k3s would see, like config.yaml
|
||||
- name: Deploy registries.yaml
|
||||
ansible.builtin.template:
|
||||
src: k3s/server/registries.yaml.j2
|
||||
dest: /etc/rancher/k3s/registries.yaml
|
||||
register: registries_config
|
||||
when:
|
||||
- "type == 'server'"
|
||||
- >-
|
||||
live_registries.content is not defined or
|
||||
(live_registries.content | b64decode | from_yaml) != (lookup('template', 'k3s/server/registries.yaml.j2') | from_yaml)
|
||||
|
||||
- import_tasks: ./install/server/setup_network.yml
|
||||
when: "type == 'server'"
|
||||
|
||||
#- import_tasks: ./install/server/install_helm.yml
|
||||
# when: "type == 'server'"
|
||||
|
||||
- import_tasks: ./install/server/install_k3s.yml
|
||||
when: "type == 'server'"
|
||||
|
||||
- import_tasks: ./install/agent/install_k3s.yml
|
||||
when: "type == 'agent'"
|
||||
@@ -0,0 +1,15 @@
|
||||
- name: Install K3s agent
|
||||
command: /root/k3s_install.sh {{ type }}
|
||||
environment:
|
||||
INSTALL_K3S_VERSION: "{{ kubernetes.version }}"
|
||||
when: k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout)
|
||||
register: command
|
||||
changed_when: "'No change detected' not in command.stdout"
|
||||
until: "command is not failed"
|
||||
retries: 2
|
||||
delay: 10
|
||||
|
||||
- name: Make sure service is started / restarted on config change
|
||||
service:
|
||||
name: k3s-agent
|
||||
state: "{{ 'restarted' if not command.changed and config.changed else 'started' }}"
|
||||
@@ -0,0 +1,17 @@
|
||||
- name: Add Balto key
|
||||
apt_key:
|
||||
url: https://baltocdn.com/helm/signing.asc
|
||||
state: present
|
||||
|
||||
- name: Add Balto Repository
|
||||
apt_repository:
|
||||
repo: "deb https://baltocdn.com/helm/stable/debian/ all main"
|
||||
state: present
|
||||
filename: kubernetes
|
||||
update_cache: yes
|
||||
|
||||
- name: Install helm
|
||||
package:
|
||||
name:
|
||||
- helm
|
||||
state: latest
|
||||
@@ -0,0 +1,50 @@
|
||||
- block:
|
||||
- name: Install K3s-server
|
||||
command: /root/k3s_install.sh {{ type }}
|
||||
environment:
|
||||
INSTALL_K3S_CHANNEL: latest
|
||||
INSTALL_K3S_VERSION: "{{ kubernetes.version }}"
|
||||
when: k3s_installed.rc != 0 or (kubernetes.version != '' and ('k3s version ' ~ kubernetes.version ~ ' ') not in k3s_installed.stdout)
|
||||
register: command
|
||||
changed_when: "'No change detected' not in command.stdout"
|
||||
until: "command is not failed"
|
||||
retries: 2
|
||||
delay: 10
|
||||
|
||||
- name: Make sure service is started / restarted on config change
|
||||
service:
|
||||
name: k3s
|
||||
enabled: true
|
||||
state: "{{ 'restarted' if not command.changed and (config.changed or registries_config.changed) else 'started' }}"
|
||||
|
||||
- name: Waiting for K3s-server to accept connections
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ inventory_hostname }}"
|
||||
port: 6443
|
||||
state: started
|
||||
throttle: 1
|
||||
|
||||
- name: Ensure the .kube directory exists
|
||||
file:
|
||||
path: "~/.kube"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Create symbolic link for k3s.yaml
|
||||
file:
|
||||
src: "/etc/rancher/k3s/k3s.yaml"
|
||||
dest: "~/.kube/config"
|
||||
state: link
|
||||
|
||||
- name: Deploy calico
|
||||
import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml
|
||||
when:
|
||||
- "kubernetes.network.plugin == 'calico'"
|
||||
|
||||
- name: Deploy cilium
|
||||
import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_cilium.yml
|
||||
when:
|
||||
- "kubernetes.network.plugin == 'cilium'"
|
||||
|
||||
- name: Deploy network-helpers
|
||||
import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml
|
||||
@@ -0,0 +1,19 @@
|
||||
- name: Deploy calico operator
|
||||
command: kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml
|
||||
register: command
|
||||
changed_when: "'created' in command.stdout"
|
||||
run_once: true
|
||||
failed_when:
|
||||
- "command.rc == 1 and 'AlreadyExists' not in command.stderr"
|
||||
|
||||
- name: Deploy calico ressource template
|
||||
ansible.builtin.template:
|
||||
src: ./k3s/server/network-plugin/calico/custom-ressource.yml.jinja2
|
||||
dest: /root/calico-ressource.yml
|
||||
run_once: true
|
||||
|
||||
- name: Deploy calico ressource
|
||||
command: kubectl apply -f /root/calico-ressource.yml
|
||||
register: command
|
||||
changed_when: "'created' in command.stdout"
|
||||
run_once: true
|
||||
@@ -0,0 +1,71 @@
|
||||
- name: Check if already installed in cluster
|
||||
command: "cilium status"
|
||||
register: cilium_status
|
||||
ignore_errors: true
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
run_once: true
|
||||
|
||||
- name: Check if the Cilium CLI is installed
|
||||
stat:
|
||||
path: /usr/local/bin/cilium
|
||||
register: cilium_cli
|
||||
|
||||
- name: Get Cilium CLI version
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- curl
|
||||
- -s
|
||||
- https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt
|
||||
register: cilium_cli_version
|
||||
check_mode: no
|
||||
changed_when: false
|
||||
run_once: true
|
||||
|
||||
- name: Determine CLI architecture
|
||||
set_fact:
|
||||
cli_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
|
||||
run_once: true
|
||||
|
||||
- name: Download and verify Cilium CLI
|
||||
block:
|
||||
- name: Download Cilium CLI tar.gz file
|
||||
get_url:
|
||||
url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version.stdout }}/cilium-linux-{{ cli_arch }}.tar.gz"
|
||||
dest: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz"
|
||||
checksum: "sha256:{{ lookup('url', 'https://github.com/cilium/cilium-cli/releases/download/' + cilium_cli_version.stdout + '/cilium-linux-' + cli_arch + '.tar.gz.sha256sum', wantlist=True)[0].split(' ')[0] }}"
|
||||
mode: '0755'
|
||||
|
||||
- name: Extract Cilium CLI tar.gz to /usr/local/bin
|
||||
unarchive:
|
||||
src: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz"
|
||||
dest: "/usr/local/bin"
|
||||
remote_src: yes
|
||||
#extra_opts: ["--strip-components=1"]
|
||||
mode: 777
|
||||
# An installed CLI is left alone, not upgraded to the latest release
|
||||
when: cilium_cli_version.stdout is defined and not cilium_cli.stat.exists
|
||||
become: yes
|
||||
|
||||
- name: Clean up temporary files
|
||||
file:
|
||||
path: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz"
|
||||
state: absent
|
||||
|
||||
# Install cilium via cli
|
||||
- block:
|
||||
- name: Install cilium
|
||||
command: >
|
||||
cilium install
|
||||
--set
|
||||
imap.mode=kubernetes,
|
||||
ipv6.enabled=true,
|
||||
#ipam.operator.clusterPoolIPv4PodCIDRList={{ kubernetes.ipPool.ipv4.cluster_cidr }},
|
||||
#ipam.operator.clusterPoolIPv6PodCIDRList={{ kubernetes.ipPool.ipv6.cluster_cidr }}
|
||||
|
||||
- name: Wait for cilium
|
||||
command: cilium status --wait
|
||||
changed_when: false
|
||||
run_once: true
|
||||
when:
|
||||
- cilium_status.rc != 0
|
||||
@@ -0,0 +1,7 @@
|
||||
- name: Deploy service-file for routing-table to wireguard-translation
|
||||
ansible.builtin.template:
|
||||
src: ./k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2
|
||||
dest: /var/lib/rancher/k3s/server/manifests/routingtabletowg.yml
|
||||
mode: u=rw,g=r,o=r
|
||||
run_once: true
|
||||
when: "kubernetes.network.helper.routingtabletowg"
|
||||
@@ -0,0 +1,6 @@
|
||||
- name: Set control-plane-dns-endpoint towards local-ip
|
||||
blockinfile:
|
||||
path: /etc/hosts
|
||||
marker: "# {mark} ANSIBLE MANAGED BLOCK | k3s"
|
||||
block: |
|
||||
{{ nodeip_ipv4 }} {{ kubernetes.control_plane.dns_name }}
|
||||
@@ -0,0 +1,9 @@
|
||||
- import_tasks: ./prerequisites.yml
|
||||
|
||||
- import_tasks: ./node.yml
|
||||
|
||||
- import_tasks: ./cpu_weight.yml
|
||||
tags: cpu_weight
|
||||
|
||||
- import_tasks: ./install.yml
|
||||
when: target_host is not defined or inventory_hostname == target_host
|
||||
@@ -0,0 +1,109 @@
|
||||
- name: Kubelet image garbage collection
|
||||
tags: kubelet_gc
|
||||
block:
|
||||
- name: Create the kubelet drop-in directory
|
||||
file:
|
||||
path: /etc/rancher/k3s/kubelet.conf.d
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Set the image garbage collection thresholds
|
||||
copy:
|
||||
dest: /etc/rancher/k3s/kubelet.conf.d/10-image-gc.conf
|
||||
mode: "0644"
|
||||
content: |
|
||||
apiVersion: kubelet.config.k8s.io/v1beta1
|
||||
kind: KubeletConfiguration
|
||||
imageGCHighThresholdPercent: 70
|
||||
imageGCLowThresholdPercent: 55
|
||||
imageMaximumGCAge: 168h
|
||||
|
||||
# KubeletConfiguration fields from kubernetes.kubelet_memory (eviction) and
|
||||
# kubernetes.kubelet_reserved (systemReserved, kubeReserved); a file is removed
|
||||
# while its variable is empty. The kubelet reads the drop-ins only when k3s
|
||||
# starts, so a change takes effect after a k3s restart. The restart_k3s handler
|
||||
# restarts only with -e k3s_restart_allowed=true, on one node per run.
|
||||
- name: Kubelet memory reservation and eviction
|
||||
tags: [kubelet_memory, kubelet_reserved]
|
||||
block:
|
||||
- name: Refuse a k3s restart on several nodes at once
|
||||
ansible.builtin.assert:
|
||||
that: ansible_play_hosts_all | length == 1
|
||||
fail_msg: k3s_restart_allowed restarts k3s on every changed node of the play; limit the run to one node (--limit <node>) to keep the etcd quorum
|
||||
when: k3s_restart_allowed | default(false) | bool
|
||||
|
||||
- name: Create the kubelet drop-in directory
|
||||
file:
|
||||
path: /etc/rancher/k3s/kubelet.conf.d
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Set the kubelet eviction thresholds
|
||||
copy:
|
||||
dest: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf
|
||||
mode: "0644"
|
||||
content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_memory) | to_nice_yaml(indent=2) }}"
|
||||
when: kubernetes.kubelet_memory | length > 0
|
||||
register: kubelet_memory_dropin
|
||||
notify: restart_k3s
|
||||
|
||||
- name: Remove the kubelet memory drop-in
|
||||
file:
|
||||
path: /etc/rancher/k3s/kubelet.conf.d/20-memory.conf
|
||||
state: absent
|
||||
when: kubernetes.kubelet_memory | length == 0
|
||||
notify: restart_k3s
|
||||
|
||||
- name: Set the kubelet system and kube reservations
|
||||
copy:
|
||||
dest: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf
|
||||
mode: "0644"
|
||||
content: "{{ {'apiVersion': 'kubelet.config.k8s.io/v1beta1', 'kind': 'KubeletConfiguration'} | combine(kubernetes.kubelet_reserved) | to_nice_yaml(indent=2) }}"
|
||||
when: kubernetes.kubelet_reserved | length > 0
|
||||
register: kubelet_reserved_dropin
|
||||
notify: restart_k3s
|
||||
|
||||
- name: Remove the kubelet reservation drop-in
|
||||
file:
|
||||
path: /etc/rancher/k3s/kubelet.conf.d/30-reserved.conf
|
||||
state: absent
|
||||
when: kubernetes.kubelet_reserved | length == 0
|
||||
notify: restart_k3s
|
||||
|
||||
# Compares the file times with the start of the running k3s, so a pending
|
||||
# restart is reported on every run, not only on the run that wrote the file.
|
||||
- name: List the kubelet drop-ins written after k3s started
|
||||
shell: |
|
||||
since=$(systemctl show k3s -p ActiveEnterTimestamp --value)
|
||||
[ -n "$since" ] || exit 0
|
||||
start=$(date -d "$since" +%s) || exit 1
|
||||
for f in /etc/rancher/k3s/kubelet.conf.d/*; do
|
||||
[ "$(stat -c %Y "$f")" -gt "$start" ] && echo "$f"
|
||||
done
|
||||
exit 0
|
||||
register: kubelet_dropins_pending
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Report the pending k3s restart
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
k3s restart pending on {{ inventory_hostname }}: the kubelet reads
|
||||
/etc/rancher/k3s/kubelet.conf.d only at start. Restart one node at a
|
||||
time with --limit {{ inventory_hostname }} -e k3s_restart_allowed=true.
|
||||
when:
|
||||
- not (k3s_restart_allowed | default(false) | bool)
|
||||
- kubelet_dropins_pending.stdout | length > 0 or kubelet_memory_dropin is changed or kubelet_reserved_dropin is changed
|
||||
|
||||
# Every pod watching its config files takes inotify instances and watches from
|
||||
# the per-user pool. An LXC container inherits these from its host. Only this
|
||||
# file is loaded: sysctl --system would also apply distribution defaults such as
|
||||
# rp_filter that differ from the live values Cilium relies on.
|
||||
- name: Raise the inotify limits
|
||||
tags: inotify
|
||||
when: ansible_facts['virtualization_type'] != 'lxc'
|
||||
ansible.posix.sysctl:
|
||||
name: "{{ item.key }}"
|
||||
value: "{{ item.value }}"
|
||||
sysctl_file: /etc/sysctl.d/90-inotify.conf
|
||||
loop: "{{ {'fs.inotify.max_user_instances': 65536, 'fs.inotify.max_user_watches': 4194304} | dict2items }}"
|
||||
@@ -0,0 +1,61 @@
|
||||
#- name: Load br_netfilter kernel-module
|
||||
# modprobe:
|
||||
# name: br_netfilter
|
||||
# state: present
|
||||
|
||||
- name: Set sysctl settings for iptables bridged traffic
|
||||
copy:
|
||||
dest: "/etc/sysctl.d/kubernetes.conf"
|
||||
content: |
|
||||
net.bridge.bridge-nf-call-ip6tables = 1
|
||||
net.bridge.bridge-nf-call-iptables = 1
|
||||
|
||||
net.ipv4.conf.all.forwarding=1
|
||||
net.ipv6.conf.all.forwarding=1
|
||||
notify: reload_sysctl
|
||||
|
||||
#- name: Disable swap
|
||||
# command: swapoff -a
|
||||
|
||||
- name: Install required packages
|
||||
package:
|
||||
name:
|
||||
#- containerd
|
||||
#- iptables
|
||||
# For Longhorn:
|
||||
- nfs-common
|
||||
- open-iscsi
|
||||
state: present
|
||||
|
||||
- import_tasks: ./prerequisites/containerd.yml
|
||||
|
||||
- name: Gather interface-name
|
||||
set_fact:
|
||||
interface: "{{ kubernetes.ipPool.nodeIp_interface | replace('-', '_') }}"
|
||||
|
||||
- name: Getting nodeIp-data from interface
|
||||
set_fact:
|
||||
nodeip_ipv4: >-
|
||||
{{ (ansible_facts[interface].ipv4.address)
|
||||
if (ansible_facts is defined and ansible_facts.get(interface) is defined and ansible_facts[interface].ipv4 is defined)
|
||||
else (ansible_default_ipv4.address | default('')) }}
|
||||
nodeip_ipv6: >-
|
||||
{{ (ansible_facts[interface].ipv6[0].address)
|
||||
if (ansible_facts is defined and ansible_facts.get(interface) is defined and ansible_facts[interface].ipv6 is defined and (ansible_facts[interface].ipv6 | length) > 0)
|
||||
else (ansible_default_ipv6.address | default('')) }}
|
||||
|
||||
- block:
|
||||
- name: Read resolv.conf
|
||||
shell: "grep '^nameserver' /etc/resolv.conf"
|
||||
register: nameserver_line
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: Write /etc/resolv-k3s.conf with only nameservers
|
||||
copy:
|
||||
content: "{{ nameserver_line.stdout }}"
|
||||
dest: /etc/resolv-k3s.conf
|
||||
when: kubernetes.network.clean_resolv_conf is true
|
||||
|
||||
- name: Run handlers to reload configurations
|
||||
meta: flush_handlers
|
||||
@@ -0,0 +1,24 @@
|
||||
- name: Check if containerd-service exists & is started
|
||||
service:
|
||||
name: containerd
|
||||
state: started
|
||||
ignore_errors: true
|
||||
register: containerd_status
|
||||
|
||||
- name: Install containerd when not exists
|
||||
package:
|
||||
name:
|
||||
- containerd
|
||||
when: containerd_status is failed
|
||||
|
||||
- name: Create containerd config-folder
|
||||
file:
|
||||
path: /etc/containerd
|
||||
state: directory
|
||||
|
||||
- name: Deploy containerd-config
|
||||
ansible.builtin.copy:
|
||||
src: containerd_config.toml
|
||||
dest: /etc/containerd/config.toml
|
||||
mode: u=rw,g=r,o=r
|
||||
notify: restart_containerd
|
||||
@@ -0,0 +1,53 @@
|
||||
{% set disabled_components = [] %}
|
||||
|
||||
## Base ##
|
||||
node-name: {{ ansible_fqdn }}
|
||||
# Join cluster
|
||||
server: https://{{ hostvars[groups['kubernetes'][0]]['nodeip_ipv4'] }}:6443
|
||||
token: '{{ kubernetes.token }}'
|
||||
|
||||
# Networking
|
||||
{% if ansible_default_ipv6 is defined %}
|
||||
node-external-ip: {{ ansible_default_ipv4.address }},{{ ansible_default_ipv6.address }}
|
||||
{% else %}
|
||||
node-external-ip: {{ ansible_default_ipv4.address }}
|
||||
{% endif %}
|
||||
|
||||
{% if kubernetes.network.clean_resolv_conf %}
|
||||
resolv-conf: /etc/resolv-k3s.conf
|
||||
{% endif %}
|
||||
|
||||
{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %}
|
||||
node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }}
|
||||
{% else %}
|
||||
node-ip: {{ nodeip_ipv4 }}
|
||||
{% endif %}
|
||||
|
||||
# Network-plugin
|
||||
{% if kubernetes.network.plugin == "flannel" %}
|
||||
{% else %}
|
||||
{% if kubernetes.network.plugin == "cilium" %}
|
||||
# Cilium requires servicelb to be disabled when replacing kube-proxy
|
||||
{% set _ = disabled_components.append('servicelb') %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
## Label
|
||||
# Region & DC
|
||||
node-label:
|
||||
{% if region is defined %}
|
||||
- topology.kubernetes.io/region={{ region }}
|
||||
{% endif %}
|
||||
{% if zone is defined %}
|
||||
- topology.kubernetes.io/zone={{ zone }}
|
||||
{% endif %}
|
||||
{% for item in kubernetes.labels_extra %}
|
||||
- {{ item }}
|
||||
{% endfor %}
|
||||
|
||||
# Disabled components
|
||||
{% if disabled_components %}
|
||||
disable: {{ disabled_components | join(',') }}
|
||||
{% endif %}
|
||||
|
||||
{{ kubernetes.config_extra | to_yaml }}
|
||||
@@ -0,0 +1,91 @@
|
||||
{% set disabled_components = [] %}
|
||||
|
||||
## Base ##
|
||||
node-name: {{ ansible_fqdn }}
|
||||
{% if inventory_hostname == groups['kubernetes'][0] %}
|
||||
# Initialize with internal etcd
|
||||
cluster-init: true
|
||||
{% else %}
|
||||
# Join cluster
|
||||
server: https://{{ hostvars[groups['kubernetes'][0]]['nodeip_ipv4'] }}:6443
|
||||
{% endif %}
|
||||
|
||||
token: '{{ kubernetes.token }}'
|
||||
tls-san:
|
||||
- {{ kubernetes.control_plane.dns_name }}
|
||||
|
||||
# Networking
|
||||
{% if ansible_default_ipv6 is defined and ansible_default_ipv6.address is defined %}
|
||||
node-external-ip: {{ ansible_default_ipv4.address }},{{ ansible_default_ipv6.address }}
|
||||
{% else %}
|
||||
node-external-ip: {{ ansible_default_ipv4.address }}
|
||||
{% endif %}
|
||||
|
||||
{% if kubernetes.network.clean_resolv_conf %}
|
||||
resolv-conf: /etc/resolv-k3s.conf
|
||||
{% endif %}
|
||||
|
||||
{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %}
|
||||
node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }}
|
||||
cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }},{{ kubernetes.ipPool.ipv6.cluster_cidr }}
|
||||
service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }},{{ kubernetes.ipPool.ipv6.service_cidr }}
|
||||
{% else %}
|
||||
node-ip: {{ nodeip_ipv4 }}
|
||||
cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }}
|
||||
service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }}
|
||||
{% endif %}
|
||||
|
||||
# Pin the apiserver advertise-address to the mesh IP: node-external-ip above
|
||||
# otherwise gets picked, leaking a LAN-only address into the default/kubernetes
|
||||
# Endpoints object that other nodes cannot route to.
|
||||
advertise-address: {{ nodeip_ipv4 }}
|
||||
|
||||
egress-selector-mode: disabled
|
||||
|
||||
# Network-plugin
|
||||
{% if kubernetes.network.plugin == "flannel" %}
|
||||
flannel-backend: vxlan
|
||||
{% else %}
|
||||
disable-network-policy: true
|
||||
flannel-backend: none
|
||||
{% if kubernetes.network.plugin == "cilium" %}
|
||||
# Cilium requires servicelb to be disabled when replacing kube-proxy
|
||||
{% set _ = disabled_components.append('servicelb') %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
# Ingress-plugin
|
||||
{% if kubernetes.ingress_controller != "traefik-ingress" %}
|
||||
{% set _ = disabled_components.append('traefik') %}
|
||||
{% endif %}
|
||||
|
||||
## Label
|
||||
# Region & DC
|
||||
node-label:
|
||||
{% if region is defined %}
|
||||
- topology.kubernetes.io/region={{ region }}
|
||||
{% endif %}
|
||||
{% if zone is defined %}
|
||||
- topology.kubernetes.io/zone={{ zone }}
|
||||
{% endif %}
|
||||
{% if kubernetes.labels_extra is defined %}
|
||||
{% for item in kubernetes.labels_extra %}
|
||||
- {{ item }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
# Disabled components
|
||||
{% if disabled_components %}
|
||||
disable: {{ disabled_components | join(',') }}
|
||||
{% endif %}
|
||||
|
||||
{{ kubernetes.config_extra | to_yaml }}
|
||||
|
||||
# Image garbage collection tuning (drop-in KubeletConfiguration)
|
||||
kubelet-arg:
|
||||
- config-dir=/etc/rancher/k3s/kubelet.conf.d
|
||||
{% if not kubernetes.config_extra['disable-etcd'] | default(false) %}
|
||||
|
||||
# etcd members only
|
||||
{{ kubernetes.etcd_config_extra | to_nice_yaml(indent=2) }}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,34 @@
|
||||
# This section includes base Calico installation configuration.
|
||||
# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation
|
||||
apiVersion: operator.tigera.io/v1
|
||||
kind: Installation
|
||||
metadata:
|
||||
name: default
|
||||
spec:
|
||||
# Configures Calico networking.
|
||||
calicoNetwork:
|
||||
# Note: The ipPools section cannot be modified post-install.
|
||||
ipPools:
|
||||
- blockSize: 26
|
||||
cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }}
|
||||
encapsulation: None
|
||||
natOutgoing: Enabled
|
||||
nodeSelector: all()
|
||||
|
||||
{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %}
|
||||
- blockSize: 122
|
||||
cidr: {{ kubernetes.ipPool.ipv6.cluster_cidr }}
|
||||
encapsulation: None
|
||||
natOutgoing: Enabled
|
||||
nodeSelector: all()
|
||||
{% endif %}
|
||||
|
||||
---
|
||||
|
||||
# This section configures the Calico API server.
|
||||
# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
|
||||
apiVersion: operator.tigera.io/v1
|
||||
kind: APIServer
|
||||
metadata:
|
||||
name: default
|
||||
spec: {}
|
||||
@@ -0,0 +1,45 @@
|
||||
# https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: routingtabletowg
|
||||
namespace: calico-system
|
||||
labels:
|
||||
app: routingtabletowg
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: routingtabletowg
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: routingtabletowg
|
||||
spec:
|
||||
tolerations:
|
||||
# this toleration is to have the daemonset runnable on master nodes
|
||||
# remove it if your masters can't run pods
|
||||
- key: node-role.kubernetes.io/master
|
||||
effect: NoSchedule
|
||||
hostNetwork: true
|
||||
containers:
|
||||
- name: routingtabletowg
|
||||
image: "ruakij/routingtabletowg:0.2.0"
|
||||
env:
|
||||
- name: INTERFACE
|
||||
value: {{ kubernetes.ipPool.nodeIp_interface }}
|
||||
- name: FILTER_PROTOCOL
|
||||
value: bird
|
||||
- name: PERIODIC_SYNC
|
||||
value: '300'
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- NET_ADMIN
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 10Mi
|
||||
limits:
|
||||
cpu: 20m
|
||||
memory: 20Mi
|
||||
---
|
||||
@@ -0,0 +1 @@
|
||||
{{ kubernetes.registries_config | to_nice_yaml(indent=2) }}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user