Add a kopia_backup role for the daily Kopia snapshot script and timer against an existing S3 repository

kopia_backup/defaults/main.yml

@@ -0,0 +1,28 @@
+---
+kopia_backup:
+  # S3 repository; it is created, and its policies and maintenance are set, outside this role.
+  endpoint: tier1.backups.ruekov.eu
+  bucket: ""
+  access_key: ""
+  secret_key: ""
+  password: ""
+  # Snapshot owner is root@<hostname>; several hosts can share one repository.
+  hostname: "{{ inventory_hostname_short }}"
+  # Kopia defaults when unset.
+  content_cache_size_mb:
+  metadata_cache_size_mb:
+  # Paths to snapshot; missing ones are skipped at run time.
+  sources: []
+  # source path -> ignore rules, replacing the policy rules on every run (null drops one).
+  ignore: {}
+  # Database dumps written to /var/backups/dumps before the snapshot, which then includes that directory.
+  # Each item: file, container, command (run with sh -c inside the container, output is the dump).
+  dumps: []
+  # Snapshot the newest k3s etcd snapshot, staged in /var/backups/etcd.
+  etcd_snapshot: false
+  on_calendar: "*-*-* 03:30"
+  randomized_delay: ""
+  # Extra units the service starts after, e.g. docker.service for dumps.
+  after: []
+  # Nice=10 and idle IO scheduling.
+  idle_priority: true

kopia_backup/tasks/main.yml

@@ -0,0 +1,90 @@
+---
+- name: Require the repository settings
+  ansible.builtin.assert:
+    that:
+      - kopia_backup.bucket | length > 0
+      - kopia_backup.access_key | length > 0
+      - kopia_backup.secret_key | length > 0
+      - kopia_backup.password | length > 0
+      - kopia_backup.sources | length > 0
+    fail_msg: kopia_backup.bucket, access_key, secret_key, password and sources must be set
+
+- name: Create /etc/apt/keyrings
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
+- name: Add the Kopia apt key
+  ansible.builtin.shell: curl -fsSL https://kopia.io/signing-key | gpg --dearmor -o /etc/apt/keyrings/kopia-keyring.gpg
+  args:
+    creates: /etc/apt/keyrings/kopia-keyring.gpg
+
+- name: Add the Kopia apt repository
+  ansible.builtin.copy:
+    dest: /etc/apt/sources.list.d/kopia.list
+    mode: "0644"
+    content: |
+      deb [signed-by=/etc/apt/keyrings/kopia-keyring.gpg] http://packages.kopia.io/apt/ stable main
+  register: kopia_backup_repo
+
+- name: Install Kopia
+  ansible.builtin.apt:
+    name: kopia
+    state: present
+    update_cache: "{{ kopia_backup_repo is changed }}"
+
+- name: Read the repository connection
+  ansible.builtin.command: kopia repository status
+  environment:
+    KOPIA_CHECK_FOR_UPDATES: "false"
+  register: kopia_backup_status
+  changed_when: false
+  failed_when: false
+  check_mode: false
+
+- name: Connect the repository
+  ansible.builtin.command:
+    argv: "{{ argv + (cache if kopia_backup.content_cache_size_mb else []) }}"
+  vars:
+    argv:
+      - kopia
+      - repository
+      - connect
+      - s3
+      - --endpoint={{ kopia_backup.endpoint }}
+      - --bucket={{ kopia_backup.bucket }}
+      - --override-hostname={{ kopia_backup.hostname }}
+      - --override-username=root
+    cache:
+      - --content-cache-size-mb={{ kopia_backup.content_cache_size_mb }}
+      - --metadata-cache-size-mb={{ kopia_backup.metadata_cache_size_mb }}
+  # Secrets through the environment, flags would show them in the process list
+  environment:
+    AWS_ACCESS_KEY_ID: "{{ kopia_backup.access_key }}"
+    AWS_SECRET_ACCESS_KEY: "{{ kopia_backup.secret_key }}"
+    KOPIA_PASSWORD: "{{ kopia_backup.password }}"
+    KOPIA_CHECK_FOR_UPDATES: "false"
+  when: kopia_backup_status.rc != 0
+  no_log: true
+
+- name: Install the backup script
+  ansible.builtin.template:
+    src: kopia-backup.j2
+    dest: /usr/local/sbin/kopia-backup
+    mode: "0755"
+
+- name: Install the systemd units
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop: [kopia-backup.service, kopia-backup.timer]
+  register: kopia_backup_units
+
+- name: Enable the timer
+  ansible.builtin.systemd:
+    name: kopia-backup.timer
+    enabled: true
+    state: started
+    daemon_reload: "{{ kopia_backup_units is changed }}"

kopia_backup/templates/kopia-backup.j2

@@ -0,0 +1,54 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Daily Kopia backup of {{ kopia_backup.hostname }} into the bucket {{ kopia_backup.bucket }} on {{ kopia_backup.endpoint }}.
+set -uo pipefail
+export KOPIA_CHECK_FOR_UPDATES=false
+failed=0
+sources=()
+{% if kopia_backup.etcd_snapshot %}
+
+# Only the latest etcd snapshot; the live etcd dir is not consistent on disk and older snapshots only add churn
+etcd=/var/backups/etcd
+latest=$(ls -t /var/lib/rancher/k3s/server/db/snapshots/etcd-snapshot-* 2>/dev/null | head -1)
+if [ -n "$latest" ]; then
+  install -d -m700 "$etcd"
+  rm -f "$etcd/etcd-snapshot"
+  ln "$latest" "$etcd/etcd-snapshot" 2>/dev/null || cp -p "$latest" "$etcd/etcd-snapshot" || failed=1
+fi
+{% endif %}
+{% if kopia_backup.dumps %}
+
+dumps=/var/backups/dumps
+install -d -m700 "$dumps"
+dump() { # file, container, command run inside the container
+  if docker exec "$2" sh -c "$3" > "$dumps/$1.tmp"; then
+    mv "$dumps/$1.tmp" "$dumps/$1"
+  else
+    echo "dump of $1 failed" >&2; rm -f "$dumps/$1.tmp"; failed=1
+  fi
+}
+{% for d in kopia_backup.dumps %}
+dump {{ d.file | quote }} {{ d.container | quote }} {{ d.command | quote }}
+{% endfor %}
+{% endif %}
+{% set ignores = kopia_backup.ignore | dict2items | rejectattr('value', 'none') | list %}
+{% if ignores %}
+
+# Ignore rules of a source, replaced as a whole (--clear-ignore runs after --add-ignore within one call).
+# Kopia sorts the rules, so negations (!) cannot be relied on to come after the rule they override.
+ignore() {
+  local path=$1; shift
+  kopia policy set "$path" --clear-ignore >/dev/null 2>&1 &&
+    kopia policy set "$path" "${@/#/--add-ignore=}" >/dev/null 2>&1 ||
+    { echo "setting the ignore rules of $path failed" >&2; failed=1; }
+}
+{% for i in ignores %}
+ignore {{ i.key | quote }} {{ i.value | map('quote') | join(' ') }}
+{% endfor %}
+{% endif %}
+
+for p in {{ kopia_backup.sources | map('quote') | join(' ') }}{{ ' "$etcd"' if kopia_backup.etcd_snapshot }}{{ ' "$dumps"' if kopia_backup.dumps }}; do
+  [ -d "$p" ] && sources+=("$p")
+done
+kopia snapshot create "${sources[@]}" || failed=1
+exit $failed

kopia_backup/templates/kopia-backup.service.j2

@@ -0,0 +1,14 @@
+# {{ ansible_managed }}
+[Unit]
+Description=Kopia backup to {{ kopia_backup.endpoint }}
+After={{ (['network-online.target'] + kopia_backup.after) | join(' ') }}
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=HOME=/root
+{% if kopia_backup.idle_priority %}
+Nice=10
+IOSchedulingClass=idle
+{% endif %}
+ExecStart=/usr/local/sbin/kopia-backup

kopia_backup/templates/kopia-backup.timer.j2

@@ -0,0 +1,13 @@
+# {{ ansible_managed }}
+[Unit]
+Description=Daily Kopia backup to {{ kopia_backup.endpoint }}
+
+[Timer]
+OnCalendar={{ kopia_backup.on_calendar }}
+{% if kopia_backup.randomized_delay %}
+RandomizedDelaySec={{ kopia_backup.randomized_delay }}
+{% endif %}
+Persistent=true
+
+[Install]
+WantedBy=timers.target