Add pause & gather_facts at end when service was started

kubernetes/defaults/main.yml

@@ -2,16 +2,40 @@
 kubernetes:
   ipPool:
     ipv4:
+      # Minimum: /24
       cluster_cidr: 10.42.0.0/16
       service_cidr: 10.43.0.0/16
     ipv6:
+      # Minimum: /120
       cluster_cidr: fd42::/56
       service_cidr: fd43::/112
 
-    # Replace - with _
+    # Interface to grab node-IPv4/v6 from
     nodeIp_interface: <interface to grab nodeIp from>
 
   control_plane:
     dns_name: <control-plane dns-reachable-name>
 
   token: <shared token for nodes to join>
+
+  network:
+    # One of [flannel, calico]
+    plugin: calico
+
+    # Helper for networking
+    helper:
+      # https://github.com/Ruakij/RoutingTableToWg
+      #  Translates received-routes from e.g. BGP to wireguard-allowedips
+      #  Helpful, when nodeIp_interface is a wireguard-interface
+      routingtabletowg: false
+
+  # One of [traefik-ingress]
+  ingress_controller: traefik-ingress
+
+  config_extra:
+    # etcd-tuning
+    #  heartbeat: 0.5-1.5x of rtt
+    #  election:  10x- of heartbeat
+    etcd-arg:
+      heartbeat-interval: 500
+      election-timeout: 5000

kubernetes/meta/main.yml

@@ -1,4 +1,3 @@
 ---
   dependencies:
     - role: docker
-    - role: netmaker

kubernetes/tasks/install.yml

@@ -8,6 +8,7 @@
   ansible.builtin.template:
     src: k3s/{{ type }}/config.yaml.jinja2
     dest: /etc/rancher/k3s/config.yaml
+  register: config
 
 - name: Download install-script
   get_url:

kubernetes/tasks/install/agent/install_k3s.yml

@@ -5,3 +5,9 @@
   until: "command is not failed"
   retries: 2
   delay: 10
+
+- name: Restart when config changed, but install already done
+  service:
+    name: k3s
+    status: restarted
+  when: "inventory_hostname != groups['kubernetes'][0] and not command.changed and config.changed"

kubernetes/tasks/install/server/install_k3s.yml

@@ -4,6 +4,12 @@
   register: command
   changed_when: "'No change detected' in command.stdout"
 
+- name: Restart when config changed, but install already done
+  service:
+    name: k3s
+    status: restarted
+  when: "inventory_hostname == groups['kubernetes'][0] and not command.changed and config.changed"
+
 - name: Waiting for K3s-server to accept connections
   ansible.builtin.wait_for:
     host: "{{ inventory_hostname }}"
@@ -20,12 +26,19 @@
   retries: 2
   delay: 10
 
+- name: Restart when config changed, but install already done
+  service:
+    name: k3s
+    status: restarted
+  when: "inventory_hostname != groups['kubernetes'][0] and not command.changed and config.changed"
+
 - name: Waiting for K3s-server to accept connections on other nodes
   ansible.builtin.wait_for:
     host: "{{ inventory_hostname }}"
     port: 6443
     state: started
   when: "inventory_hostname != groups['kubernetes'][0]"
+
 #- name: Add Kubernetes environment-vars to /etc/profile.d/
 #  blockinfile:
 #    path: /etc/profile.d/k3s-bin.sh
@@ -33,3 +46,10 @@
 #    block: |
 #      export KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
 #    create: true
+
+- name: Deploy calico
+  import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_calico.yml
+  when: "kubernetes.network.plugin == 'calico'"
+
+- name: Deploy network-helpers
+  import_tasks: ./roles/kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml

kubernetes/tasks/install/server/network-plugin/deploy_calico.yml

@@ -0,0 +1,17 @@
+- name: Deploy calico operator
+  command: kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/tigera-operator.yaml
+  register: command
+  changed_when: "'created' in command.stdout"
+  run_once: true
+
+- name: Deploy calico ressource template
+  ansible.builtin.template:
+    src: ./k3s/server/network-plugin/calico/custom-ressource.yml.jinja2
+    dest: /root/calico-ressource.yml
+  run_once: true
+
+- name: Deploy calico ressource
+  command: kubectl apply -f /root/calico-ressource.yml
+  register: command
+  changed_when: "'created' in command.stdout"
+  run_once: true

kubernetes/tasks/install/server/network-plugin/deploy_network_helper.yml

@@ -0,0 +1,7 @@
+- name: Deploy service-file for routing-table to wireguard-translation
+  ansible.builtin.template:
+    src: ./k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2
+    dest: /var/lib/rancher/k3s/server/manifests/routingtabletowg.yml
+    mode: u=rw,g=r,o=r
+  run_once: true
+  when: "kubernetes.network.helper.routingtabletowg"

kubernetes/tasks/prerequisites.yml

@@ -17,19 +17,26 @@
 #- name: Disable swap
 #  command: swapoff -a
 
-#- name: Install iptables
+- name: Install required packages
-#  package:
+  package:
-#      name:
+      name:
-#          #- containerd
+          #- containerd
-#          - iptables
+          #- iptables
-#      state: latest
+          # For Longhorn:
+          - nfs-common
+          - open-iscsi
+      state: latest
 
 -  import_tasks: ./prerequisites/containerd.yml
 
+- name: Gather interface-name
+  set_fact:
+    interface: "{{ kubernetes.ipPool.nodeIp_interface | replace('-', '_') }}"
+
 - name: Getting nodeIp-data from interface
   set_fact:
-    nodeip_ipv4: "{{ ansible_facts[ kubernetes.ipPool.nodeIp_interface ].ipv4.address }}"
+    nodeip_ipv4: "{{ ansible_facts[ interface ].ipv4.address }}"
-    nodeip_ipv6: "{{ ansible_facts[ kubernetes.ipPool.nodeIp_interface ].ipv6[0].address }}"
+    nodeip_ipv6: "{{ ansible_facts[ interface ].ipv6[0].address if ansible_facts[ interface ].ipv6 is defined }}"
 
 - name: Run handlers to reload configurations
   meta: flush_handlers

kubernetes/templates/k3s/agent/config.yaml.jinja2

@@ -1,7 +1,21 @@
 server: https://{{ kubernetes.control_plane.dns_name }}:6443
-token: {{ kubernetes.token }}
+token: '{{ kubernetes.token }}'
 
+{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %}
 node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }} 
 
 # FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses
 kubelet-arg: "--node-ip=0.0.0.0"
+{% else %}
+node-ip: {{ nodeip_ipv4 }}
+{% endif %}
+
+## Label
+# Region & DC
+node-label:
+{% if region is defined %}
+  - topology.kubernetes.io/region={{ region }}
+{% endif %}
+{% if zone is defined %}
+  - topology.kubernetes.io/zone={{ zone }}
+{% endif %}

kubernetes/templates/k3s/server/config.yaml.jinja2

@@ -1,23 +1,52 @@
 ## Base ##
 {% if inventory_hostname == groups['kubernetes'][0] %}
+# Initialize with internal etcd
 cluster-init: true
 {% else %}
-server: https://{{ groups['kubernetes'][0] }}:6443
+server: https://{{ hostvars[groups['kubernetes'][0]]['nodeip_ipv4'] }}:6443
 {% endif %}
 
-token: {{ kubernetes.token }}
+token: '{{ kubernetes.token }}'
 tls-san:
   - {{ kubernetes.control_plane.dns_name }}
 
 # Networking
+{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %}
 node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }}
 cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }},{{ kubernetes.ipPool.ipv6.cluster_cidr }}
 service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }},{{ kubernetes.ipPool.ipv6.service_cidr }}  
 
+# FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses
+kubelet-arg: "--node-ip=0.0.0.0"
+{% else %}
+node-ip: {{ nodeip_ipv4 }}
+cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }}
+service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }}
+{% endif %}
+
 egress-selector-mode: disabled
 
 # Network-plugin
+{% if kubernetes.network_plugin == "flannel" %}
 flannel-backend: vxlan
+{% else %}
+disable-network-policy: true
+flannel-backend: none
+{% endif %}
 
-# FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses
+# Ingress-plugin
-kubelet-arg: "--node-ip=0.0.0.0"
+{% if kubernetes.ingress_controller != "traefik-ingress" %}
+disable: traefik
+{% endif %}
+
+## Label
+# Region & DC
+node-label:
+{% if region is defined %}
+  - topology.kubernetes.io/region={{ region }}
+{% endif %}
+{% if zone is defined %}
+  - topology.kubernetes.io/zone={{ zone }}
+{% endif %}
+
+{{ kubernetes.config_extra | to_yaml }}

kubernetes/templates/k3s/server/network-plugin/calico/custom-ressource.yml.jinja2

@@ -0,0 +1,34 @@
+# This section includes base Calico installation configuration.
+# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+    - blockSize: 26
+      cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }}
+      encapsulation: None
+      natOutgoing: Enabled
+      nodeSelector: all()
+
+{% if nodeip_ipv6 != "" and kubernetes.ipPool.ipv6 is defined %}
+    - blockSize: 122
+      cidr: {{ kubernetes.ipPool.ipv6.cluster_cidr }}
+      encapsulation: None
+      natOutgoing: Enabled
+      nodeSelector: all()
+{% endif %}
+
+---
+
+# This section configures the Calico API server.
+# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
+apiVersion: operator.tigera.io/v1
+kind: APIServer 
+metadata: 
+  name: default 
+spec: {}

kubernetes/templates/k3s/server/network-plugin/calico/routingtabletowg.yml.jinja2

@@ -0,0 +1,45 @@
+# https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: routingtabletowg
+  namespace: calico-system
+  labels:
+    app: routingtabletowg
+spec:
+  selector:
+    matchLabels:
+      app: routingtabletowg
+  template:
+    metadata:
+      labels:
+        app: routingtabletowg
+    spec:
+      tolerations:
+      # this toleration is to have the daemonset runnable on master nodes
+      # remove it if your masters can't run pods
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      hostNetwork: true
+      containers:
+        - name: routingtabletowg
+          image: "ruakij/routingtabletowg:0.2.0"
+          env:
+            - name: INTERFACE
+              value: {{ kubernetes.ipPool.nodeIp_interface }}
+            - name: FILTER_PROTOCOL
+              value: bird
+            - name: PERIODIC_SYNC
+              value: '300'
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+          resources:
+            requests:
+              cpu: 10m
+              memory: 10Mi
+            limits:
+              cpu: 20m
+              memory: 20Mi
+---

netbird_client/defaults/netbird_client.yml

@@ -0,0 +1,6 @@
+netbird_client:
+  # Key and url to join a network
+  #  leave empty to ignore
+  join_network:
+    setup_key: 
+    management_url: 

netbird_client/tasks/install.yml

@@ -0,0 +1,26 @@
+- name: Install Packages
+#  when: docker_file.stat.exists == False
+  package:
+    name:
+      - ca-certificates
+      - curl
+      - gnupg
+
+- name: Add netbird-key
+  apt_key:
+    url: https://pkgs.wiretrustee.com/debian/public.key
+    state: present
+
+- name: Add netbird-repository
+  apt_repository:
+    repo: "deb https://pkgs.wiretrustee.com/debian stable main"
+    state: present
+    filename: netbird
+    update_cache: yes
+
+- name: Install wireguard & netbird
+  package:
+    name:
+      - wireguard
+      - netbird
+    state: latest

netbird_client/tasks/join-network.yml

@@ -0,0 +1,16 @@
+- name: Join netbird-network
+  when: "netbird_client.join_network.setup_key is defined"
+  command: "netbird up --management-url {{ netbird_client.join_network.management_url }} --setup-key {{ netbird_client.join_network.setup_key }}"
+  failed_when: command.rc != 0
+  changed_when: "'Connected' in command.stdout"
+  register: command
+
+- name: Wait for netbird-interface to exist
+  wait_for:
+    path: "/sys/class/net/wt0"
+    state: present
+  when: command.changed
+
+- name: Gather facts to get changes
+  ansible.builtin.gather_facts:
+  when: command.changed

netbird_client/tasks/main.yml

@@ -0,0 +1,4 @@
+- import_tasks: ./install.yml
+
+- import_tasks: ./join-network.yml
+

nomad/defaults/main.yml

@@ -1,3 +0,0 @@
----
-nomad:
-  version: 1.4.4

nomad/files/systemd-service

@@ -1,48 +0,0 @@
-[Unit]
-Description=Nomad
-Documentation=https://www.nomadproject.io/docs/
-Wants=network-online.target
-After=network-online.target
-
-# When using Nomad with Consul it is not necessary to start Consul first. These
-# lines start Consul before Nomad as an optimization to avoid Nomad logging
-# that Consul is unavailable at startup.
-#Wants=consul.service
-#After=consul.service
-
-[Service]
-
-# Nomad server should be run as the nomad user. Nomad clients
-# should be run as root
-User=root
-Group=root
-
-ExecReload=/bin/kill -HUP $MAINPID
-ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d
-KillMode=process
-KillSignal=SIGINT
-LimitNOFILE=65536
-LimitNPROC=infinity
-Restart=on-failure
-RestartSec=2
-
-## Configure unit start rate limiting. Units which are started more than
-## *burst* times within an *interval* time span are not permitted to start any
-## more. Use `StartLimitIntervalSec` or `StartLimitInterval` (depending on
-## systemd version) to configure the checking interval and `StartLimitBurst`
-## to configure how many starts per interval are allowed. The values in the
-## commented lines are defaults.
-
-# StartLimitBurst = 5
-
-## StartLimitIntervalSec is used for systemd versions >= 230
-# StartLimitIntervalSec = 10s
-
-## StartLimitInterval is used for systemd versions < 230
-# StartLimitInterval = 10s
-
-TasksMax=infinity
-OOMScoreAdjust=-1000
-
-[Install]
-WantedBy=multi-user.target

nomad/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  #- role: docker

nomad/tasks/install.yml

@@ -1,43 +0,0 @@
-- name: Download binary
-  ansible.builtin.unarchive:
-    remote_src: true
-    src: https://releases.hashicorp.com/nomad/{{ nomad.version }}/nomad_{{ nomad.version }}_{{ ansible_system | lower }}_{{ 'amd64' if ansible_architecture == 'x86_64' else ansible_architecture }}.zip
-    dest: /usr/local/bin/
-    mode: "755"
-
-- name: Deploy systemd-service file
-  ansible.builtin.copy:
-    src: systemd-service
-    dest: /etc/systemd/system/nomad.service
-    mode: u=rw,g=r,o=r
-
-- name: Create nomad user
-  ansible.builtin.user:
-    name: nomad
-    groups:
-      - docker
-    append: true
-
-- name: Create directory for configs
-  ansible.builtin.file:
-    path: /etc/nomad.d
-    state: directory
-    mode: "0755"
-    owner: "nomad"
-    group: "nomad"
-
-- name: Create nomad.hcl configuration file
-  ansible.builtin.template:
-    src: nomad.hcl.j2
-    dest: /etc/nomad.d/nomad.hcl
-    mode: "0644"
-    owner: "nomad"
-    group: "nomad"
-
-- name: Create directory for data
-  ansible.builtin.file:
-    path: /opt/nomad
-    state: directory
-    mode: "0755"
-    owner: "nomad"
-    group: "nomad"

nomad/tasks/launch.yml

@@ -1,8 +0,0 @@
-- name: Start service
-  ansible.builtin.service:
-    name: nomad
-    state: restarted
-
-- name: Waiting for service to accept connections
-  ansible.builtin.wait_for:
-    port: 4646

nomad/tasks/main.yml

@@ -1,3 +0,0 @@
-- import_tasks: ./install.yml
-
-- import_tasks: ./launch.yml

nomad/templates/nomad.hcl.j2

@@ -1,71 +0,0 @@
-data_dir = "/opt/nomad"
-datacenter = "{{ datacenter }}"
-
-
-bind_addr = "0.0.0.0"
-
-advertise {
-  # Defaults to the first private IP address.
-  #http = "1.2.3.4"
-  #rpc  = "1.2.3.4"
-  #serf = "1.2.3.4:5648" # non-default ports may be specified
-}
-
-{# TODO: Get interface-ip from hosts marked with type=server #}
-{% set server_hosts = ansible_play_batch | difference([inventory_hostname]) %}
-{% if type is defined and type == "server" %}
-server {
-  enabled = true
-  bootstrap_expect = {{ server_hosts | length }}
-
-  server_join {
-    retry_join = [ "{{ server_hosts | join('", "') }}" ]
-    retry_max = 6
-    retry_interval = "15s"
-  }
-
-  default_scheduler_config {
-    scheduler_algorithm             = "binpack"
-    memory_oversubscription_enabled = true
-    reject_job_registration         = false
-    pause_eval_broker               = false # New in Nomad 1.3.2
-
-    preemption_config {
-      batch_scheduler_enabled    = true
-      system_scheduler_enabled   = true
-      service_scheduler_enabled  = true
-      sysbatch_scheduler_enabled = true # New in Nomad 1.2
-    }
-  }
-}
-{% endif %}
-
-client {
-  enabled = true
-
-  {% if type != "server" %}
-  servers = [ "{{ server_hosts | join('", "') }}" ]
-  {% endif %}
-
-  meta {
-    node_type = "{{ type }}"
-    {% if storage is defined and storage %}
-    seaweedfs_volume = "true"
-    {% endif %}
-  }
-}
-
-
-plugin "raw_exec" {
-  config {
-    enabled = true
-  }
-}
-
-plugin "docker" {
-  config {
-    {% if type is defined and type == "server" %}
-    allow_privileged = true
-    {% endif %}
-  }
-}

wireguard-ipv6-converter/defaults/wireguard-ipv6-converter.yml

@@ -0,0 +1,12 @@
+wireguard_ipv6_converter:
+  version: latest
+
+  # see https://github.com/Ruakij/wg-ipv6-converter#31-environment
+  setup:
+    interface: wg0
+    #ipv6_format: fc12::%02x%02x:%02x%02x/%d
+    #filter_prefix: 100.100
+    #recheck_interval: 60s
+
+  service:
+    #bindTo: netbird.service

wireguard-ipv6-converter/tasks/deploy.yml

@@ -0,0 +1,11 @@
+- name: Get architecture
+  set_fact:
+    arch: "{{ 'amd64' if ansible_architecture == 'x86_64' else 'arm64' }}"
+    versionUri: "{% if wireguard_ipv6_converter.version == 'latest' %}latest/download{% else %}download/{{ wireguard_ipv6_converter.version }}{% endif %}"
+
+- name: Download binary
+  get_url:
+    url: https://github.com/Ruakij/wg-ipv6-converter/releases/{{ versionUri }}/wg-ipv6-converter_{{ arch }}
+    dest: /usr/local/bin/wg-ipv6-converter
+    mode: "744"
+  register: deployDownload

wireguard-ipv6-converter/tasks/main.yml

@@ -0,0 +1,3 @@
+- import_tasks: ./deploy.yml
+
+- import_tasks: ./setup-service.yml

wireguard-ipv6-converter/tasks/setup-service.yml

@@ -0,0 +1,27 @@
+- name: Deploy service
+  ansible.builtin.template:
+    src: wg-ipv6-conv.service.jinja2
+    dest: /etc/systemd/system/wg-ipv6-converter_{{ wireguard_ipv6_converter.setup.interface }}.service
+  register: serviceFile
+
+- name: Enable service
+  ansible.builtin.service:
+    name: wg-ipv6-converter_{{ wireguard_ipv6_converter.setup.interface }}
+    daemon-reload: true
+    enabled: true
+
+- name: Start service if interface exists already
+  ansible.builtin.service:
+    name: wg-ipv6-converter_{{ wireguard_ipv6_converter.setup.interface }}
+    state: "{{ 'restarted' if deployDownload.changed or serviceFile.changed else 'started' }}"
+  register: service
+  when: "wireguard_ipv6_converter.setup.interface in ansible_interfaces"
+
+- name: Pause for 5s to wait for program to have run
+  ansible.builtin.pause:
+    seconds: 5
+  when: "service.changed"
+
+- name: Gather facts to get changes
+  ansible.builtin.gather_facts:
+  when: "service.changed"

wireguard-ipv6-converter/templates/wg-ipv6-conv.service.jinja2

@@ -0,0 +1,29 @@
+[Unit]
+Description=WireGuard IPv6 converter for {{ wireguard_ipv6_converter.setup.interface }}
+{% if wireguard_ipv6_converter.service.bindTo is defined %}
+BindsTo={{ wireguard_ipv6_converter.service.bindTo }}
+After={{ wireguard_ipv6_converter.service.bindTo }}
+{% endif %}
+
+[Service]
+Type=simple
+{% if wireguard_ipv6_converter.service.bindTo is defined %}
+ExecStartPre=/bin/sleep 10
+{% endif %}
+ExecStart=/usr/local/bin/wg-ipv6-converter
+Restart=always
+RestartSec=30
+
+Environment="INTERFACE={{ wireguard_ipv6_converter.setup.interface }}"
+{% if wireguard_ipv6_converter.setup.ipv6_format is defined %}
+Environment="IPV6_FORMAT={{ wireguard_ipv6_converter.setup.ipv6_format }}"
+{% endif %}
+{% if wireguard_ipv6_converter.setup.filter_prefix is defined %}
+Environment="FILTER_PREFIX={{ wireguard_ipv6_converter.setup.filter_prefix }}"
+{% endif %}
+{% if wireguard_ipv6_converter.setup.recheck_interval is defined %}
+Environment="RECHECK_INTERVAL={{ wireguard_ipv6_converter.setup.recheck_interval }}"
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target