diff --git a/kubernetes/defaults/main.yml b/kubernetes/defaults/main.yml new file mode 100644 index 0000000..57e82a4 --- /dev/null +++ b/kubernetes/defaults/main.yml @@ -0,0 +1,17 @@ +--- +kubernetes: + ipPool: + ipv4: + cluster_cidr: 10.42.0.0/16 + service_cidr: 10.43.0.0/16 + ipv6: + cluster_cidr: fd42::/56 + service_cidr: fd43::/112 + + # Replace - with _ + nodeIp_interface: + + control_plane: + dns_name: + + token: diff --git a/kubernetes/docs/architecture.puml b/kubernetes/docs/architecture.puml new file mode 100644 index 0000000..8cfc894 --- /dev/null +++ b/kubernetes/docs/architecture.puml @@ -0,0 +1,33 @@ +@startuml + +rectangle "Control-Plane" as control_plane { + rectangle "Node" as sn1 { + component "netclient" as sn1_netclient + + component etcd as sn1_etcd + component "k3s-server" as sn1_k3s_server + sn1_k3s_server - sn1_etcd + } + + rectangle "Node" as sn2 { + component "netclient" as sn2_netclient + + component etcd as sn2_etcd + component "k3s-server" as sn2_k3s_server + sn2_k3s_server - sn2_etcd + } + + sn1_netclient -- sn2_netclient + sn1_etcd -- sn2_etcd +} + +rectangle "Workers" { + rectangle "Node" as an1 { + component "netclient" as an1_netclient + + component "k3s-agent" as sn1_k3s_agent + } +} + + +@enduml diff --git a/kubernetes/files/containerd_config.toml b/kubernetes/files/containerd_config.toml new file mode 100644 index 0000000..c324a4b --- /dev/null +++ b/kubernetes/files/containerd_config.toml @@ -0,0 +1,35 @@ +# Copyright 2018-2022 Docker Inc. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +disabled_plugins = [] + +#root = "/var/lib/containerd" +#state = "/run/containerd" +#subreaper = true +#oom_score = 0 + +#[grpc] +# address = "/run/containerd/containerd.sock" +# uid = 0 +# gid = 0 + +#[debug] +# address = "/run/containerd/debug.sock" +# uid = 0 +# gid = 0 +# level = "info" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true diff --git a/kubernetes/handlers/main.yml b/kubernetes/handlers/main.yml new file mode 100644 index 0000000..5b55d71 --- /dev/null +++ b/kubernetes/handlers/main.yml @@ -0,0 +1,19 @@ +- name: reload_sysctl + command: sysctl --system + +- name: restart_containerd + ansible.builtin.service: + name: containerd + state: restarted + +- name: reload_networking + service: + name: networking + state: restarted + async: 5 + poll: 0 + notify: wait_for_connection + +- name: wait_for_connection + wait_for_connection: + delay: 5 diff --git a/kubernetes/meta/main.yml b/kubernetes/meta/main.yml new file mode 100644 index 0000000..613d878 --- /dev/null +++ b/kubernetes/meta/main.yml @@ -0,0 +1,4 @@ +--- + dependencies: + - role: docker + - role: netmaker diff --git a/kubernetes/tasks/install.yml b/kubernetes/tasks/install.yml new file mode 100644 index 0000000..d06ac15 --- /dev/null +++ b/kubernetes/tasks/install.yml @@ -0,0 +1,29 @@ +- name: Create k3s-folder + ansible.builtin.file: + path: /etc/rancher/k3s/ + state: directory + mode: '0755' + +- name: Deploy k3s config + ansible.builtin.template: + src: k3s/{{ type }}/config.yaml.jinja2 + dest: /etc/rancher/k3s/config.yaml + +- name: Download install-script + get_url: + url: https://get.k3s.io + dest: /root/k3s_install.sh + mode: '744' +# todo: update when file changed? + +- import_tasks: ./install/server/setup_network.yml + when: "type == 'server'" + +- import_tasks: ./install/server/install_helm.yml + when: "type == 'server'" + +- import_tasks: ./install/server/install_k3s.yml + when: "type == 'server'" + +- import_tasks: ./install/agent/install_k3s.yml + when: "type == 'agent'" diff --git a/kubernetes/tasks/install/agent/install_k3s.yml b/kubernetes/tasks/install/agent/install_k3s.yml new file mode 100644 index 0000000..b465095 --- /dev/null +++ b/kubernetes/tasks/install/agent/install_k3s.yml @@ -0,0 +1,7 @@ +- name: Install K3s agent + command: /root/k3s_install.sh {{ type }} + register: command + changed_when: "'No change detected' in command.stdout" + until: "command is not failed" + retries: 2 + delay: 10 diff --git a/kubernetes/tasks/install/server/install_helm.yml b/kubernetes/tasks/install/server/install_helm.yml new file mode 100644 index 0000000..0a5a361 --- /dev/null +++ b/kubernetes/tasks/install/server/install_helm.yml @@ -0,0 +1,17 @@ +- name: Add Balto key + apt_key: + url: https://baltocdn.com/helm/signing.asc + state: present + +- name: Add Balto Repository + apt_repository: + repo: "deb https://baltocdn.com/helm/stable/debian/ all main" + state: present + filename: kubernetes + update_cache: yes + +- name: Install helm + package: + name: + - helm + state: latest diff --git a/kubernetes/tasks/install/server/install_k3s.yml b/kubernetes/tasks/install/server/install_k3s.yml new file mode 100644 index 0000000..e060725 --- /dev/null +++ b/kubernetes/tasks/install/server/install_k3s.yml @@ -0,0 +1,36 @@ +- name: Install K3s-server for 1st-node + command: /root/k3s_install.sh {{ type }} + when: "inventory_hostname == groups['kubernetes'][0]" + register: command + changed_when: "'No change detected' in command.stdout" + +- name: Waiting for K3s-server to accept connections + ansible.builtin.wait_for: + host: "{{ inventory_hostname }}" + port: 6443 + state: started + when: "inventory_hostname == groups['kubernetes'][0]" + +- name: Install K3s-server for other nodes + command: /root/k3s_install.sh {{ type }} + when: "inventory_hostname != groups['kubernetes'][0]" + register: command + changed_when: "'No change detected' in command.stdout" + until: "command is not failed" + retries: 2 + delay: 10 + +- name: Waiting for K3s-server to accept connections on other nodes + ansible.builtin.wait_for: + host: "{{ inventory_hostname }}" + port: 6443 + state: started + when: "inventory_hostname != groups['kubernetes'][0]" + +#- name: Add Kubernetes environment-vars to /etc/profile.d/ +# blockinfile: +# path: /etc/profile.d/k3s-bin.sh +# marker: "# {mark} ANSIBLE MANAGED BLOCK | k3s" +# block: | +# export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" +# create: true diff --git a/kubernetes/tasks/install/server/setup_network.yml b/kubernetes/tasks/install/server/setup_network.yml new file mode 100644 index 0000000..ac197d7 --- /dev/null +++ b/kubernetes/tasks/install/server/setup_network.yml @@ -0,0 +1,6 @@ +- name: Set control-plane-dns-endpoint towards local-ip + blockinfile: + path: /etc/hosts + marker: "# {mark} ANSIBLE MANAGED BLOCK | k3s" + block: | + {{ nodeip_ipv4 }} {{ kubernetes.control_plane.dns_name }} diff --git a/kubernetes/tasks/main.yml b/kubernetes/tasks/main.yml new file mode 100644 index 0000000..7574953 --- /dev/null +++ b/kubernetes/tasks/main.yml @@ -0,0 +1,4 @@ +- import_tasks: ./prerequisites.yml + +- import_tasks: ./install.yml + diff --git a/kubernetes/tasks/prerequisites.yml b/kubernetes/tasks/prerequisites.yml new file mode 100644 index 0000000..5ac2bae --- /dev/null +++ b/kubernetes/tasks/prerequisites.yml @@ -0,0 +1,35 @@ +#- name: Load br_netfilter kernel-module +# modprobe: +# name: br_netfilter +# state: present + +- name: Set sysctl settings for iptables bridged traffic + copy: + dest: "/etc/sysctl.d/kubernetes.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + + net.ipv4.conf.all.forwarding=1 + net.ipv6.conf.all.forwarding=1 + notify: reload_sysctl + +#- name: Disable swap +# command: swapoff -a + +#- name: Install iptables +# package: +# name: +# #- containerd +# - iptables +# state: latest + +- import_tasks: ./prerequisites/containerd.yml + +- name: Getting nodeIp-data from interface + set_fact: + nodeip_ipv4: "{{ ansible_facts[ kubernetes.ipPool.nodeIp_interface ].ipv4.address }}" + nodeip_ipv6: "{{ ansible_facts[ kubernetes.ipPool.nodeIp_interface ].ipv6[0].address }}" + +- name: Run handlers to reload configurations + meta: flush_handlers diff --git a/kubernetes/tasks/prerequisites/containerd.yml b/kubernetes/tasks/prerequisites/containerd.yml new file mode 100644 index 0000000..6ff10b5 --- /dev/null +++ b/kubernetes/tasks/prerequisites/containerd.yml @@ -0,0 +1,24 @@ +- name: Check if containerd-service exists & is started + service: + name: containerd + state: started + ignore_errors: true + register: containerd_status + +- name: Install containerd when not exists + package: + name: + - containerd + when: containerd_status is failed + +- name: Create containerd config-folder + file: + path: /etc/containerd + state: directory + +- name: Deploy containerd-config + ansible.builtin.copy: + src: containerd_config.toml + dest: /etc/containerd/config.toml + mode: u=rw,g=r,o=r + notify: restart_containerd diff --git a/kubernetes/templates/k3s/agent/config.yaml.jinja2 b/kubernetes/templates/k3s/agent/config.yaml.jinja2 new file mode 100644 index 0000000..417f719 --- /dev/null +++ b/kubernetes/templates/k3s/agent/config.yaml.jinja2 @@ -0,0 +1,7 @@ +server: https://{{ kubernetes.control_plane.dns_name }}:6443 +token: {{ kubernetes.token }} + +node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }} + +# FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses +kubelet-arg: "--node-ip=0.0.0.0" diff --git a/kubernetes/templates/k3s/server/config.yaml.jinja2 b/kubernetes/templates/k3s/server/config.yaml.jinja2 new file mode 100644 index 0000000..134ac05 --- /dev/null +++ b/kubernetes/templates/k3s/server/config.yaml.jinja2 @@ -0,0 +1,23 @@ +## Base ## +{% if inventory_hostname == groups['kubernetes'][0] %} +cluster-init: true +{% else %} +server: https://{{ groups['kubernetes'][0] }}:6443 +{% endif %} + +token: {{ kubernetes.token }} +tls-san: + - {{ kubernetes.control_plane.dns_name }} + +# Networking +node-ip: {{ nodeip_ipv4 }},{{ nodeip_ipv6 }} +cluster-cidr: {{ kubernetes.ipPool.ipv4.cluster_cidr }},{{ kubernetes.ipPool.ipv6.cluster_cidr }} +service-cidr: {{ kubernetes.ipPool.ipv4.service_cidr }},{{ kubernetes.ipPool.ipv6.service_cidr }} + +egress-selector-mode: disabled + +# Network-plugin +flannel-backend: vxlan + +# FIXME: Workaround for bug in Kubernetes 1.24/1.25 ignoring node IPv6 addresses +kubelet-arg: "--node-ip=0.0.0.0"