feat: allow custom cni setups
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Robert Kaussow 2022-10-05 22:54:36 +02:00
parent cdf71aec07
commit f9edbc0f2a
Signed by: xoxys
GPG Key ID: 4E692A2EAECC03C0
12 changed files with 108 additions and 24 deletions

View File

@ -10,10 +10,6 @@ k3s_token: "secure-token"
k3s_node_ip: "{{ ansible_default_ipv4.address }}"
k3s_init_log_enabled: False
k3s_config_dir: /etc/rancher/k3s
k3s_data_dir: /var/lib/rancher/k3s
k3s_log_dir: /var/log/rancher/k3s
k3s_server: True
k3s_server_bind_ip: "0.0.0.0"
k3s_server_nodes:
@ -22,6 +18,9 @@ k3s_server_flannel_backend_enabled: True
k3s_server_network_policy_enabled: True
k3s_server_feature_gates: []
k3s_server_manifests_templates: []
k3s_server_manifests_urls: []
k3s_server_admission_plugins:
- NodeRestriction
- EventRateLimit

View File

@ -7,6 +7,13 @@
- container-selinux
- selinux-policy-base
- https://github.com/k3s-io/k3s-selinux/releases/download/v1.2.stable.2/k3s-selinux-1.2-2.el8.noarch.rpm
k3s_server_flannel_backend_enabled: False
k3s_server_network_policy_enabled: False
k3s_server_manifests_templates:
- "calico-installation.yaml.j2"
k3s_server_manifests_urls:
- url: https://docs.projectcalico.org/archive/v3.24/manifests/tigera-operator.yaml
dest: tigera-operator.yaml
k3s_init_log_enabled: True
pre_tasks:
- name: Override host variables

View File

@ -0,0 +1,15 @@
---
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
name: default
spec:
calicoNetwork:
ipPools:
- blockSize: 26
cidr: "10.42.0.0/16"
encapsulation: "VXLANCrossSubnet"
natOutgoing: Enabled
nodeSelector: all()
nodeMetricsPort: 9091
typhaMetricsPort: 9093

View File

@ -1,3 +1,4 @@
import json
import os
import testinfra.utils.ansible_runner
@ -24,3 +25,9 @@ def test_k3s_cluster_ready(host):
cluster = host.run("kubectl get --raw='/readyz'").stdout
assert cluster == "ok"
def test_k3s_cni(host):
cni = json.loads(host.file("/etc/cni/net.d/10-calico.conflist").content_string)
assert cni["plugins"][0]["type"] == "calico"

View File

@ -48,7 +48,7 @@
- name: Save k3s-init logs
copy:
content: "{{ __k3s_init_log.stdout }}"
dest: "{{ k3s_log_dir }}/k3s-init.log"
dest: "{{ __k3s_log_dir }}/k3s-init.log"
mode: 0640
when: k3s_init_log_enabled | bool

View File

@ -11,6 +11,11 @@
state: present
loop: "{{ k3s_packages_extra }}"
- name: Check if cluster is installed
stat:
path: "{{ __k3s_config_dir }}/k3s.yaml"
register: __k3s_installed
- name: Prepare directory structure
file:
path: "{{ item }}"
@ -19,10 +24,11 @@
mode: 0700
state: directory
loop:
- "{{ k3s_config_dir }}"
- "{{ k3s_config_dir }}/server"
- "{{ k3s_data_dir }}"
- "{{ k3s_log_dir }}"
- "{{ __k3s_config_dir }}"
- "{{ __k3s_config_dir }}/server"
- "{{ __k3s_data_dir }}"
- "{{ __k3s_manifests_dir }}"
- "{{ __k3s_log_dir }}"
- name: Download K3s binary
get_url:
@ -36,7 +42,7 @@
- name: Copy K3s config file
template:
src: "etc/rancher/k3s/config.yaml.j2"
dest: "/etc/rancher/k3s/config.yaml"
dest: "{{ __k3s_config_dir }}/config.yaml"
owner: root
group: root
mode: 0600
@ -45,7 +51,7 @@
- name: Copy K3s server config files
template:
src: "etc/rancher/k3s/server/{{ item }}.j2"
dest: "/etc/rancher/k3s/server/{{ item }}"
dest: "{{ __k3s_config_dir }}/server/{{ item }}"
owner: root
group: root
mode: 0600
@ -54,6 +60,30 @@
when: k3s_server | bool
notify: __k3s_restart
- name: Copy auto-deploying manifests to the server
ansible.builtin.template:
src: "{{ item }}"
dest: "{{ __k3s_manifests_dir }}/{{ item | basename | replace('.j2', '') }}"
mode: 0644
loop: "{{ k3s_server_manifests_templates }}"
loop_control:
label: "{{ __k3s_manifests_dir }}/{{ item | basename | replace('.j2', '') }}"
when:
- ansible_hostname == hostvars[k3s_server_nodes[0]]['ansible_hostname']
- not __k3s_installed.stat.exists
- name: Download auto-deploying manifests to the server
ansible.builtin.get_url:
url: "{{ item.url }}"
dest: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url | basename) }}"
mode: 0644
loop: "{{ k3s_server_manifests_urls }}"
loop_control:
label: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url | basename) }}"
when:
- ansible_hostname == hostvars[k3s_server_nodes[0]]['ansible_hostname']
- not __k3s_installed.stat.exists
- name: Create kubectl symlink
file:
src: /usr/local/bin/k3s

View File

@ -9,7 +9,7 @@
- k3s
- k3s-init
- name: Kill containerd-shim-runc"
- name: Kill containerd-shim-runc
register: __k3s_pkill_containerd_shim_runc
command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc"
changed_when: __k3s_pkill_containerd_shim_runc.rc == 0
@ -24,7 +24,7 @@
- /var/lib/kubelet/pods
- /var/lib/kubelet/plugins
- /run/netns/cni-
- "{{ k3s_data_dir }}"
- "{{ __k3s_data_dir }}"
register: __k3s_mounted_fs
args:
executable: /bin/bash
@ -44,9 +44,10 @@
loop:
- /usr/local/bin/k3s
- "{{ __k3s_service_file }}"
- "{{ k3s_config_dir }}"
- "{{ k3s_data_dir }}"
- "{{ k3s_log_dir }}"
- "{{ __k3s_config_dir }}"
- "{{ __k3s_data_dir }}"
- "{{ __k3s_log_dir }}"
- /etc/cni
- /run/k3s
- /run/flannel
- /var/lib/kubelet

View File

@ -21,11 +21,11 @@
- name: Wait for node-token
wait_for:
path: "{{ k3s_data_dir }}/server/node-token"
path: "{{ __k3s_data_dir }}/server/node-token"
- name: Read node-token from server
slurp:
path: "{{ k3s_data_dir }}/server/node-token"
path: "{{ __k3s_data_dir }}/server/node-token"
register: __k3s_node_token
- name: Store server node-token
@ -36,7 +36,7 @@
command: >-
k3s kubectl config set-cluster default
--server=https://{{ __k3s_server_ip }}:6443
--kubeconfig /etc/rancher/k3s/k3s.yaml
--kubeconfig {{ __k3s_config_dir }}/k3s.yaml
changed_when: False
- name: Create directory .kube
@ -49,11 +49,30 @@
- name: Copy kube config to user home
copy:
src: /etc/rancher/k3s/k3s.yaml
src: "{{ __k3s_config_dir }}/k3s.yaml"
dest: "{{ ansible_user_dir }}/.kube/config"
remote_src: True
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: "preserve"
- name: Wait for initial setup
command: >-
kubectl get deployment coredns
-n kube-system
-o go-template={% raw %}'{{ .status.availableReplicas }}'{% endraw %}
register: __k3s_init_setup
until: __k3s_init_setup.rc == 0 and __k3s_init_setup.stdout == "1"
retries: 6
delay: 10
changed_when: False
- name: Remove auto-deploying manifests
file:
path: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url) | default(item) | basename | replace('.j2', '') }}"
state: absent
loop: "{{ k3s_server_manifests_urls + k3s_server_manifests_templates }}"
loop_control:
label: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url) | default(item) | basename | replace('.j2', '') }}"
become: True
become_user: root

View File

@ -4,7 +4,7 @@
- profiling=0
- service-account-lookup=true
- tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
- admission-control-config-file=/etc/rancher/k3s/server/admission-config.yaml
- admission-control-config-file={{ __k3s_config_dir }}/server/admission-config.yaml
{% if k3s_server_admission_plugins | length > 0 %}
- enable-admission-plugins={{ k3s_server_admission_plugins | join(',') }}
{% endif %}

View File

@ -1,13 +1,14 @@
#jinja2: lstrip_blocks: True
---
{% set __k3s_kube_apiserver_arg = lookup('template', '_internal/apiserver-arg.yaml.j2') | from_yaml %}
data-dir: "{{ k3s_data_dir }}"
token: "{{ k3s_token }}"
node-ip: "{{ k3s_node_ip }}"
{% if k3s_server | bool %}
bind-address: "{{ k3s_server_bind_ip }}"
secrets-encryption: True
write-kubeconfig: "{{ __k3s_config_dir }}/k3s.yaml"
write-kubeconfig-mode: "0600"
disable:
- traefik
{% if __k3s_kube_apiserver_arg is iterable %}

View File

@ -9,7 +9,7 @@ After=network-online.target
Type=notify
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/k3s {{ "server" if k3s_server | bool else "agent" }} --config {{ k3s_config_dir }}/config.yaml
ExecStart=/usr/local/bin/k3s {{ "server" if k3s_server | bool else "agent" }} --config {{ __k3s_config_dir }}/config.yaml
KillMode=process
Delegate=yes
LimitNOFILE=1048576

View File

@ -9,6 +9,11 @@ __k3s_server_init_args: >-
--server https://{{ __k3s_server_ip }}:6443
{% endif %}
{% endif %}
--config {{ k3s_config_dir }}/config.yaml
--config {{ __k3s_config_dir }}/config.yaml
__k3s_service_file: /etc/systemd/system/k3s.service
__k3s_config_dir: /etc/rancher/k3s
__k3s_data_dir: /var/lib/rancher/k3s
__k3s_manifests_dir: "{{ __k3s_data_dir }}/server/manifests"
__k3s_log_dir: /var/log/rancher/k3s