feat: allow custom cni setups

defaults/main.yml

@@ -10,10 +10,6 @@ k3s_token: "secure-token"
 k3s_node_ip: "{{ ansible_default_ipv4.address }}"
 k3s_init_log_enabled: False
 
-k3s_config_dir: /etc/rancher/k3s
-k3s_data_dir: /var/lib/rancher/k3s
-k3s_log_dir: /var/log/rancher/k3s
-
 k3s_server: True
 k3s_server_bind_ip: "0.0.0.0"
 k3s_server_nodes:
@@ -22,6 +18,9 @@ k3s_server_flannel_backend_enabled: True
 k3s_server_network_policy_enabled: True
 k3s_server_feature_gates: []
 
+k3s_server_manifests_templates: []
+k3s_server_manifests_urls: []
+
 k3s_server_admission_plugins:
   - NodeRestriction
   - EventRateLimit

molecule/rocky9/converge.yml

@@ -7,6 +7,13 @@
       - container-selinux
       - selinux-policy-base
       - https://github.com/k3s-io/k3s-selinux/releases/download/v1.2.stable.2/k3s-selinux-1.2-2.el8.noarch.rpm
+    k3s_server_flannel_backend_enabled: False
+    k3s_server_network_policy_enabled: False
+    k3s_server_manifests_templates:
+      - "calico-installation.yaml.j2"
+    k3s_server_manifests_urls:
+      - url: https://docs.projectcalico.org/archive/v3.24/manifests/tigera-operator.yaml
+        dest: tigera-operator.yaml
     k3s_init_log_enabled: True
   pre_tasks:
     - name: Override host variables

molecule/rocky9/templates/calico-installation.yaml.j2

@@ -0,0 +1,15 @@
+---
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  calicoNetwork:
+    ipPools:
+      - blockSize: 26
+        cidr: "10.42.0.0/16"
+        encapsulation: "VXLANCrossSubnet"
+        natOutgoing: Enabled
+        nodeSelector: all()
+  nodeMetricsPort: 9091
+  typhaMetricsPort: 9093

molecule/rocky9/tests/test_default.py

@@ -1,3 +1,4 @@
+import json
 import os
 
 import testinfra.utils.ansible_runner
@@ -24,3 +25,9 @@ def test_k3s_cluster_ready(host):
     cluster = host.run("kubectl get --raw='/readyz'").stdout
 
     assert cluster == "ok"
+
+
+def test_k3s_cni(host):
+    cni = json.loads(host.file("/etc/cni/net.d/10-calico.conflist").content_string)
+
+    assert cni["plugins"][0]["type"] == "calico"

tasks/init.yml

@@ -48,7 +48,7 @@
     - name: Save k3s-init logs
       copy:
         content: "{{ __k3s_init_log.stdout }}"
-        dest: "{{ k3s_log_dir }}/k3s-init.log"
+        dest: "{{ __k3s_log_dir }}/k3s-init.log"
         mode: 0640
       when: k3s_init_log_enabled | bool
 

tasks/prepare.yml

@@ -11,6 +11,11 @@
         state: present
       loop: "{{ k3s_packages_extra }}"
 
+    - name: Check if cluster is installed
+      stat:
+        path: "{{ __k3s_config_dir }}/k3s.yaml"
+      register: __k3s_installed
+
     - name: Prepare directory structure
       file:
         path: "{{ item }}"
@@ -19,10 +24,11 @@
         mode: 0700
         state: directory
       loop:
-        - "{{ k3s_config_dir }}"
+        - "{{ __k3s_config_dir }}"
-        - "{{ k3s_config_dir }}/server"
+        - "{{ __k3s_config_dir }}/server"
-        - "{{ k3s_data_dir }}"
+        - "{{ __k3s_data_dir }}"
-        - "{{ k3s_log_dir }}"
+        - "{{ __k3s_manifests_dir }}"
+        - "{{ __k3s_log_dir }}"
 
     - name: Download K3s binary
       get_url:
@@ -36,7 +42,7 @@
     - name: Copy K3s config file
       template:
         src: "etc/rancher/k3s/config.yaml.j2"
-        dest: "/etc/rancher/k3s/config.yaml"
+        dest: "{{ __k3s_config_dir }}/config.yaml"
         owner: root
         group: root
         mode: 0600
@@ -45,7 +51,7 @@
     - name: Copy K3s server config files
       template:
         src: "etc/rancher/k3s/server/{{ item }}.j2"
-        dest: "/etc/rancher/k3s/server/{{ item }}"
+        dest: "{{ __k3s_config_dir }}/server/{{ item }}"
         owner: root
         group: root
         mode: 0600
@@ -54,6 +60,30 @@
       when: k3s_server | bool
       notify: __k3s_restart
 
+    - name: Copy auto-deploying manifests to the server
+      ansible.builtin.template:
+        src: "{{ item }}"
+        dest: "{{ __k3s_manifests_dir }}/{{ item | basename | replace('.j2', '') }}"
+        mode: 0644
+      loop: "{{ k3s_server_manifests_templates }}"
+      loop_control:
+        label: "{{ __k3s_manifests_dir }}/{{ item | basename | replace('.j2', '') }}"
+      when:
+        - ansible_hostname == hostvars[k3s_server_nodes[0]]['ansible_hostname']
+        - not __k3s_installed.stat.exists
+
+    - name: Download auto-deploying manifests to the server
+      ansible.builtin.get_url:
+        url: "{{ item.url }}"
+        dest: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url | basename) }}"
+        mode: 0644
+      loop: "{{ k3s_server_manifests_urls }}"
+      loop_control:
+        label: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url | basename) }}"
+      when:
+        - ansible_hostname == hostvars[k3s_server_nodes[0]]['ansible_hostname']
+        - not __k3s_installed.stat.exists
+
     - name: Create kubectl symlink
       file:
         src: /usr/local/bin/k3s

tasks/reset.yml

@@ -9,7 +9,7 @@
     - k3s
     - k3s-init
 
-- name: Kill containerd-shim-runc"
+- name: Kill containerd-shim-runc
   register: __k3s_pkill_containerd_shim_runc
   command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc"
   changed_when: __k3s_pkill_containerd_shim_runc.rc == 0
@@ -24,7 +24,7 @@
     - /var/lib/kubelet/pods
     - /var/lib/kubelet/plugins
     - /run/netns/cni-
-    - "{{ k3s_data_dir }}"
+    - "{{ __k3s_data_dir }}"
   register: __k3s_mounted_fs
   args:
     executable: /bin/bash
@@ -44,9 +44,10 @@
   loop:
     - /usr/local/bin/k3s
     - "{{ __k3s_service_file }}"
-    - "{{ k3s_config_dir }}"
+    - "{{ __k3s_config_dir }}"
-    - "{{ k3s_data_dir }}"
+    - "{{ __k3s_data_dir }}"
-    - "{{ k3s_log_dir }}"
+    - "{{ __k3s_log_dir }}"
+    - /etc/cni
     - /run/k3s
     - /run/flannel
     - /var/lib/kubelet

tasks/setup.yml

@@ -21,11 +21,11 @@
 
     - name: Wait for node-token
       wait_for:
-        path: "{{ k3s_data_dir }}/server/node-token"
+        path: "{{ __k3s_data_dir }}/server/node-token"
 
     - name: Read node-token from server
       slurp:
-        path: "{{ k3s_data_dir }}/server/node-token"
+        path: "{{ __k3s_data_dir }}/server/node-token"
       register: __k3s_node_token
 
     - name: Store server node-token
@@ -36,7 +36,7 @@
       command: >-
         k3s kubectl config set-cluster default
           --server=https://{{ __k3s_server_ip }}:6443
-          --kubeconfig /etc/rancher/k3s/k3s.yaml
+          --kubeconfig {{ __k3s_config_dir }}/k3s.yaml
       changed_when: False
 
     - name: Create directory .kube
@@ -49,11 +49,30 @@
 
     - name: Copy kube config to user home
       copy:
-        src: /etc/rancher/k3s/k3s.yaml
+        src: "{{ __k3s_config_dir }}/k3s.yaml"
         dest: "{{ ansible_user_dir }}/.kube/config"
         remote_src: True
         owner: "{{ ansible_user }}"
         group: "{{ ansible_user }}"
         mode: "preserve"
+
+    - name: Wait for initial setup
+      command: >-
+        kubectl get deployment coredns
+          -n kube-system
+          -o go-template={% raw %}'{{ .status.availableReplicas }}'{% endraw %}
+      register: __k3s_init_setup
+      until: __k3s_init_setup.rc == 0 and __k3s_init_setup.stdout == "1"
+      retries: 6
+      delay: 10
+      changed_when: False
+
+    - name: Remove auto-deploying manifests
+      file:
+        path: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url) | default(item) | basename | replace('.j2', '') }}"
+        state: absent
+      loop: "{{ k3s_server_manifests_urls + k3s_server_manifests_templates }}"
+      loop_control:
+        label: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url) | default(item) | basename | replace('.j2', '') }}"
   become: True
   become_user: root

templates/_internal/apiserver-arg.yaml.j2

@@ -4,7 +4,7 @@
 - profiling=0
 - service-account-lookup=true
 - tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
-- admission-control-config-file=/etc/rancher/k3s/server/admission-config.yaml
+- admission-control-config-file={{ __k3s_config_dir }}/server/admission-config.yaml
 {% if k3s_server_admission_plugins | length > 0 %}
 - enable-admission-plugins={{ k3s_server_admission_plugins | join(',') }}
 {% endif %}

templates/etc/rancher/k3s/config.yaml.j2

@@ -1,13 +1,14 @@
 #jinja2: lstrip_blocks: True
 ---
 {% set __k3s_kube_apiserver_arg = lookup('template', '_internal/apiserver-arg.yaml.j2') | from_yaml %}
-data-dir: "{{ k3s_data_dir }}"
 token: "{{ k3s_token }}"
 node-ip: "{{ k3s_node_ip }}"
 
 {% if k3s_server | bool %}
 bind-address: "{{ k3s_server_bind_ip }}"
 secrets-encryption: True
+write-kubeconfig: "{{ __k3s_config_dir }}/k3s.yaml"
+write-kubeconfig-mode: "0600"
 disable:
   - traefik
 {% if __k3s_kube_apiserver_arg is iterable %}

templates/etc/systemd/system/k3s.service.j2

@@ -9,7 +9,7 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s {{ "server" if k3s_server | bool else "agent" }} --config {{ k3s_config_dir }}/config.yaml
+ExecStart=/usr/local/bin/k3s {{ "server" if k3s_server | bool else "agent" }} --config {{ __k3s_config_dir }}/config.yaml
 KillMode=process
 Delegate=yes
 LimitNOFILE=1048576

vars/main.yml

@@ -9,6 +9,11 @@ __k3s_server_init_args: >-
       --server https://{{ __k3s_server_ip }}:6443
     {% endif %}
   {% endif %}
-  --config {{ k3s_config_dir }}/config.yaml
+  --config {{ __k3s_config_dir }}/config.yaml
 
 __k3s_service_file: /etc/systemd/system/k3s.service
+
+__k3s_config_dir: /etc/rancher/k3s
+__k3s_data_dir: /var/lib/rancher/k3s
+__k3s_manifests_dir: "{{ __k3s_data_dir }}/server/manifests"
+__k3s_log_dir: /var/log/rancher/k3s