diff --git a/defaults/main.yml b/defaults/main.yml index 5c297e8..77e5556 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -10,10 +10,6 @@ k3s_token: "secure-token" k3s_node_ip: "{{ ansible_default_ipv4.address }}" k3s_init_log_enabled: False -k3s_config_dir: /etc/rancher/k3s -k3s_data_dir: /var/lib/rancher/k3s -k3s_log_dir: /var/log/rancher/k3s - k3s_server: True k3s_server_bind_ip: "0.0.0.0" k3s_server_nodes: @@ -22,6 +18,9 @@ k3s_server_flannel_backend_enabled: True k3s_server_network_policy_enabled: True k3s_server_feature_gates: [] +k3s_server_manifests_templates: [] +k3s_server_manifests_urls: [] + k3s_server_admission_plugins: - NodeRestriction - EventRateLimit diff --git a/molecule/rocky9/converge.yml b/molecule/rocky9/converge.yml index 5d5bcc8..ac7c677 100644 --- a/molecule/rocky9/converge.yml +++ b/molecule/rocky9/converge.yml @@ -7,6 +7,13 @@ - container-selinux - selinux-policy-base - https://github.com/k3s-io/k3s-selinux/releases/download/v1.2.stable.2/k3s-selinux-1.2-2.el8.noarch.rpm + k3s_server_flannel_backend_enabled: False + k3s_server_network_policy_enabled: False + k3s_server_manifests_templates: + - "calico-installation.yaml.j2" + k3s_server_manifests_urls: + - url: https://docs.projectcalico.org/archive/v3.24/manifests/tigera-operator.yaml + dest: tigera-operator.yaml k3s_init_log_enabled: True pre_tasks: - name: Override host variables diff --git a/molecule/rocky9/templates/calico-installation.yaml.j2 b/molecule/rocky9/templates/calico-installation.yaml.j2 new file mode 100644 index 0000000..06b0fb8 --- /dev/null +++ b/molecule/rocky9/templates/calico-installation.yaml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + calicoNetwork: + ipPools: + - blockSize: 26 + cidr: "10.42.0.0/16" + encapsulation: "VXLANCrossSubnet" + natOutgoing: Enabled + nodeSelector: all() + nodeMetricsPort: 9091 + typhaMetricsPort: 9093 diff --git a/molecule/rocky9/tests/test_default.py b/molecule/rocky9/tests/test_default.py index 092df11..547e3f1 100644 --- a/molecule/rocky9/tests/test_default.py +++ b/molecule/rocky9/tests/test_default.py @@ -1,3 +1,4 @@ +import json import os import testinfra.utils.ansible_runner @@ -24,3 +25,9 @@ def test_k3s_cluster_ready(host): cluster = host.run("kubectl get --raw='/readyz'").stdout assert cluster == "ok" + + +def test_k3s_cni(host): + cni = json.loads(host.file("/etc/cni/net.d/10-calico.conflist").content_string) + + assert cni["plugins"][0]["type"] == "calico" diff --git a/tasks/init.yml b/tasks/init.yml index 02ecdd3..ecff028 100644 --- a/tasks/init.yml +++ b/tasks/init.yml @@ -48,7 +48,7 @@ - name: Save k3s-init logs copy: content: "{{ __k3s_init_log.stdout }}" - dest: "{{ k3s_log_dir }}/k3s-init.log" + dest: "{{ __k3s_log_dir }}/k3s-init.log" mode: 0640 when: k3s_init_log_enabled | bool diff --git a/tasks/prepare.yml b/tasks/prepare.yml index bfe9f3a..9bcece4 100644 --- a/tasks/prepare.yml +++ b/tasks/prepare.yml @@ -11,6 +11,11 @@ state: present loop: "{{ k3s_packages_extra }}" + - name: Check if cluster is installed + stat: + path: "{{ __k3s_config_dir }}/k3s.yaml" + register: __k3s_installed + - name: Prepare directory structure file: path: "{{ item }}" @@ -19,10 +24,11 @@ mode: 0700 state: directory loop: - - "{{ k3s_config_dir }}" - - "{{ k3s_config_dir }}/server" - - "{{ k3s_data_dir }}" - - "{{ k3s_log_dir }}" + - "{{ __k3s_config_dir }}" + - "{{ __k3s_config_dir }}/server" + - "{{ __k3s_data_dir }}" + - "{{ __k3s_manifests_dir }}" + - "{{ __k3s_log_dir }}" - name: Download K3s binary get_url: @@ -36,7 +42,7 @@ - name: Copy K3s config file template: src: "etc/rancher/k3s/config.yaml.j2" - dest: "/etc/rancher/k3s/config.yaml" + dest: "{{ __k3s_config_dir }}/config.yaml" owner: root group: root mode: 0600 @@ -45,7 +51,7 @@ - name: Copy K3s server config files template: src: "etc/rancher/k3s/server/{{ item }}.j2" - dest: "/etc/rancher/k3s/server/{{ item }}" + dest: "{{ __k3s_config_dir }}/server/{{ item }}" owner: root group: root mode: 0600 @@ -54,6 +60,30 @@ when: k3s_server | bool notify: __k3s_restart + - name: Copy auto-deploying manifests to the server + ansible.builtin.template: + src: "{{ item }}" + dest: "{{ __k3s_manifests_dir }}/{{ item | basename | replace('.j2', '') }}" + mode: 0644 + loop: "{{ k3s_server_manifests_templates }}" + loop_control: + label: "{{ __k3s_manifests_dir }}/{{ item | basename | replace('.j2', '') }}" + when: + - ansible_hostname == hostvars[k3s_server_nodes[0]]['ansible_hostname'] + - not __k3s_installed.stat.exists + + - name: Download auto-deploying manifests to the server + ansible.builtin.get_url: + url: "{{ item.url }}" + dest: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url | basename) }}" + mode: 0644 + loop: "{{ k3s_server_manifests_urls }}" + loop_control: + label: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url | basename) }}" + when: + - ansible_hostname == hostvars[k3s_server_nodes[0]]['ansible_hostname'] + - not __k3s_installed.stat.exists + - name: Create kubectl symlink file: src: /usr/local/bin/k3s diff --git a/tasks/reset.yml b/tasks/reset.yml index 8b7f07c..b25b1c9 100644 --- a/tasks/reset.yml +++ b/tasks/reset.yml @@ -9,7 +9,7 @@ - k3s - k3s-init -- name: Kill containerd-shim-runc" +- name: Kill containerd-shim-runc register: __k3s_pkill_containerd_shim_runc command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" changed_when: __k3s_pkill_containerd_shim_runc.rc == 0 @@ -24,7 +24,7 @@ - /var/lib/kubelet/pods - /var/lib/kubelet/plugins - /run/netns/cni- - - "{{ k3s_data_dir }}" + - "{{ __k3s_data_dir }}" register: __k3s_mounted_fs args: executable: /bin/bash @@ -44,9 +44,10 @@ loop: - /usr/local/bin/k3s - "{{ __k3s_service_file }}" - - "{{ k3s_config_dir }}" - - "{{ k3s_data_dir }}" - - "{{ k3s_log_dir }}" + - "{{ __k3s_config_dir }}" + - "{{ __k3s_data_dir }}" + - "{{ __k3s_log_dir }}" + - /etc/cni - /run/k3s - /run/flannel - /var/lib/kubelet diff --git a/tasks/setup.yml b/tasks/setup.yml index 2b12f35..4cabde4 100644 --- a/tasks/setup.yml +++ b/tasks/setup.yml @@ -21,11 +21,11 @@ - name: Wait for node-token wait_for: - path: "{{ k3s_data_dir }}/server/node-token" + path: "{{ __k3s_data_dir }}/server/node-token" - name: Read node-token from server slurp: - path: "{{ k3s_data_dir }}/server/node-token" + path: "{{ __k3s_data_dir }}/server/node-token" register: __k3s_node_token - name: Store server node-token @@ -36,7 +36,7 @@ command: >- k3s kubectl config set-cluster default --server=https://{{ __k3s_server_ip }}:6443 - --kubeconfig /etc/rancher/k3s/k3s.yaml + --kubeconfig {{ __k3s_config_dir }}/k3s.yaml changed_when: False - name: Create directory .kube @@ -49,11 +49,30 @@ - name: Copy kube config to user home copy: - src: /etc/rancher/k3s/k3s.yaml + src: "{{ __k3s_config_dir }}/k3s.yaml" dest: "{{ ansible_user_dir }}/.kube/config" remote_src: True owner: "{{ ansible_user }}" group: "{{ ansible_user }}" mode: "preserve" + + - name: Wait for initial setup + command: >- + kubectl get deployment coredns + -n kube-system + -o go-template={% raw %}'{{ .status.availableReplicas }}'{% endraw %} + register: __k3s_init_setup + until: __k3s_init_setup.rc == 0 and __k3s_init_setup.stdout == "1" + retries: 6 + delay: 10 + changed_when: False + + - name: Remove auto-deploying manifests + file: + path: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url) | default(item) | basename | replace('.j2', '') }}" + state: absent + loop: "{{ k3s_server_manifests_urls + k3s_server_manifests_templates }}" + loop_control: + label: "{{ __k3s_manifests_dir }}/{{ item.dest | default(item.url) | default(item) | basename | replace('.j2', '') }}" become: True become_user: root diff --git a/templates/_internal/apiserver-arg.yaml.j2 b/templates/_internal/apiserver-arg.yaml.j2 index 569eccd..70f91d4 100644 --- a/templates/_internal/apiserver-arg.yaml.j2 +++ b/templates/_internal/apiserver-arg.yaml.j2 @@ -4,7 +4,7 @@ - profiling=0 - service-account-lookup=true - tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 -- admission-control-config-file=/etc/rancher/k3s/server/admission-config.yaml +- admission-control-config-file={{ __k3s_config_dir }}/server/admission-config.yaml {% if k3s_server_admission_plugins | length > 0 %} - enable-admission-plugins={{ k3s_server_admission_plugins | join(',') }} {% endif %} diff --git a/templates/etc/rancher/k3s/config.yaml.j2 b/templates/etc/rancher/k3s/config.yaml.j2 index 6b7f601..52406cf 100644 --- a/templates/etc/rancher/k3s/config.yaml.j2 +++ b/templates/etc/rancher/k3s/config.yaml.j2 @@ -1,13 +1,14 @@ #jinja2: lstrip_blocks: True --- {% set __k3s_kube_apiserver_arg = lookup('template', '_internal/apiserver-arg.yaml.j2') | from_yaml %} -data-dir: "{{ k3s_data_dir }}" token: "{{ k3s_token }}" node-ip: "{{ k3s_node_ip }}" {% if k3s_server | bool %} bind-address: "{{ k3s_server_bind_ip }}" secrets-encryption: True +write-kubeconfig: "{{ __k3s_config_dir }}/k3s.yaml" +write-kubeconfig-mode: "0600" disable: - traefik {% if __k3s_kube_apiserver_arg is iterable %} diff --git a/templates/etc/systemd/system/k3s.service.j2 b/templates/etc/systemd/system/k3s.service.j2 index 36285cd..63362db 100644 --- a/templates/etc/systemd/system/k3s.service.j2 +++ b/templates/etc/systemd/system/k3s.service.j2 @@ -9,7 +9,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s {{ "server" if k3s_server | bool else "agent" }} --config {{ k3s_config_dir }}/config.yaml +ExecStart=/usr/local/bin/k3s {{ "server" if k3s_server | bool else "agent" }} --config {{ __k3s_config_dir }}/config.yaml KillMode=process Delegate=yes LimitNOFILE=1048576 diff --git a/vars/main.yml b/vars/main.yml index efe53c9..82ab523 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -9,6 +9,11 @@ __k3s_server_init_args: >- --server https://{{ __k3s_server_ip }}:6443 {% endif %} {% endif %} - --config {{ k3s_config_dir }}/config.yaml + --config {{ __k3s_config_dir }}/config.yaml __k3s_service_file: /etc/systemd/system/k3s.service + +__k3s_config_dir: /etc/rancher/k3s +__k3s_data_dir: /var/lib/rancher/k3s +__k3s_manifests_dir: "{{ __k3s_data_dir }}/server/manifests" +__k3s_log_dir: /var/log/rancher/k3s