Simplify K3s service startup for HA

Signed-off-by: Derek Nola <derek.nola@suse.com>

Vagrantfile

@@ -28,10 +28,10 @@ def provision(vm, role, node_num)
     ansible.extra_vars = {
       k3s_version: "v1.26.5+k3s1",
       api_endpoint: "#{NETWORK_PREFIX}.100",
-      token: "myyagrant",
+      token: "myvagrant",
       # Required to use the private network configured above
       extra_server_args: "--node-external-ip #{node_ip} --flannel-iface eth1", 
-      extra_agent_args: "",
+      extra_agent_args: "--node-external-ip #{node_ip} --flannel-iface eth1",
     }
   end
 end

roles/k3s/agent/tasks/main.yml

@@ -12,5 +12,5 @@
   ansible.builtin.systemd:
     name: k3s-agent
     daemon_reload: true
-    state: restarted
+    state: started
     enabled: true

roles/k3s/agent/templates/k3s-agent.service.j2

@@ -1,13 +1,17 @@
 [Unit]
 Description=Lightweight Kubernetes
 Documentation=https://k3s.io
+Wants=network-online.target
 After=network-online.target
 
+[Install]
+WantedBy=multi-user.target
+
 [Service]
 Type=notify
-ExecStartPre=-/sbin/modprobe br_netfilter
-ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_agent_args }}
+EnvironmentFile=-/etc/default/%N
+EnvironmentFile=-/etc/sysconfig/%N
+EnvironmentFile=-/etc/systemd/system/k3s.service.env
 KillMode=process
 Delegate=yes
 # Having non-zero Limit*s causes performance problems due to accounting overhead
@@ -19,6 +23,7 @@ TasksMax=infinity
 TimeoutStartSec=0
 Restart=always
 RestartSec=5s
-
-[Install]
-WantedBy=multi-user.target
+ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service'
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_agent_args }}

roles/k3s/server/tasks/main.yml

@@ -2,22 +2,30 @@
 - name: Init first server node
   when: ansible_hostname == groups['server'][0]
   block:
-    - name: Start temporary service for HA cluster
-      ansible.builtin.command:
-        cmd: >
-          systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server
-          --cluster-init --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args}}
-        # noqa: jinja[spacing]
-        creates: "{{ k3s_server_location }}/server/node-token"
-      when: groups['server'] | length > 1
-
-    - name: Start temporary service for single server cluster
-      ansible.builtin.command:
-        cmd: >
-          systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server
-          --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }}
-        creates: "{{ k3s_server_location }}/server/node-token"
+    - name: Copy K3s service file [Single]
       when: groups['server'] | length == 1
+      ansible.builtin.template:
+        src: "k3s-single.service.j2"
+        dest: "{{ systemd_dir }}/k3s.service"
+        owner: root
+        group: root
+        mode: 0644
+
+    - name: Copy K3s service file [HA]
+      when: groups['server'] | length > 1
+      ansible.builtin.template:
+        src: "k3s-cluster-init.service.j2"
+        dest: "{{ systemd_dir }}/k3s.service"
+        owner: root
+        group: root
+        mode: 0644
+
+    - name: Enable and check K3s service
+      ansible.builtin.systemd:
+        name: k3s
+        daemon_reload: true
+        state: started
+        enabled: true
 
     - name: Create directory .kube
       ansible.builtin.file:
@@ -26,6 +34,10 @@
         owner: "{{ ansible_user }}"
         mode: "u=rwx,g=rx,o="
 
+    - name: Pause to allow server startup
+      ansible.builtin.pause:
+        seconds: 10
+
     - name: Copy config file to user home directory
       ansible.builtin.copy:
         src: /etc/rancher/k3s/k3s.yaml
@@ -48,15 +60,25 @@
         flat: true
 
 - name: Start other server if any and verify status
+  when:
+    - (groups['server'] | length) > 1
+    - ansible_hostname != groups['server'][0]
   block:
-    - name: Init additonal server nodes
-      ansible.builtin.command:
-        cmd: >
-          systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server
-          --token "{{ hostvars[groups['server'][0]]['token'] }}" --server https://{{ api_endpoint }}:{{ api_port }}
-          --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }}
-        creates: "{{ k3s_server_location }}/server/node-token"
-      when: ansible_hostname != groups['server'][0]
+    - name: Copy K3s service file [HA]
+      when: groups['server'] | length > 1
+      ansible.builtin.template:
+        src: "k3s-ha.service.j2"
+        dest: "{{ systemd_dir }}/k3s.service"
+        owner: root
+        group: root
+        mode: 0644
+
+    - name: Enable and check K3s service
+      ansible.builtin.systemd:
+        name: k3s
+        daemon_reload: true
+        state: started
+        enabled: true
 
     - name: Verify that all server nodes joined
       when: (groups['server'] | length) > 1
@@ -68,28 +90,6 @@
       retries: 20
       delay: 10
       changed_when: false
-  always:
-    - name: Kill the temporary init service
-      ansible.builtin.systemd:
-        name: k3s-init
-        state: stopped
-      failed_when: false
-
-- name: Copy K3s service file
-  ansible.builtin.template:
-    src: "k3s-server.service.j2"
-    dest: "{{ systemd_dir }}/k3s-server.service"
-    owner: root
-    group: root
-    mode: 0644
-  register: k3s_service
-
-- name: Enable and check K3s service
-  ansible.builtin.systemd:
-    name: k3s-server
-    daemon_reload: true
-    state: restarted
-    enabled: true
 
 - name: Create symlinks
   ansible.builtin.file:

roles/k3s/server/templates/k3s-cluster-init.service.j2

@@ -0,0 +1,28 @@
+[Unit]
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+Wants=network-online.target
+After=network-online.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/default/%N
+EnvironmentFile=-/etc/sysconfig/%N
+EnvironmentFile=-/etc/systemd/system/k3s.service.env
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s server --cluster-init --data-dir {{ k3s_server_location }} --token {{ token }} {{ extra_server_args }} 

roles/k3s/server/templates/k3s-ha.service.j2

@@ -0,0 +1,28 @@
+[Unit]
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+Wants=network-online.target
+After=network-online.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/default/%N
+EnvironmentFile=-/etc/sysconfig/%N
+EnvironmentFile=-/etc/systemd/system/k3s.service.env
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_server_args }} 

roles/k3s/server/templates/k3s-single.service.j2

@@ -1,13 +1,17 @@
 [Unit]
 Description=Lightweight Kubernetes
 Documentation=https://k3s.io
+Wants=network-online.target
 After=network-online.target
 
+[Install]
+WantedBy=multi-user.target
+
 [Service]
 Type=notify
-ExecStartPre=-/sbin/modprobe br_netfilter
-ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args }} --token {{ token }}
+EnvironmentFile=-/etc/default/%N
+EnvironmentFile=-/etc/sysconfig/%N
+EnvironmentFile=-/etc/systemd/system/k3s.service.env
 KillMode=process
 Delegate=yes
 # Having non-zero Limit*s causes performance problems due to accounting overhead
@@ -19,6 +23,6 @@ TasksMax=infinity
 TimeoutStartSec=0
 Restart=always
 RestartSec=5s
-
-[Install]
-WantedBy=multi-user.target
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} --token {{ token }} {{ extra_server_args }} 

roles/prereq/tasks/main.yml

@@ -59,12 +59,6 @@
     validate: 'visudo -cf %s'
   when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux','RedHat']
 
-- name: Make k3s directory
-  ansible.builtin.file:
-    path: "/var/lib/rancher"
-    mode: 0755
-    state: directory
-
 - name: Create symlink
   ansible.builtin.file:
     dest: /var/lib/rancher/k3s