diff --git a/Vagrantfile b/Vagrantfile index 80cee41..9c3c96f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,10 +28,10 @@ def provision(vm, role, node_num) ansible.extra_vars = { k3s_version: "v1.26.5+k3s1", api_endpoint: "#{NETWORK_PREFIX}.100", - token: "myyagrant", + token: "myvagrant", # Required to use the private network configured above extra_server_args: "--node-external-ip #{node_ip} --flannel-iface eth1", - extra_agent_args: "", + extra_agent_args: "--node-external-ip #{node_ip} --flannel-iface eth1", } end end diff --git a/roles/k3s/agent/tasks/main.yml b/roles/k3s/agent/tasks/main.yml index 2ab6b66..b89520b 100644 --- a/roles/k3s/agent/tasks/main.yml +++ b/roles/k3s/agent/tasks/main.yml @@ -12,5 +12,5 @@ ansible.builtin.systemd: name: k3s-agent daemon_reload: true - state: restarted + state: started enabled: true diff --git a/roles/k3s/agent/templates/k3s-agent.service.j2 b/roles/k3s/agent/templates/k3s-agent.service.j2 index e0157b2..59261e3 100644 --- a/roles/k3s/agent/templates/k3s-agent.service.j2 +++ b/roles/k3s/agent/templates/k3s-agent.service.j2 @@ -1,13 +1,17 @@ [Unit] Description=Lightweight Kubernetes Documentation=https://k3s.io +Wants=network-online.target After=network-online.target +[Install] +WantedBy=multi-user.target + [Service] Type=notify -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_agent_args }} +EnvironmentFile=-/etc/default/%N +EnvironmentFile=-/etc/sysconfig/%N +EnvironmentFile=-/etc/systemd/system/k3s.service.env KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead @@ -19,6 +23,7 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s - -[Install] -WantedBy=multi-user.target +ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_agent_args }} \ No newline at end of file diff --git a/roles/k3s/server/tasks/main.yml b/roles/k3s/server/tasks/main.yml index 2d8b0b0..63196c2 100644 --- a/roles/k3s/server/tasks/main.yml +++ b/roles/k3s/server/tasks/main.yml @@ -2,22 +2,30 @@ - name: Init first server node when: ansible_hostname == groups['server'][0] block: - - name: Start temporary service for HA cluster - ansible.builtin.command: - cmd: > - systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server - --cluster-init --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args}} - # noqa: jinja[spacing] - creates: "{{ k3s_server_location }}/server/node-token" - when: groups['server'] | length > 1 - - - name: Start temporary service for single server cluster - ansible.builtin.command: - cmd: > - systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server - --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }} - creates: "{{ k3s_server_location }}/server/node-token" + - name: Copy K3s service file [Single] when: groups['server'] | length == 1 + ansible.builtin.template: + src: "k3s-single.service.j2" + dest: "{{ systemd_dir }}/k3s.service" + owner: root + group: root + mode: 0644 + + - name: Copy K3s service file [HA] + when: groups['server'] | length > 1 + ansible.builtin.template: + src: "k3s-cluster-init.service.j2" + dest: "{{ systemd_dir }}/k3s.service" + owner: root + group: root + mode: 0644 + + - name: Enable and check K3s service + ansible.builtin.systemd: + name: k3s + daemon_reload: true + state: started + enabled: true - name: Create directory .kube ansible.builtin.file: @@ -26,6 +34,10 @@ owner: "{{ ansible_user }}" mode: "u=rwx,g=rx,o=" + - name: Pause to allow server startup + ansible.builtin.pause: + seconds: 10 + - name: Copy config file to user home directory ansible.builtin.copy: src: /etc/rancher/k3s/k3s.yaml @@ -48,15 +60,25 @@ flat: true - name: Start other server if any and verify status + when: + - (groups['server'] | length) > 1 + - ansible_hostname != groups['server'][0] block: - - name: Init additonal server nodes - ansible.builtin.command: - cmd: > - systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server - --token "{{ hostvars[groups['server'][0]]['token'] }}" --server https://{{ api_endpoint }}:{{ api_port }} - --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }} - creates: "{{ k3s_server_location }}/server/node-token" - when: ansible_hostname != groups['server'][0] + - name: Copy K3s service file [HA] + when: groups['server'] | length > 1 + ansible.builtin.template: + src: "k3s-ha.service.j2" + dest: "{{ systemd_dir }}/k3s.service" + owner: root + group: root + mode: 0644 + + - name: Enable and check K3s service + ansible.builtin.systemd: + name: k3s + daemon_reload: true + state: started + enabled: true - name: Verify that all server nodes joined when: (groups['server'] | length) > 1 @@ -68,28 +90,6 @@ retries: 20 delay: 10 changed_when: false - always: - - name: Kill the temporary init service - ansible.builtin.systemd: - name: k3s-init - state: stopped - failed_when: false - -- name: Copy K3s service file - ansible.builtin.template: - src: "k3s-server.service.j2" - dest: "{{ systemd_dir }}/k3s-server.service" - owner: root - group: root - mode: 0644 - register: k3s_service - -- name: Enable and check K3s service - ansible.builtin.systemd: - name: k3s-server - daemon_reload: true - state: restarted - enabled: true - name: Create symlinks ansible.builtin.file: diff --git a/roles/k3s/server/templates/k3s-cluster-init.service.j2 b/roles/k3s/server/templates/k3s-cluster-init.service.j2 new file mode 100644 index 0000000..0b79305 --- /dev/null +++ b/roles/k3s/server/templates/k3s-cluster-init.service.j2 @@ -0,0 +1,28 @@ +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +Wants=network-online.target +After=network-online.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=notify +EnvironmentFile=-/etc/default/%N +EnvironmentFile=-/etc/sysconfig/%N +EnvironmentFile=-/etc/systemd/system/k3s.service.env +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s server --cluster-init --data-dir {{ k3s_server_location }} --token {{ token }} {{ extra_server_args }} \ No newline at end of file diff --git a/roles/k3s/server/templates/k3s-ha.service.j2 b/roles/k3s/server/templates/k3s-ha.service.j2 new file mode 100644 index 0000000..bf61e62 --- /dev/null +++ b/roles/k3s/server/templates/k3s-ha.service.j2 @@ -0,0 +1,28 @@ +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +Wants=network-online.target +After=network-online.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=notify +EnvironmentFile=-/etc/default/%N +EnvironmentFile=-/etc/sysconfig/%N +EnvironmentFile=-/etc/systemd/system/k3s.service.env +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_server_args }} \ No newline at end of file diff --git a/roles/k3s/server/templates/k3s-server.service.j2 b/roles/k3s/server/templates/k3s-single.service.j2 similarity index 75% rename from roles/k3s/server/templates/k3s-server.service.j2 rename to roles/k3s/server/templates/k3s-single.service.j2 index 92a1220..8690939 100644 --- a/roles/k3s/server/templates/k3s-server.service.j2 +++ b/roles/k3s/server/templates/k3s-single.service.j2 @@ -1,13 +1,17 @@ [Unit] Description=Lightweight Kubernetes Documentation=https://k3s.io +Wants=network-online.target After=network-online.target +[Install] +WantedBy=multi-user.target + [Service] Type=notify -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args }} --token {{ token }} +EnvironmentFile=-/etc/default/%N +EnvironmentFile=-/etc/sysconfig/%N +EnvironmentFile=-/etc/systemd/system/k3s.service.env KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead @@ -19,6 +23,6 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s - -[Install] -WantedBy=multi-user.target +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} --token {{ token }} {{ extra_server_args }} \ No newline at end of file diff --git a/roles/prereq/tasks/main.yml b/roles/prereq/tasks/main.yml index 7a7932d..be707b0 100644 --- a/roles/prereq/tasks/main.yml +++ b/roles/prereq/tasks/main.yml @@ -59,12 +59,6 @@ validate: 'visudo -cf %s' when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux','RedHat'] -- name: Make k3s directory - ansible.builtin.file: - path: "/var/lib/rancher" - mode: 0755 - state: directory - - name: Create symlink ansible.builtin.file: dest: /var/lib/rancher/k3s