From 5b3e2a5db6b8a6bba73f298033f156a1e66591ce Mon Sep 17 00:00:00 2001 From: Nicholas Malcolm Date: Mon, 30 Jan 2023 20:25:56 -0500 Subject: [PATCH] Reorganize server tasks, copy config to local - Inital server starts first so it can grab the token for other servers and copy the config to localhost - More cleanup of agent/server naming scheme Signed-off-by: Derek Nola --- .gitignore | 3 +- playbook/reboot.yml | 1 + playbook/site.yml | 2 +- roles/k3s/agent/tasks/main.yml | 6 +- .../{k3s.service.j2 => k3s-agent.service.j2} | 0 roles/k3s/server/defaults/main.yml | 10 -- roles/k3s/server/tasks/main.yml | 158 +++++++++++------- .../{k3s.service.j2 => k3s-server.service.j2} | 0 roles/reset/tasks/main.yml | 18 +- 9 files changed, 114 insertions(+), 84 deletions(-) rename roles/k3s/agent/templates/{k3s.service.j2 => k3s-agent.service.j2} (100%) rename roles/k3s/server/templates/{k3s.service.j2 => k3s-server.service.j2} (100%) diff --git a/.gitignore b/.gitignore index 12dcaf0..8a73ca1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .vscode -.vagrant \ No newline at end of file +.vagrant +inventory.yml diff --git a/playbook/reboot.yml b/playbook/reboot.yml index 1233d92..8e5fbe5 100644 --- a/playbook/reboot.yml +++ b/playbook/reboot.yml @@ -3,6 +3,7 @@ hosts: k3s_cluster become: true gather_facts: true + serial: 1 tasks: - name: Reboot ansible.builtin.reboot: diff --git a/playbook/site.yml b/playbook/site.yml index 2f37b71..e6e4ef0 100644 --- a/playbook/site.yml +++ b/playbook/site.yml @@ -15,7 +15,7 @@ - role: k3s/server - name: Setup K3S agent - hosts: node + hosts: agent become: true roles: - role: k3s/agent diff --git a/roles/k3s/agent/tasks/main.yml b/roles/k3s/agent/tasks/main.yml index 059d594..8167567 100644 --- a/roles/k3s/agent/tasks/main.yml +++ b/roles/k3s/agent/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Copy K3s service file ansible.builtin.template: - src: "k3s.service.j2" + src: "k3s-agent.service.j2" dest: "{{ systemd_dir }}/k3s-agent.service" owner: root group: root @@ -10,6 +10,6 @@ - name: Enable and check K3s service ansible.builtin.systemd: name: k3s-agent - daemon_reload: yes + daemon_reload: true state: restarted - enabled: yes + enabled: true diff --git a/roles/k3s/agent/templates/k3s.service.j2 b/roles/k3s/agent/templates/k3s-agent.service.j2 similarity index 100% rename from roles/k3s/agent/templates/k3s.service.j2 rename to roles/k3s/agent/templates/k3s-agent.service.j2 diff --git a/roles/k3s/server/defaults/main.yml b/roles/k3s/server/defaults/main.yml index b31b388..c56778f 100644 --- a/roles/k3s/server/defaults/main.yml +++ b/roles/k3s/server/defaults/main.yml @@ -1,12 +1,2 @@ --- k3s_server_location: /var/lib/rancher/k3s -server_init_args: >- - {% if groups['server'] | length > 1 %} - --token {{ k3s_token }} - {% if ansible_host == hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) %} - --cluster-init - {% else %} - --server https://{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}:{{ api_port }} - {% endif %} - {% endif %} - {{ extra_server_args | default('') }} diff --git a/roles/k3s/server/tasks/main.yml b/roles/k3s/server/tasks/main.yml index e6e7670..2c4c521 100644 --- a/roles/k3s/server/tasks/main.yml +++ b/roles/k3s/server/tasks/main.yml @@ -1,12 +1,96 @@ --- -- name: Init cluster using temporary service - ansible.builtin.command: - cmd: "systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server {{ server_init_args }}" - creates: "{{ systemd_dir }}/k3s.service" +- name: Clean previous runs of k3s-init + ansible.builtin.command: systemctl reset-failed k3s-init + failed_when: false + changed_when: false -- name: Verification +- name: Init first server node block: - - name: Verify that all nodes joined + - name: Start temporary service with cluster-init + ansible.builtin.command: + cmd: > + systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server + --cluster-init --tls-san {{ api_endpoint }} {{ extra_server_args | default('') }} + creates: "{{ systemd_dir }}/k3s-init.service" + when: groups['server'] | length > 1 + + - name: Start temporary service + ansible.builtin.command: + cmd: > + systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server + --tls-san {{ api_endpoint }} { extra_server_args | default('') }} + creates: "{{ systemd_dir }}/k3s-init.service" + when: groups['server'] | length == 1 + + - name: Wait for node-token + ansible.builtin.wait_for: + path: "{{ k3s_server_location }}/server/node-token" + + - name: Register node-token file access mode + ansible.builtin.stat: + path: "{{ k3s_server_location }}/server/node-token" + register: p + + - name: Change file access node-token + ansible.builtin.file: + path: "{{ k3s_server_location }}/server/node-token" + mode: "g+rx,o+rx" + + - name: Read node-token from server + ansible.builtin.slurp: + path: "{{ k3s_server_location }}/server/node-token" + register: node_token + + - name: Store server node-token + ansible.builtin.set_fact: + token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" + + - name: Restore node-token file access + ansible.builtin.file: + path: "{{ k3s_server_location }}/server/node-token" + mode: "{{ p.stat.mode }}" + + - name: Create directory .kube + ansible.builtin.file: + path: ~{{ ansible_user }}/.kube + state: directory + owner: "{{ ansible_user }}" + mode: "u=rwx,g=rx,o=" + + - name: Copy config file to user home directory + ansible.builtin.copy: + src: /etc/rancher/k3s/k3s.yaml + dest: ~{{ ansible_user }}/.kube/config + remote_src: true + owner: "{{ ansible_user }}" + mode: "u=rw,g=,o=" + + - name: Change server to API endpoint instead of localhost + ansible.builtin.command: >- + k3s kubectl config set-cluster default + --server=https://{{ api_endpoint }}:{{ api_port }} + --kubeconfig ~{{ ansible_user }}/.kube/config + changed_when: true + + - name: Copy kubectl config to local machine + ansible.builtin.fetch: + src: ~{{ ansible_user }}/.kube/config + dest: ~/.kube/config + flat: true + when: ansible_hostname == groups['server'][0] + +- name: Init additonal server nodes + ansible.builtin.command: + cmd: > + systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server --token "{{ hostvars[groups['server'][0]]['token'] }}" + --server https://{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}:{{ api_port }} + {{ extra_server_args | default('') }} + creates: "{{ systemd_dir }}/k3s-init.service" + when: ansible_hostname != groups['server'][0] + +- name: Verification and cleanup + block: + - name: Verify that all server nodes joined ansible.builtin.command: cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/control-plane=true" -o=jsonpath="{.items[*].metadata.name}" register: nodes @@ -23,69 +107,19 @@ - name: Copy K3s service file register: k3s_service - ansible.builtin.template: - src: "k3s.service.j2" - dest: "{{ systemd_dir }}/k3s.service" + template: + src: "k3s-server.service.j2" + dest: "{{ systemd_dir }}/k3s-server.service" owner: root group: root mode: 0644 - name: Enable and check K3s service ansible.builtin.systemd: - name: k3s - daemon_reload: yes + name: k3s-server + daemon_reload: true state: restarted - enabled: yes - -- name: Wait for node-token - ansible.builtin.wait_for: - path: "{{ k3s_server_location }}/server/node-token" - -- name: Register node-token file access mode - ansible.builtin.stat: - path: "{{ k3s_server_location }}/server/node-token" - register: p - -- name: Change file access node-token - ansible.builtin.file: - path: "{{ k3s_server_location }}/server/node-token" - mode: "g+rx,o+rx" - -- name: Read node-token from server - ansible.builtin.slurp: - path: "{{ k3s_server_location }}/server/node-token" - register: node_token - -- name: Store server node-token - ansible.builtin.set_fact: - token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" - -- name: Restore node-token file access - ansible.builtin.file: - path: "{{ k3s_server_location }}/server/node-token" - mode: "{{ p.stat.mode }}" - -- name: Create directory .kube - ansible.builtin.file: - path: ~{{ ansible_user }}/.kube - state: directory - owner: "{{ ansible_user }}" - mode: "u=rwx,g=rx,o=" - -- name: Copy config file to user home directory - ansible.builtin.copy: - src: /etc/rancher/k3s/k3s.yaml - dest: ~{{ ansible_user }}/.kube/config - remote_src: yes - owner: "{{ ansible_user }}" - mode: "u=rw,g=,o=" - -- name: Change server to API endpoint instead of localhost - ansible.builtin.command: >- - /usr/local/bin/k3s kubectl config set-cluster default - --server=https://{{ api_endpoint }}:{{ api_port }} - --kubeconfig ~{{ ansible_user }}/.kube/config - changed_when: true + enabled: true - name: Create kubectl symlink ansible.builtin.file: diff --git a/roles/k3s/server/templates/k3s.service.j2 b/roles/k3s/server/templates/k3s-server.service.j2 similarity index 100% rename from roles/k3s/server/templates/k3s.service.j2 rename to roles/k3s/server/templates/k3s-server.service.j2 diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index e0e1a89..b9de7a6 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -1,14 +1,18 @@ --- +- name: Clean previous runs of k3s-init + command: systemctl reset-failed k3s-init + failed_when: false + changed_when: false + - name: Disable services ansible.builtin.systemd: name: "{{ item }}" state: stopped - enabled: no + enabled: false failed_when: false with_items: - - k3s - - k3s-node - - k3s-init + - k3s-server + - k3s-agent - name: Kill container shim register: pkill_containerd_shim_runc @@ -32,12 +36,12 @@ state: absent with_items: - /usr/local/bin/k3s - - "{{ systemd_dir }}/k3s.service" - - "{{ systemd_dir }}/k3s-node.service" + - "{{ systemd_dir }}/k3s-server.service" + - "{{ systemd_dir }}/k3s-agent.service" - /etc/rancher/k3s - /var/lib/kubelet - /var/lib/rancher/k3s - name: Systemd daemon reload ansible.builtin.systemd: - daemon_reload: yes + daemon_reload: true