Merge branch 'multi_master_support' into feature/add_manifests

.travis.yml

@@ -8,6 +8,7 @@ services: docker
 env:
   global:
     - ROLE_NAME: k3s
+    - MOLECULE_SCENARIO: default
   matrix:
     - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
     - MOLECULE_DISTRO: geerlingguy/docker-centos7-ansible:latest
@@ -37,6 +38,15 @@ env:
     # Test auto deploying manifests
     - MOLECULE_DISTRO: geerlingguy/docker-ubuntu1804-ansible:latest
       MOLECULE_PLAYBOOK: playbook-auto-deploying-manifests.yml
+      
+    # Test multiple masters in control plane with PostgreSQL
+    - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
+      MOLECULE_SCENARIO: highavailability
+
+    # Test multiple masters in control plane with DQLite
+    - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
+      MOLECULE_SCENARIO: highavailability
+      MOLECULE_PLAYBOOK: playbook-dqlite.yml
 
 install:
   # Install test dependencies.
@@ -45,12 +55,12 @@ install:
 before_script:
   # Use actual Ansible Galaxy role name for the project directory.
   - cd ../
-  - mv ansible-role-$ROLE_NAME xanmanning.$ROLE_NAME
-  - cd xanmanning.$ROLE_NAME
+  - mv ansible-role-${ROLE_NAME} xanmanning.${ROLE_NAME}
+  - cd xanmanning.${ROLE_NAME}
 
 script:
   # Run tests.
-  - molecule test
+  - molecule test --scenario-name "${MOLECULE_SCENARIO}"
 
 notifications:
   webhooks: https://galaxy.ansible.com/api/v1/notifications/

README.md

@@ -72,6 +72,11 @@ consistency.
 | `k3s_disable_cloud_controller` | Disable k3s default cloud controller manager.                            | `false`                        |
 | `k3s_disable_network_policy`   | Disable k3s default network policy controller.                           | `false`                        |
 | `k3s_write_kubeconfig_mode`    | Define the file mode from the generated KubeConfig, eg. `644`            | _NULL_                         |
+| `k3s_datastore_endpoint`       | Define the database or etcd cluster endpoint for HA.                     | _NULL_                         |
+| `k3s_datastore_cafile`         | Define the database TLS CA file.                                         | _NULL_                         |
+| `k3s_datastore_certfile`       | Define the database TLS Cert file.                                       | _NULL_                         |
+| `k3s_datastore_keyfile`        | Define the database TLS Key file.                                        | _NULL_                         |
+| `k3s_dqlite_datastore`         | Use DQLite as the database backend for HA. (EXPERIMENTAL)                | `false`                        |
 
 #### Important note about `k3s_release_version`
 
@@ -124,13 +129,25 @@ Below are variables that are set against specific hosts in your inventory.
 | `k3s_server_manifests_templates` | A list of Auto-Deploying Manifests Templates.                            | []                                      |
 
 
-#### Important note about `k3s_control_node`
+#### Important note about `k3s_control_node` and High Availability (HA)
 
-Currently only one host can be defined as a control node, if multiple hosts are
-set to true the play will fail.
+By default only one host will be defined as a control node by Ansible, If you 
+do not set a host as a control node, the role will automatically delegate
+the first play host as a control node (master). This is not suitable for use in
+a Production workload.
 
-If you do not set a host as a control node, the role will automatically delegate
-the first play host as a control node.
+If multiple hosts have `k3s_control_node` set to true, you must also set
+`k3s_datastore_endpoint` as the connection string to a MySQL or PostgreSQL
+database, or etcd cluster else the play will fail.
+
+If using TLS, the CA, Certificate and Key need to already be available on
+the play hosts.
+
+See: [High Availability with an External DB](https://rancher.com/docs/k3s/latest/en/installation/ha/)
+
+Since K3s v1.0.0 it is possible to use DQLite as the backend database, and this
+is done by setting `k3s_dqlite_datastore` to true. As this is an experimental
+feature you will also need to set `k3s_use_experimental` to true.
 
 #### Important note about `k3s_flannel_interface`
 

defaults/main.yml

@@ -24,6 +24,27 @@ k3s_server_manifests_templates: []
 # Use experimental features in k3s?
 k3s_use_experimental: false
 
+# Use a database or etcd cluster to enable HA. Examples below:
+# MySQL:
+#   k3s_datastore_endpoint "mysql://username:password@tcp(hostname:3306)/database-name"
+# PostgreSQL:
+#   k3s_datastore_endpoint: "postgres://username:password@hostname:port/database-name"
+# Etcd:
+#   k3s_datastore_endpoint: "https://etcd-host-1:2379,https://etcd-host-2:2379,https://etcd-host-3:2379"
+k3s_datastore_endpoint: false
+
+# If using a database endpoint for HA, you can optionally set the CA file,
+# Cert file and Key file for connecting to the database using TLS.
+#
+# These need to already be present on the play hosts.
+#
+# k3s_datastore_cafile: /path/to/ca.crt
+# k3s_datastore_certfile: /path/to/cert.crt
+# k3s_datastore_keyfile: /path/to/key.pem
+
+# Use DQLite for HA Datastore? (EXPERIMENTAL)
+k3s_dqlite_datastore: false
+
 # Are control hosts also worker nodes?
 k3s_control_workers: true
 

molecule/default/prepare-rootless.yml

@@ -1,6 +1,6 @@
 ---
 - name: Prepare
-  hosts: all
+  hosts: node1
   become: true
   tasks:
     - name: Ensure a user group exists

molecule/highavailability/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

molecule/highavailability/INSTALL.rst

@@ -0,0 +1,22 @@
+*******
+Docker driver installation guide
+*******
+
+Requirements
+============
+
+* Docker Engine
+
+Install
+=======
+
+Please refer to the `Virtual environment`_ documentation for installation best
+practices. If not using a virtual environment, please consider passing the
+widely recommended `'--user' flag`_ when invoking ``pip``.
+
+.. _Virtual environment: https://virtualenv.pypa.io/en/latest/
+.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site
+
+.. code-block:: bash
+
+    $ pip install 'molecule[docker]'

molecule/highavailability/molecule.yml

@@ -0,0 +1,67 @@
+---
+
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: node1
+    image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    networks:
+      - name: k3snet
+  - name: node2
+    image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    networks:
+      - name: k3snet
+  - name: node3
+    image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    networks:
+      - name: k3snet
+  - name: database
+    image: postgres:11-alpine
+    pre_build_image: true
+    command: "postgres"
+    env:
+      POSTGRES_PASSWORD: "verybadpass"
+    networks:
+      - name: k3snet
+  - name: loadbalancer
+    image: nginx:1.17-alpine
+    pre_build_image: true
+    ports:
+      - "6443:6443"
+    volumes:
+      - ${MOLECULE_SCENARIO_DIRECTORY}/nginx-loadbalancer.conf:/etc/nginx/nginx.conf:ro
+    command: "nginx -g 'daemon off;'"
+    networks:
+      - name: k3snet
+provisioner:
+  name: ansible
+  options:
+    verbose: true
+  lint:
+    name: ansible-lint
+  playbooks:
+    prepare: ${MOLECULE_PREPARE_PLAYBOOK:-prepare.yml}
+    converge: ${MOLECULE_PLAYBOOK:-playbook.yml}
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

molecule/highavailability/nginx-loadbalancer.conf

@@ -0,0 +1,34 @@
+worker_processes 4;
+worker_rlimit_nofile 40000;
+
+events {
+    worker_connections 8192;
+}
+
+stream {
+    # Set up our upstream of control (master) nodes. The default load balancing
+    # algorithm for nginx is to round-robin. Perfect!
+    upstream control_plane {
+        server node2:6443 max_fails=3 fail_timeout=5s;
+        server node3:6443 max_fails=3 fail_timeout=5s;
+    }
+
+    upstream control_plane_443 {
+        server node2:443 max_fails=3 fail_timeout=5s;
+        server node3:443 max_fails=3 fail_timeout=5s;
+    }
+
+    # Listen on port 6443, this is our default control plane port, then pass
+    # all traffic to one of the control (master) nodes.
+    server {
+        listen 6443;
+        proxy_pass control_plane;
+    }
+
+    # Listen on port 443, this is our default ssl port, then pass
+    # all traffic to one of the control (master) nodes.
+    server {
+        listen 443;
+        proxy_pass control_plane_443;
+    }
+}

molecule/highavailability/playbook-dqlite.yml

@@ -0,0 +1,15 @@
+---
+- name: Converge
+  hosts: node*
+  become: true
+  vars:
+    molecule_is_test: true
+    k3s_dqlite_datastore: true
+    k3s_use_experimental: true
+  pre_tasks:
+    - name: Set each node to be a control node
+      set_fact:
+        k3s_control_node: true
+      when: inventory_hostname in ['node2', 'node3']
+  roles:
+    - role: xanmanning.k3s

molecule/highavailability/playbook.yml

@@ -0,0 +1,15 @@
+---
+- name: Converge
+  hosts: node*
+  become: true
+  vars:
+    molecule_is_test: true
+    k3s_control_node_address: loadbalancer
+    k3s_datastore_endpoint: "postgres://postgres:verybadpass@database:5432/postgres?sslmode=disable"
+  pre_tasks:
+    - name: Set each node to be a control node
+      set_fact:
+        k3s_control_node: true
+      when: inventory_hostname in ['node2', 'node3']
+  roles:
+    - role: xanmanning.k3s

molecule/highavailability/prepare.yml

@@ -0,0 +1,7 @@
+---
+- name: Prepare
+  hosts: node*
+  tasks:
+    - name: Debug Message
+      debug:
+        msg: No prepare steps required

molecule/highavailability/tests/test_default.py

@@ -0,0 +1,14 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_hosts_file(host):
+    f = host.file('/etc/hosts')
+
+    assert f.exists
+    assert f.user == 'root'
+    assert f.group == 'root'

tasks/build/configure-k3s-cluster.yml

@@ -8,12 +8,14 @@
       @@@
       {{ hostvars[item].ansible_host | default(hostvars[item].ansible_fqdn) }}
       @@@
-      {{ hostvars[item].k3s_control_node }}
+      C_{{ hostvars[item].k3s_control_node }}
+      @@@
+      P_{{ hostvars[item].k3s_primary_control_node | default(False) }}
     create: true
   loop: "{{ play_hosts }}"
 
 - name: Lookup control node from file
-  command: "grep 'True' /tmp/inventory.txt"
+  command: "grep '{{ 'P_True' if (k3s_controller_count | length > 1) else 'C_True' }}' /tmp/inventory.txt"
   changed_when: false
   register: k3s_control_delegate_raw
 

tasks/build/install-k3s.yml

@@ -5,13 +5,15 @@
     path: "{{ k3s_systemd_unit_directory }}"
     state: directory
     recurse: true
-  when: k3s_control_node
+  when: (k3s_control_node and k3s_controller_count | length == 1)
+        or (k3s_primary_control_node and k3s_controller_count | length > 1)
 
 - name: Ensure k3s service unit file is present on control plane
   template:
     src: k3s.service.j2
     dest: "{{ k3s_systemd_unit_directory }}/k3s.service"
-  when: k3s_control_node
+  when: (k3s_control_node and k3s_controller_count | length == 1)
+        or (k3s_primary_control_node and k3s_controller_count | length > 1)
   notify:
     - reload systemd
 
@@ -36,4 +38,5 @@
     state: started
     enabled: true
     scope: "{{ k3s_systemd_context }}"
-  when: k3s_control_node
+  when: (k3s_control_node and k3s_controller_count | length == 1)
+        or (k3s_primary_control_node and k3s_controller_count | length > 1)

tasks/build/preconfigure-k3s.yml

@@ -5,6 +5,11 @@
     k3s_control_node: false
   when: k3s_control_node is not defined
 
+- name: Ensure k3s master control node fact is set
+  set_fact:
+    k3s_primary_control_node: false
+  when: k3s_primary_control_node is not defined
+
 - name: Ensure a k3s control node is defined if none are found in play_hosts
   block:
     - name: Set control host
@@ -16,5 +21,11 @@
 - name: Ensure a count of control masters is generated
   set_fact:
     k3s_controller_count: "{{ k3s_controller_count + [ item ] }}"
-  when: item
-  loop: "{{ hostvars | json_query('*.k3s_control_node') }}"
+  when: hostvars[item].k3s_control_node
+  loop: "{{ play_hosts }}"
+
+- name: Ensure a primary k3s control node is defined if multiple are found in play_hosts
+  set_fact:
+    k3s_primary_control_node: true
+  when: k3s_controller_count | length > 1
+        and inventory_hostname == k3s_controller_count[0]

tasks/validate/check-experimental-variables.yml

@@ -1,9 +1,10 @@
 ---
 
-- name: Check k3s_non_root is enabled with k3s_use_experimental
+- name: Check if any experimental variables are configure and if they are enabled with k3s_use_experimental
   assert:
     that:
       - k3s_use_experimental is defined and k3s_use_experimental
     success_msg: "Experimental variables are defined and enabled."
     fail_msg: "Experimental variables have been configured. If you want to use them ensure you set k3s_use_experimental"
-  when: k3s_non_root is defined and k3s_non_root
+  when: (k3s_non_root is defined and k3s_non_root)
+        or (k3s_dqlite_datastore is defined and k3s_dqlite_datastore)

tasks/validate/check-master-count.yml

@@ -1,8 +1,21 @@
 ---
 
-- name: Check if multiple controllers defined
+- name: Check the conditions when a single controller is defined
   assert:
     that:
-      - k3s_controller_count | length == 1
-    success_msg: "Only one controller has been defined."
-    fail_msg: "Multiple controllers have been defined. This is not yet supported."
+      - (k3s_controller_count | length == 1)
+        and (k3s_datastore_endpoint is not defined or not k3s_datastore_endpoint)
+        and (k3s_dqlite_datastore is not defined or not k3s_dqlite_datastore)
+    success_msg: "Control plane configuration is valid."
+    fail_msg: "Control plane configuration is invalid. Please see notes about k3s_control_node and HA in README.md."
+  when: k3s_controller_count | length == 1
+
+- name: Check the conditions when multiple controllers are defined
+  assert:
+    that:
+      - (k3s_controller_count | length >= 2)
+        and ((k3s_datastore_endpoint is defined and k3s_datastore_endpoint)
+              or k3s_dqlite_datastore is defined and k3s_dqlite_datastore)
+    success_msg: "Control plane configuration is valid."
+    fail_msg: "Control plane configuration is invalid. Please see notes about k3s_control_node and HA in README.md."
+  when: k3s_controller_count | length >= 2

tasks/validate/check-variables.yml

@@ -137,3 +137,43 @@
     success_msg: "--disable-scheduler supported in {{ k3s_release_version }}"
     fail_msg: "--disable-scheduler not supported in {{ k3s_release_version }}"
   when: k3s_disable_scheduler is defined and k3s_disable_scheduler
+
+- name: Check k3s_datastore_endpoint against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_endpoint is defined and k3s_datastore_endpoint
+
+- name: Check k3s_dqlite_datastore against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--cluster-init supported in {{ k3s_release_version }}"
+    fail_msg: "--cluster-init not supported in {{ k3s_release_version }}"
+  when: k3s_dqlite_datastore is defined and k3s_dqlite_datastore
+
+- name: Check k3s_datastore_cafile against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_cafile is defined and k3s_datastore_cafile
+
+- name: Check k3s_datastore_certfile against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_certfile is defined and k3s_datastore_certfile
+
+- name: Check k3s_datastore_keyfile against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_keyfile is defined and k3s_datastore_keyfile

templates/k3s.service.j2

@@ -54,6 +54,26 @@ ExecStart={{ k3s_install_dir }}/k3s
     {% if k3s_cluster_domain is defined and k3s_cluster_domain != "cluster.local" %}
         --cluster-domain {{ k3s_cluster_domain }}
     {% endif %}
+    {% if k3s_datastore_endpoint is defined and k3s_datastore_endpoint %}
+        --datastore-endpoint "{{ k3s_datastore_endpoint }}"
+        {% if k3s_datastore_cafile is defined and k3s_datastore_cafile %}
+            --datastore-cafile {{ k3s_datastore_cafile }}
+        {% endif %}
+        {% if k3s_datastore_certfile is defined and k3s_datastore_certfile %}
+            --datastore-certfile {{ k3s_datastore_certfile }}
+        {% endif %}
+        {% if k3s_datastore_keyfile is defined and k3s_datastore_keyfile %}
+            --datastore-keyfile {{ k3s_datastore_keyfile }}
+        {% endif %}
+    {% endif %}
+    {% if k3s_dqlite_datastore is defined and k3s_dqlite_datastore %}
+        {% if k3s_primary_control_node is defined and k3s_primary_control_node %}
+            --cluster-init
+        {% else %}
+            --server https://{{ k3s_control_node_address }}:{{ k3s_https_port }}
+            --token {{ k3s_control_token }}
+        {% endif %}
+    {% endif %}
 {% else %}
     agent
     --server https://{{ k3s_control_node_address }}:{{ k3s_https_port }}