From 734e49a7e5c76bf4fae626176c59d292941d702d Mon Sep 17 00:00:00 2001
From: Xan Manning <xan.manning@gmail.com>
Date: Sat, 11 Jan 2020 12:31:23 +0000
Subject: [PATCH 1/5] Documentation, and validation logic for HA configuration
 added.

---
 README.md                                     | 27 ++++++++++---
 defaults/main.yml                             | 21 ++++++++++
 .../validate/check-experimental-variables.yml |  3 +-
 tasks/validate/check-master-count.yml         | 21 ++++++++--
 tasks/validate/check-variables.yml            | 40 +++++++++++++++++++
 5 files changed, 102 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 573f7e6..6df7735 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,11 @@ consistency.
 | `k3s_disable_cloud_controller` | Disable k3s default cloud controller manager.                            | `false`                        |
 | `k3s_disable_network_policy`   | Disable k3s default network policy controller.                           | `false`                        |
 | `k3s_write_kubeconfig_mode`    | Define the file mode from the generated KubeConfig, eg. `644`            | _NULL_                         |
+| `k3s_datastore_endpoint`       | Define the database or etcd cluster endpoint for HA.                     | _NULL_                         |
+| `k3s_datastore_cafile`         | Define the database TLS CA file.                                         | _NULL_                         |
+| `k3s_datastore_certfile`       | Define the database TLS Cert file.                                       | _NULL_                         |
+| `k3s_datastore_keyfile`        | Define the database TLS Key file.                                        | _NULL_                         |
+| `k3s_dqlite_datastore`         | Use DQLite as the database backend for HA. (EXPERIMENTAL)                | `false`                        |
 
 #### Important note about `k3s_release_version`
 
@@ -121,13 +126,25 @@ Below are variables that are set against specific hosts in your inventory.
 | `k3s_node_data_dir`         | Folder to hold state.                                                    | `/var/lib/rancher/k3s` |
 | `k3s_tls_san`               | Add additional hosname or IP as Subject Alternate Name in the TLS cert.  | _NULL_                 |
 
-#### Important note about `k3s_control_node`
+#### Important note about `k3s_control_node` and High Availability (HA)
 
-Currently only one host can be defined as a control node, if multiple hosts are
-set to true the play will fail.
+By default only one host will be defined as a control node by Ansible, If you 
+do not set a host as a control node, the role will automatically delegate
+the first play host as a control node (master). This is not suitable for use in
+a Production workload.
 
-If you do not set a host as a control node, the role will automatically delegate
-the first play host as a control node.
+If multiple hosts have `k3s_control_node` set to true, you must also set
+`k3s_datastore_endpoint` as the connection string to a MySQL or PostgreSQL
+database, or etcd cluster else the play will fail.
+
+If using TLS, the CA, Certificate and Key need to already be available on
+the play hosts.
+
+See: [High Availability with an External DB](https://rancher.com/docs/k3s/latest/en/installation/ha/)
+
+Since K3s v1.0.0 it is possible to use DQLite as the backend database, and this
+is done by setting `k3s_dqlite_datastore` to true. As this is an experimental
+feature you will also need to set `k3s_use_experimental` to true.
 
 #### Important note about `k3s_flannel_interface`
 
diff --git a/defaults/main.yml b/defaults/main.yml
index e4f27a2..1d077b7 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -17,6 +17,27 @@ k3s_install_dir: /usr/local/bin
 # Use experimental features in k3s?
 k3s_use_experimental: false
 
+# Use a database or etcd cluster to enable HA. Examples below:
+# MySQL:
+#   k3s_datastore_endpoint "mysql://username:password@tcp(hostname:3306)/database-name"
+# PostgreSQL:
+#   k3s_datastore_endpoint: "postgres://username:password@hostname:port/database-name"
+# Etcd:
+#   k3s_datastore_endpoint: "https://etcd-host-1:2379,https://etcd-host-2:2379,https://etcd-host-3:2379"
+k3s_datastore_endpoint: false
+
+# If using a database endpoint for HA, you can optionally set the CA file,
+# Cert file and Key file for connecting to the database using TLS.
+#
+# These need to already be present on the play hosts.
+#
+# k3s_datastore_cafile: /path/to/ca.crt
+# k3s_datastore_certfile: /path/to/cert.crt
+# k3s_datastore_keyfile: /path/to/key.pem
+
+# Use DQLite for HA Datastore? (EXPERIMENTAL)
+k3s_dqlite_datastore: false
+
 # Are control hosts also worker nodes?
 k3s_control_workers: true
 
diff --git a/tasks/validate/check-experimental-variables.yml b/tasks/validate/check-experimental-variables.yml
index ee9b16a..fc3c992 100644
--- a/tasks/validate/check-experimental-variables.yml
+++ b/tasks/validate/check-experimental-variables.yml
@@ -1,9 +1,10 @@
 ---
 
-- name: Check k3s_non_root is enabled with k3s_use_experimental
+- name: Check if any experimental variables are configure and if they are enabled with k3s_use_experimental
   assert:
     that:
       - k3s_use_experimental is defined and k3s_use_experimental
+      - k3s_dqlite_datastore is defined and k3s_use_experimental
     success_msg: "Experimental variables are defined and enabled."
     fail_msg: "Experimental variables have been configured. If you want to use them ensure you set k3s_use_experimental"
   when: k3s_non_root is defined and k3s_non_root
diff --git a/tasks/validate/check-master-count.yml b/tasks/validate/check-master-count.yml
index 5c34ccd..8e248ca 100644
--- a/tasks/validate/check-master-count.yml
+++ b/tasks/validate/check-master-count.yml
@@ -1,8 +1,21 @@
 ---
 
-- name: Check if multiple controllers defined
+- name: Check the conditions when a single controller is defined
   assert:
     that:
-      - k3s_controller_count | length == 1
-    success_msg: "Only one controller has been defined."
-    fail_msg: "Multiple controllers have been defined. This is not yet supported."
+      - (k3s_controller_count | length == 1)
+        and (k3s_datastore_endpoint is not defined or not k3s_datastore_endpoint)
+        and (k3s_dqlite_datastore is not defined or not k3s_dqlite_datastore)
+    success_msg: "Control plane configuration is valid."
+    fail_msg: "Control plane configuration is invalid. Please see notes about k3s_control_node and HA in README.md."
+  when: k3s_controller_count | length == 1
+
+- name: Check the conditions when multiple controllers are defined
+  assert:
+    that:
+      - (k3s_controller_count | length >= 2)
+        and ((k3s_datastore_endpoint is defined and k3s_datastore_endpoint)
+              or k3s_dqlite_datastore is defined and k3s_dqlite_datastore)
+    success_msg: "Control plane configuration is valid."
+    fail_msg: "Control plane configuration is invalid. Please see notes about k3s_control_node and HA in README.md."
+  when: k3s_controller_count | length >= 2
diff --git a/tasks/validate/check-variables.yml b/tasks/validate/check-variables.yml
index 263735d..f5f9dd6 100644
--- a/tasks/validate/check-variables.yml
+++ b/tasks/validate/check-variables.yml
@@ -137,3 +137,43 @@
     success_msg: "--disable-scheduler supported in {{ k3s_release_version }}"
     fail_msg: "--disable-scheduler not supported in {{ k3s_release_version }}"
   when: k3s_disable_scheduler is defined and k3s_disable_scheduler
+
+- name: Check k3s_datastore_endpoint against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_endpoint is defined and k3s_datastore_endpoint
+
+- name: Check k3s_dqlite_datastore against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--cluster-init supported in {{ k3s_release_version }}"
+    fail_msg: "--cluster-init not supported in {{ k3s_release_version }}"
+  when: k3s_dqlite_datastore is defined and k3s_dqlite_datastore
+
+- name: Check k3s_datastore_cafile against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_cafile is defined and k3s_datastore_cafile
+
+- name: Check k3s_datastore_certfile against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_certfile is defined and k3s_datastore_certfile
+
+- name: Check k3s_datastore_keyfile against k3s version
+  assert:
+    that:
+      - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
+    success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
+    fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
+  when: k3s_datastore_keyfile is defined and k3s_datastore_keyfile

From c3ae2b79eb56854c730f98a1f479883d87e0ab19 Mon Sep 17 00:00:00 2001
From: Xan Manning <xan.manning@gmail.com>
Date: Sat, 11 Jan 2020 19:20:52 +0000
Subject: [PATCH 2/5] Added database container and proved connectivity. Logic
 needs to be changed for HA.

---
 .travis.yml                                         |  4 ++++
 molecule/default/molecule.yml                       |  8 ++++++++
 .../default/playbook-docker-altport-wireguard.yml   |  2 +-
 molecule/default/playbook-docker.yml                |  2 +-
 molecule/default/playbook-download.yml              |  2 +-
 molecule/default/playbook-ha-postgres.yml           | 13 +++++++++++++
 molecule/default/playbook-no-deploy.yml             |  2 +-
 molecule/default/playbook-restart-cluster.yml       |  2 +-
 molecule/default/playbook-start-cluster.yml         |  2 +-
 molecule/default/playbook-stop-cluster.yml          |  2 +-
 molecule/default/playbook-version.yml               |  2 +-
 molecule/default/playbook.yml                       |  2 +-
 molecule/default/prepare-rootless.yml               |  2 +-
 molecule/default/prepare.yml                        |  2 +-
 tasks/validate/check-experimental-variables.yml     |  4 ++--
 templates/k3s.service.j2                            |  3 +++
 16 files changed, 41 insertions(+), 13 deletions(-)
 create mode 100644 molecule/default/playbook-ha-postgres.yml

diff --git a/.travis.yml b/.travis.yml
index 5664c3a..60015f4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,6 +34,10 @@ env:
     - MOLECULE_DISTRO: geerlingguy/docker-fedora31-ansible:latest
       MOLECULE_PLAYBOOK: playbook-no-deploy.yml
 
+    # Test multiple masters in control plane with PostgreSQL
+    - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
+      MOLECULE_PLAYBOOK: playbook-ha-postgres.yml
+
 install:
   # Install test dependencies.
   - pip install molecule docker jmespath
diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml
index b0173c6..5a3ae42 100644
--- a/molecule/default/molecule.yml
+++ b/molecule/default/molecule.yml
@@ -34,6 +34,14 @@ platforms:
     pre_build_image: ${MOLECULE_PREBUILT:-true}
     networks:
       - name: k3snet
+  - name: database
+    image: postgres:11-alpine
+    pre_build_image: true
+    command: "postgres"
+    env:
+      POSTGRES_PASSWORD: "verybadpass"
+    networks:
+      - name: k3snet
 provisioner:
   name: ansible
   options:
diff --git a/molecule/default/playbook-docker-altport-wireguard.yml b/molecule/default/playbook-docker-altport-wireguard.yml
index 1db8114..e417c05 100644
--- a/molecule/default/playbook-docker-altport-wireguard.yml
+++ b/molecule/default/playbook-docker-altport-wireguard.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-docker.yml b/molecule/default/playbook-docker.yml
index a780b33..9d93375 100644
--- a/molecule/default/playbook-docker.yml
+++ b/molecule/default/playbook-docker.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-download.yml b/molecule/default/playbook-download.yml
index 9e3efc4..5f869bc 100644
--- a/molecule/default/playbook-download.yml
+++ b/molecule/default/playbook-download.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-ha-postgres.yml b/molecule/default/playbook-ha-postgres.yml
new file mode 100644
index 0000000..ec17daa
--- /dev/null
+++ b/molecule/default/playbook-ha-postgres.yml
@@ -0,0 +1,13 @@
+---
+- name: Converge
+  hosts: node*
+  become: true
+  vars:
+    molecule_is_test: true
+    k3s_datastore_endpoint: "postgres://postgres:verybadpass@database:5432/postgres?sslmode=disable"
+  pre_tasks:
+    - name: Set each node to be a control node
+      set_fact:
+        k3s_control_node: true
+  roles:
+    - role: xanmanning.k3s
diff --git a/molecule/default/playbook-no-deploy.yml b/molecule/default/playbook-no-deploy.yml
index 16a8ab4..fece13b 100644
--- a/molecule/default/playbook-no-deploy.yml
+++ b/molecule/default/playbook-no-deploy.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-restart-cluster.yml b/molecule/default/playbook-restart-cluster.yml
index d89782b..45add58 100644
--- a/molecule/default/playbook-restart-cluster.yml
+++ b/molecule/default/playbook-restart-cluster.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-start-cluster.yml b/molecule/default/playbook-start-cluster.yml
index e206333..bc9fbbf 100644
--- a/molecule/default/playbook-start-cluster.yml
+++ b/molecule/default/playbook-start-cluster.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-stop-cluster.yml b/molecule/default/playbook-stop-cluster.yml
index 477b431..1c1ab52 100644
--- a/molecule/default/playbook-stop-cluster.yml
+++ b/molecule/default/playbook-stop-cluster.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-version.yml b/molecule/default/playbook-version.yml
index 740d9c9..5f8cac9 100644
--- a/molecule/default/playbook-version.yml
+++ b/molecule/default/playbook-version.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml
index 394481e..92c2184 100644
--- a/molecule/default/playbook.yml
+++ b/molecule/default/playbook.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: all
+  hosts: node*
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/prepare-rootless.yml b/molecule/default/prepare-rootless.yml
index df685fb..e517e70 100644
--- a/molecule/default/prepare-rootless.yml
+++ b/molecule/default/prepare-rootless.yml
@@ -1,6 +1,6 @@
 ---
 - name: Prepare
-  hosts: all
+  hosts: node1
   become: true
   tasks:
     - name: Ensure a user group exists
diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml
index 5aed1ec..567499e 100644
--- a/molecule/default/prepare.yml
+++ b/molecule/default/prepare.yml
@@ -1,6 +1,6 @@
 ---
 - name: Prepare
-  hosts: all
+  hosts: node*
   tasks:
     - name: Debug Message
       debug:
diff --git a/tasks/validate/check-experimental-variables.yml b/tasks/validate/check-experimental-variables.yml
index fc3c992..0132564 100644
--- a/tasks/validate/check-experimental-variables.yml
+++ b/tasks/validate/check-experimental-variables.yml
@@ -4,7 +4,7 @@
   assert:
     that:
       - k3s_use_experimental is defined and k3s_use_experimental
-      - k3s_dqlite_datastore is defined and k3s_use_experimental
     success_msg: "Experimental variables are defined and enabled."
     fail_msg: "Experimental variables have been configured. If you want to use them ensure you set k3s_use_experimental"
-  when: k3s_non_root is defined and k3s_non_root
+  when: (k3s_non_root is defined and k3s_non_root)
+        or (k3s_dqlite_datastore is defined and k3s_dqlite_datastore)
diff --git a/templates/k3s.service.j2 b/templates/k3s.service.j2
index 1a127e9..8b1dfb5 100644
--- a/templates/k3s.service.j2
+++ b/templates/k3s.service.j2
@@ -54,6 +54,9 @@ ExecStart={{ k3s_install_dir }}/k3s
     {% if k3s_cluster_domain is defined and k3s_cluster_domain != "cluster.local" %}
         --cluster-domain {{ k3s_cluster_domain }}
     {% endif %}
+    {% if k3s_datastore_endpoint is defined and k3s_datastore_endpoint %}
+        --datastore-endpoint "{{ k3s_datastore_endpoint }}"
+    {% endif %}
 {% else %}
     agent
     --server https://{{ k3s_control_node_address }}:{{ k3s_https_port }}

From 09fc37e6ec8cf9a042e7968a0e6c1a498df9e699 Mon Sep 17 00:00:00 2001
From: Xan Manning <xan.manning@gmail.com>
Date: Sat, 11 Jan 2020 22:42:29 +0000
Subject: [PATCH 3/5] Fixed provisioning of multi-master, need to test LB with
 k3s_control_node_address

---
 molecule/default/playbook-ha-postgres.yml |  1 +
 tasks/build/install-k3s.yml               |  9 ++++++---
 tasks/build/preconfigure-k3s.yml          | 15 +++++++++++++--
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/molecule/default/playbook-ha-postgres.yml b/molecule/default/playbook-ha-postgres.yml
index ec17daa..c1a613b 100644
--- a/molecule/default/playbook-ha-postgres.yml
+++ b/molecule/default/playbook-ha-postgres.yml
@@ -9,5 +9,6 @@
     - name: Set each node to be a control node
       set_fact:
         k3s_control_node: true
+      when: inventory_hostname in ['node2', 'node3']
   roles:
     - role: xanmanning.k3s
diff --git a/tasks/build/install-k3s.yml b/tasks/build/install-k3s.yml
index 476845a..8de5737 100644
--- a/tasks/build/install-k3s.yml
+++ b/tasks/build/install-k3s.yml
@@ -5,13 +5,15 @@
     path: "{{ k3s_systemd_unit_directory }}"
     state: directory
     recurse: true
-  when: k3s_control_node
+  when: (k3s_control_node and k3s_controller_count | length == 1)
+        or (k3s_primary_control_node and k3s_controller_count | length > 1)
 
 - name: Ensure k3s service unit file is present on control plane
   template:
     src: k3s.service.j2
     dest: "{{ k3s_systemd_unit_directory }}/k3s.service"
-  when: k3s_control_node
+  when: (k3s_control_node and k3s_controller_count | length == 1)
+        or (k3s_primary_control_node and k3s_controller_count | length > 1)
   notify:
     - reload systemd
 
@@ -36,4 +38,5 @@
     state: started
     enabled: true
     scope: "{{ k3s_systemd_context }}"
-  when: k3s_control_node
+  when: (k3s_control_node and k3s_controller_count | length == 1)
+        or (k3s_primary_control_node and k3s_controller_count | length > 1)
diff --git a/tasks/build/preconfigure-k3s.yml b/tasks/build/preconfigure-k3s.yml
index 50cf661..b83b8cc 100644
--- a/tasks/build/preconfigure-k3s.yml
+++ b/tasks/build/preconfigure-k3s.yml
@@ -5,6 +5,11 @@
     k3s_control_node: false
   when: k3s_control_node is not defined
 
+- name: Ensure k3s master control node fact is set
+  set_fact:
+    k3s_primary_control_node: false
+  when: k3s_primary_control_node is not defined
+
 - name: Ensure a k3s control node is defined if none are found in play_hosts
   block:
     - name: Set control host
@@ -16,5 +21,11 @@
 - name: Ensure a count of control masters is generated
   set_fact:
     k3s_controller_count: "{{ k3s_controller_count + [ item ] }}"
-  when: item
-  loop: "{{ hostvars | json_query('*.k3s_control_node') }}"
+  when: hostvars[item].k3s_control_node
+  loop: "{{ play_hosts }}"
+
+- name: Ensure a primary k3s control node is defined if multiple are found in play_hosts
+  set_fact:
+    k3s_primary_control_node: true
+  when: k3s_controller_count | length > 1
+        and inventory_hostname == k3s_controller_count[0]

From 7e7cf2b97ddcdc5468f70ed9a10b9412fb4b55f1 Mon Sep 17 00:00:00 2001
From: Xan Manning <xan.manning@gmail.com>
Date: Sun, 12 Jan 2020 12:50:03 +0000
Subject: [PATCH 4/5] Moved HA testing to a new scenario

---
 .travis.yml                                   |   9 +--
 molecule/default/molecule.yml                 |   8 ---
 .../playbook-docker-altport-wireguard.yml     |   2 +-
 molecule/default/playbook-docker.yml          |   2 +-
 molecule/default/playbook-download.yml        |   2 +-
 molecule/default/playbook-no-deploy.yml       |   2 +-
 molecule/default/playbook-restart-cluster.yml |   2 +-
 molecule/default/playbook-start-cluster.yml   |   2 +-
 molecule/default/playbook-stop-cluster.yml    |   2 +-
 molecule/default/playbook-version.yml         |   2 +-
 molecule/default/playbook.yml                 |   2 +-
 molecule/default/prepare.yml                  |   2 +-
 molecule/highavailability/Dockerfile.j2       |  14 ++++
 molecule/highavailability/INSTALL.rst         |  22 ++++++
 molecule/highavailability/molecule.yml        |  67 ++++++++++++++++++
 .../highavailability/nginx-loadbalancer.conf  |  20 ++++++
 .../playbook.yml}                             |   1 +
 molecule/highavailability/prepare.yml         |   7 ++
 .../highavailability/tests/test_default.py    |  14 ++++
 .../highavailability/tests/test_default.pyc   | Bin 0 -> 1025 bytes
 20 files changed, 160 insertions(+), 22 deletions(-)
 create mode 100644 molecule/highavailability/Dockerfile.j2
 create mode 100644 molecule/highavailability/INSTALL.rst
 create mode 100644 molecule/highavailability/molecule.yml
 create mode 100644 molecule/highavailability/nginx-loadbalancer.conf
 rename molecule/{default/playbook-ha-postgres.yml => highavailability/playbook.yml} (89%)
 create mode 100644 molecule/highavailability/prepare.yml
 create mode 100644 molecule/highavailability/tests/test_default.py
 create mode 100644 molecule/highavailability/tests/test_default.pyc

diff --git a/.travis.yml b/.travis.yml
index 60015f4..2687786 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,7 @@ services: docker
 env:
   global:
     - ROLE_NAME: k3s
+    - MOLECULE_SCENARIO: default
   matrix:
     - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
     - MOLECULE_DISTRO: geerlingguy/docker-centos7-ansible:latest
@@ -36,7 +37,7 @@ env:
 
     # Test multiple masters in control plane with PostgreSQL
     - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
-      MOLECULE_PLAYBOOK: playbook-ha-postgres.yml
+      MOLECULE_SCENATIO: highavailability
 
 install:
   # Install test dependencies.
@@ -45,12 +46,12 @@ install:
 before_script:
   # Use actual Ansible Galaxy role name for the project directory.
   - cd ../
-  - mv ansible-role-$ROLE_NAME xanmanning.$ROLE_NAME
-  - cd xanmanning.$ROLE_NAME
+  - mv ansible-role-${ROLE_NAME} xanmanning.${ROLE_NAME}
+  - cd xanmanning.${ROLE_NAME}
 
 script:
   # Run tests.
-  - molecule test
+  - molecule test --scenario-name "${MOLECULE_SCENARIO}"
 
 notifications:
   webhooks: https://galaxy.ansible.com/api/v1/notifications/
diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml
index 5a3ae42..b0173c6 100644
--- a/molecule/default/molecule.yml
+++ b/molecule/default/molecule.yml
@@ -34,14 +34,6 @@ platforms:
     pre_build_image: ${MOLECULE_PREBUILT:-true}
     networks:
       - name: k3snet
-  - name: database
-    image: postgres:11-alpine
-    pre_build_image: true
-    command: "postgres"
-    env:
-      POSTGRES_PASSWORD: "verybadpass"
-    networks:
-      - name: k3snet
 provisioner:
   name: ansible
   options:
diff --git a/molecule/default/playbook-docker-altport-wireguard.yml b/molecule/default/playbook-docker-altport-wireguard.yml
index e417c05..1db8114 100644
--- a/molecule/default/playbook-docker-altport-wireguard.yml
+++ b/molecule/default/playbook-docker-altport-wireguard.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-docker.yml b/molecule/default/playbook-docker.yml
index 9d93375..a780b33 100644
--- a/molecule/default/playbook-docker.yml
+++ b/molecule/default/playbook-docker.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-download.yml b/molecule/default/playbook-download.yml
index 5f869bc..9e3efc4 100644
--- a/molecule/default/playbook-download.yml
+++ b/molecule/default/playbook-download.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-no-deploy.yml b/molecule/default/playbook-no-deploy.yml
index fece13b..16a8ab4 100644
--- a/molecule/default/playbook-no-deploy.yml
+++ b/molecule/default/playbook-no-deploy.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-restart-cluster.yml b/molecule/default/playbook-restart-cluster.yml
index 45add58..d89782b 100644
--- a/molecule/default/playbook-restart-cluster.yml
+++ b/molecule/default/playbook-restart-cluster.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-start-cluster.yml b/molecule/default/playbook-start-cluster.yml
index bc9fbbf..e206333 100644
--- a/molecule/default/playbook-start-cluster.yml
+++ b/molecule/default/playbook-start-cluster.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-stop-cluster.yml b/molecule/default/playbook-stop-cluster.yml
index 1c1ab52..477b431 100644
--- a/molecule/default/playbook-stop-cluster.yml
+++ b/molecule/default/playbook-stop-cluster.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook-version.yml b/molecule/default/playbook-version.yml
index 5f8cac9..740d9c9 100644
--- a/molecule/default/playbook-version.yml
+++ b/molecule/default/playbook-version.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml
index 92c2184..394481e 100644
--- a/molecule/default/playbook.yml
+++ b/molecule/default/playbook.yml
@@ -1,6 +1,6 @@
 ---
 - name: Converge
-  hosts: node*
+  hosts: all
   become: true
   vars:
     molecule_is_test: true
diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml
index 567499e..5aed1ec 100644
--- a/molecule/default/prepare.yml
+++ b/molecule/default/prepare.yml
@@ -1,6 +1,6 @@
 ---
 - name: Prepare
-  hosts: node*
+  hosts: all
   tasks:
     - name: Debug Message
       debug:
diff --git a/molecule/highavailability/Dockerfile.j2 b/molecule/highavailability/Dockerfile.j2
new file mode 100644
index 0000000..e6aa95d
--- /dev/null
+++ b/molecule/highavailability/Dockerfile.j2
@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi
diff --git a/molecule/highavailability/INSTALL.rst b/molecule/highavailability/INSTALL.rst
new file mode 100644
index 0000000..6a44bde
--- /dev/null
+++ b/molecule/highavailability/INSTALL.rst
@@ -0,0 +1,22 @@
+*******
+Docker driver installation guide
+*******
+
+Requirements
+============
+
+* Docker Engine
+
+Install
+=======
+
+Please refer to the `Virtual environment`_ documentation for installation best
+practices. If not using a virtual environment, please consider passing the
+widely recommended `'--user' flag`_ when invoking ``pip``.
+
+.. _Virtual environment: https://virtualenv.pypa.io/en/latest/
+.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site
+
+.. code-block:: bash
+
+    $ pip install 'molecule[docker]'
diff --git a/molecule/highavailability/molecule.yml b/molecule/highavailability/molecule.yml
new file mode 100644
index 0000000..83e4c58
--- /dev/null
+++ b/molecule/highavailability/molecule.yml
@@ -0,0 +1,67 @@
+---
+
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: node1
+    image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    networks:
+      - name: k3snet
+  - name: node2
+    image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    networks:
+      - name: k3snet
+  - name: node3
+    image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
+    command: ${MOLECULE_DOCKER_COMMAND:-""}
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    privileged: true
+    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    networks:
+      - name: k3snet
+  - name: database
+    image: postgres:11-alpine
+    pre_build_image: true
+    command: "postgres"
+    env:
+      POSTGRES_PASSWORD: "verybadpass"
+    networks:
+      - name: k3snet
+  - name: loadbalancer
+    image: nginx:1.16-alpine
+    pre_build_image: true
+    ports:
+      - "6443:6443"
+    volumes:
+      - ${MOLECULE_SCENARIO_DIRECTORY}/nginx-loadbalancer.conf:/etc/nginx/nginx.conf:ro
+    command: "nginx -g 'daemon off;'"
+    networks:
+      - name: k3snet
+provisioner:
+  name: ansible
+  options:
+    verbose: true
+  lint:
+    name: ansible-lint
+  playbooks:
+    prepare: ${MOLECULE_PREPARE_PLAYBOOK:-prepare.yml}
+    converge: ${MOLECULE_PLAYBOOK:-playbook.yml}
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
diff --git a/molecule/highavailability/nginx-loadbalancer.conf b/molecule/highavailability/nginx-loadbalancer.conf
new file mode 100644
index 0000000..5c2464e
--- /dev/null
+++ b/molecule/highavailability/nginx-loadbalancer.conf
@@ -0,0 +1,20 @@
+events { }
+
+http {
+    # Set up our upstream of control (master) nodes. The default load balancing
+    # algorithm for nginx is to round-robin. Perfect!
+    upstream control_plane {
+        server node2:6443;
+        server node3:6443;
+    }
+
+    # Listen on port 6443, this is our default control plane port, then pass
+    # all traffic to one of the control (master) nodes.
+    server {
+        listen 6443;
+
+        location / {
+            proxy_pass http://control_plane;
+        }
+    }
+}
diff --git a/molecule/default/playbook-ha-postgres.yml b/molecule/highavailability/playbook.yml
similarity index 89%
rename from molecule/default/playbook-ha-postgres.yml
rename to molecule/highavailability/playbook.yml
index c1a613b..fcc3562 100644
--- a/molecule/default/playbook-ha-postgres.yml
+++ b/molecule/highavailability/playbook.yml
@@ -4,6 +4,7 @@
   become: true
   vars:
     molecule_is_test: true
+    k3s_control_node_address: loadbalancer
     k3s_datastore_endpoint: "postgres://postgres:verybadpass@database:5432/postgres?sslmode=disable"
   pre_tasks:
     - name: Set each node to be a control node
diff --git a/molecule/highavailability/prepare.yml b/molecule/highavailability/prepare.yml
new file mode 100644
index 0000000..567499e
--- /dev/null
+++ b/molecule/highavailability/prepare.yml
@@ -0,0 +1,7 @@
+---
+- name: Prepare
+  hosts: node*
+  tasks:
+    - name: Debug Message
+      debug:
+        msg: No prepare steps required
diff --git a/molecule/highavailability/tests/test_default.py b/molecule/highavailability/tests/test_default.py
new file mode 100644
index 0000000..eedd64a
--- /dev/null
+++ b/molecule/highavailability/tests/test_default.py
@@ -0,0 +1,14 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_hosts_file(host):
+    f = host.file('/etc/hosts')
+
+    assert f.exists
+    assert f.user == 'root'
+    assert f.group == 'root'
diff --git a/molecule/highavailability/tests/test_default.pyc b/molecule/highavailability/tests/test_default.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..28c2e2d42688dc5ec97e4aa0c0223353ad18569d
GIT binary patch
literal 1025
zcmdr~&2H2%5FRJnUs3jiP%k~_(#@gbh7eViszu9I+5!?Qp(st9)>*F|*^?DjyBFZ$
z;3aqo9-tjNoBqB5`5BMDjK7)rCgH!W&A-3HZ#gWlkk<F~+7Dz3dKgH6$(_$&5&#dN
z3Zd#jwE`TVT7@Ej?Vk*)H7G(LXR;1O4}Q_f4M%Yrtq`5Kp=*S%$^CdTOb$K|lk|A>
zB^iA>8GlU=kB14m^qG`7qcvE*2ejHDnQKbrE20Do2G^92f!zTN;77X+T_Mu|?A`;g
z*MUb%klH$pT&9uTBpY+g;|p!kVvn{)YxL#%T`45TRf73<;UZp=yKgNwD6~q9(Z+{0
z4sku68{O2AE8XLc7F{!CL<YREf5?k*R>!rdcZIS@9q&%%g_ysIXIk>Sk^J7twVn#X
z5V?tM&Q)fFj_2HnQV@dNqEMyD;;XBNp@WVoErj7YYIC_*JdNq-QkBj4^Uqz9M-(n$
zTJSP!Bo69J>{B-$^Ae^`unmIjIvHU7pwC{h$mvXD0w6thoZnSFXi!Ky$dnallBcFo
zij;H&cOhO^ALiA661qj9`|as*dw7iyuI9pM<%!NYrtNsUQoUX+Hg=WW&*py|kQ>C?
UnJ(zBdgp#cKbaQ+-E2F&1#loHZU6uP

literal 0
HcmV?d00001


From 3a1c7e7b35ac175eb7bf0adb0e4d516c4ea9e096 Mon Sep 17 00:00:00 2001
From: Xan Manning <xan.manning@gmail.com>
Date: Sun, 12 Jan 2020 16:16:40 +0000
Subject: [PATCH 5/5] Added workflow for Database backed and DQLite HA

---
 .travis.yml                                   |  7 ++++-
 molecule/highavailability/molecule.yml        |  2 +-
 .../highavailability/nginx-loadbalancer.conf  | 28 ++++++++++++++-----
 molecule/highavailability/playbook-dqlite.yml | 15 ++++++++++
 tasks/build/configure-k3s-cluster.yml         |  6 ++--
 templates/k3s.service.j2                      | 17 +++++++++++
 6 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100644 molecule/highavailability/playbook-dqlite.yml

diff --git a/.travis.yml b/.travis.yml
index 2687786..4b1b599 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -37,7 +37,12 @@ env:
 
     # Test multiple masters in control plane with PostgreSQL
     - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
-      MOLECULE_SCENATIO: highavailability
+      MOLECULE_SCENARIO: highavailability
+
+    # Test multiple masters in control plane with DQLite
+    - MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
+      MOLECULE_SCENARIO: highavailability
+      MOLECULE_PLAYBOOK: playbook-dqlite.yml
 
 install:
   # Install test dependencies.
diff --git a/molecule/highavailability/molecule.yml b/molecule/highavailability/molecule.yml
index 83e4c58..03b5567 100644
--- a/molecule/highavailability/molecule.yml
+++ b/molecule/highavailability/molecule.yml
@@ -43,7 +43,7 @@ platforms:
     networks:
       - name: k3snet
   - name: loadbalancer
-    image: nginx:1.16-alpine
+    image: nginx:1.17-alpine
     pre_build_image: true
     ports:
       - "6443:6443"
diff --git a/molecule/highavailability/nginx-loadbalancer.conf b/molecule/highavailability/nginx-loadbalancer.conf
index 5c2464e..69b37b2 100644
--- a/molecule/highavailability/nginx-loadbalancer.conf
+++ b/molecule/highavailability/nginx-loadbalancer.conf
@@ -1,20 +1,34 @@
-events { }
+worker_processes 4;
+worker_rlimit_nofile 40000;
 
-http {
+events {
+    worker_connections 8192;
+}
+
+stream {
     # Set up our upstream of control (master) nodes. The default load balancing
     # algorithm for nginx is to round-robin. Perfect!
     upstream control_plane {
-        server node2:6443;
-        server node3:6443;
+        server node2:6443 max_fails=3 fail_timeout=5s;
+        server node3:6443 max_fails=3 fail_timeout=5s;
+    }
+
+    upstream control_plane_443 {
+        server node2:443 max_fails=3 fail_timeout=5s;
+        server node3:443 max_fails=3 fail_timeout=5s;
     }
 
     # Listen on port 6443, this is our default control plane port, then pass
     # all traffic to one of the control (master) nodes.
     server {
         listen 6443;
+        proxy_pass control_plane;
+    }
 
-        location / {
-            proxy_pass http://control_plane;
-        }
+    # Listen on port 443, this is our default ssl port, then pass
+    # all traffic to one of the control (master) nodes.
+    server {
+        listen 443;
+        proxy_pass control_plane_443;
     }
 }
diff --git a/molecule/highavailability/playbook-dqlite.yml b/molecule/highavailability/playbook-dqlite.yml
new file mode 100644
index 0000000..4d41510
--- /dev/null
+++ b/molecule/highavailability/playbook-dqlite.yml
@@ -0,0 +1,15 @@
+---
+- name: Converge
+  hosts: node*
+  become: true
+  vars:
+    molecule_is_test: true
+    k3s_dqlite_datastore: true
+    k3s_use_experimental: true
+  pre_tasks:
+    - name: Set each node to be a control node
+      set_fact:
+        k3s_control_node: true
+      when: inventory_hostname in ['node2', 'node3']
+  roles:
+    - role: xanmanning.k3s
diff --git a/tasks/build/configure-k3s-cluster.yml b/tasks/build/configure-k3s-cluster.yml
index 0ec7572..1db74ec 100644
--- a/tasks/build/configure-k3s-cluster.yml
+++ b/tasks/build/configure-k3s-cluster.yml
@@ -8,12 +8,14 @@
       @@@
       {{ hostvars[item].ansible_host | default(hostvars[item].ansible_fqdn) }}
       @@@
-      {{ hostvars[item].k3s_control_node }}
+      C_{{ hostvars[item].k3s_control_node }}
+      @@@
+      P_{{ hostvars[item].k3s_primary_control_node | default(False) }}
     create: true
   loop: "{{ play_hosts }}"
 
 - name: Lookup control node from file
-  command: "grep 'True' /tmp/inventory.txt"
+  command: "grep '{{ 'P_True' if (k3s_controller_count | length > 1) else 'C_True' }}' /tmp/inventory.txt"
   changed_when: false
   register: k3s_control_delegate_raw
 
diff --git a/templates/k3s.service.j2 b/templates/k3s.service.j2
index 8b1dfb5..89ebd1a 100644
--- a/templates/k3s.service.j2
+++ b/templates/k3s.service.j2
@@ -56,6 +56,23 @@ ExecStart={{ k3s_install_dir }}/k3s
     {% endif %}
     {% if k3s_datastore_endpoint is defined and k3s_datastore_endpoint %}
         --datastore-endpoint "{{ k3s_datastore_endpoint }}"
+        {% if k3s_datastore_cafile is defined and k3s_datastore_cafile %}
+            --datastore-cafile {{ k3s_datastore_cafile }}
+        {% endif %}
+        {% if k3s_datastore_certfile is defined and k3s_datastore_certfile %}
+            --datastore-certfile {{ k3s_datastore_certfile }}
+        {% endif %}
+        {% if k3s_datastore_keyfile is defined and k3s_datastore_keyfile %}
+            --datastore-keyfile {{ k3s_datastore_keyfile }}
+        {% endif %}
+    {% endif %}
+    {% if k3s_dqlite_datastore is defined and k3s_dqlite_datastore %}
+        {% if k3s_primary_control_node is defined and k3s_primary_control_node %}
+            --cluster-init
+        {% else %}
+            --server https://{{ k3s_control_node_address }}:{{ k3s_https_port }}
+            --token {{ k3s_control_token }}
+        {% endif %}
     {% endif %}
 {% else %}
     agent