From 5e39160ed9371efec5a25320e22b7d3e13abe14a Mon Sep 17 00:00:00 2001 From: Xan Manning Date: Sat, 26 Oct 2019 22:49:48 +0100 Subject: [PATCH] Added a number of extra options to configure K3s in systemd unit file. Testing: - Added docker networking, ensure that test output is verbose. - Fix build for AmazonLinux 2 - No-deploy flag test added --- .travis.yml | 12 +++- README.md | 58 +++++++++++++-- defaults/main.yml | 39 +++++++++- handlers/main.yml | 3 - molecule/default/molecule.yml | 8 +++ .../playbook-docker-altport-wireguard.yml | 13 ++++ molecule/default/playbook-docker.yml | 2 +- molecule/default/playbook-no-deploy.yml | 21 ++++++ tasks/configure-k3s-cluster.yml | 2 +- tasks/install-docker-amazon.yml | 10 +++ tasks/install-docker-prerequisites-redhat.yml | 1 + tasks/install-k3s.yml | 3 +- tasks/main.yml | 22 ++++-- templates/k3s.service.j2 | 71 ++++++++++++++++++- 14 files changed, 239 insertions(+), 26 deletions(-) create mode 100644 molecule/default/playbook-docker-altport-wireguard.yml create mode 100644 molecule/default/playbook-no-deploy.yml create mode 100644 tasks/install-docker-amazon.yml diff --git a/.travis.yml b/.travis.yml index 8eb0096..f4bcedf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,16 +11,24 @@ env: matrix: - MOLECULE_DISTRO: centos8 - MOLECULE_DISTRO: centos7 - - MOLECULE_DISTRO: ubuntu1804 - MOLECULE_DISTRO: debian10 - MOLECULE_DISTRO: fedora29 - MOLECULE_DISTRO: fedora30 + - MOLECULE_DISTRO: fedora31 - MOLECULE_DISTRO: amazonlinux2 - # Test other role features. + # Test installing docker - MOLECULE_DISTRO: centos7 MOLECULE_PLAYBOOK: playbook-docker.yml + # Test using alternate port and using wireguard as the flannel backend + - MOLECULE_DISTRO: amazonlinux2 + MOLECULE_PLAYBOOK: playbook-docker-altport-wireguard.yml + + # Test disabling all deployments + - MOLECULE_DISTRO: fedora31 + MOLECULE_PLAYBOOK: playbook-no-deploy.yml + install: # Install test dependencies. - pip install molecule docker jmespath diff --git a/README.md b/README.md index ff637ae..0169643 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,16 @@ Kubernetes") as either a standalone server or cluster. This role has been tested on Ansible 2.6.0+ against the following Linux Distributions: + - Amazon Linux 2 + - CentOS 8 - CentOS 7 - Debian 9 - Debian 10 + - Fedora 29 + - Fedora 30 + - Fedora 31 - openSUSE Leap 15 - Ubuntu 18.04 LTS - - Amazon Linux 2 ## Disclaimer @@ -37,8 +41,18 @@ consistency. | `k3s_github_url` | Set the GitHub URL to install k3s from. | https://github.com/rancher/k3s | | `k3s_install_dir` | Installation directory for k3s. | `/usr/local/bin` | | `k3s_control_workers` | Are control hosts also workers? | `true` | -| `k3s_ensure_docker_installed ` | Use Docker rather than Containerd? | `false` | +| `k3s_https_port` | HTTPS port listening port. | 6443 | +| `k3s_use_docker` | Use Docker rather than Containerd? | `false` | | `k3s_no_flannel` | Do not use Flannel | `false` | +| `k3s_flannel_backend` | Flannel backend ('none', 'vxlan', 'ipsec', or 'wireguard') | vxlan | +| `k3s_no_coredns` | Do not use CoreDNS | `false` | +| `k3s_cluster_dns` | Cluster IP for CoreDNS service. Should be in your service-cidr range. | _NULL_ | +| `k3s_cluster_domain` | Cluster Domain. | cluster.local | +| `k3s_no_traefik` | Do not use Traefik | `false` | +| `k3s_no_servicelb` | Do not use ServiceLB, necessary for using something like MetalLB. | `false` | +| `k3s_disable_scheduler` | Disable Kubernetes default scheduler | `false` | +| `k3s_disable_cloud_controller` | Disable k3s default cloud controller manager. | `false` | +| `k3s_disable_network_policy` | Disable k3s default network policy controller. | `false` | #### Important note about `k3s_release_version` @@ -54,10 +68,16 @@ k3s_release_version: v0.2.0 Below are variables that are set against specific hosts in your inventory. -| Variable | Description | Default Value | -|-------------------------|--------------------------------------------------------|---------------| -| `k3s_control_node` | Define the host as a control plane node, (True/False). | `false` | -| `k3s_flannel_interface` | Define the flannel proxy interface for this node. | | +| Variable | Description | Default Value | +|-----------------------------|--------------------------------------------------------|---------------| +| `k3s_control_node` | Define the host as a control plane node, (True/False). | `false` | +| `k3s_node_name` | Define the name of this node. | `$(hostname)` | +| `k3s_flannel_interface` | Define the flannel proxy interface for this node. | _NULL_ | +| `k3s_bind_address` | Define the bind address for this node. | localhost | +| `k3s_node_ip_address` | IP Address to advertise for this node. | _NULL_ | +| `k3s_node_external_address` | External IP Address to advertise for this node. | _NULL_ | +| `k3s_node_labels` | List of node labels. | _NULL_ | +| `k3s_node_taints` | List of node taints. | _NULL_ | #### Important note about `k3s_control_node` @@ -73,6 +93,30 @@ If you are running k3s on systems with multiple network interfaces, it is necessary to have the flannel interface on a network interface that is routable to the master node(s). +#### Notes about `k3s_node_labels` and `k3s_node_taints` + +Both these variables are lists that will be iterated on. The below example will +output the following: + +**YAML**: + +```yaml +k3s_node_labels: + - foo: bar + - hello: world + +k3s_node_taints: + - key1: value1:NoExecute +``` + +**ARGS**: + +```text +--node-label foo=bar \ +--node-label hello=world \ +--node-taint key1=value1:NoExecute +``` + ## Dependencies No dependencies on other roles. @@ -84,7 +128,7 @@ Example playbook: ```yaml - hosts: k3s_nodes roles: - - { role: xanmanning.k3s, k3s_release_version: v0.2.0 } + - { role: xanmanning.k3s, k3s_release_version: v0.10.2 } ``` ## License diff --git a/defaults/main.yml b/defaults/main.yml index c04c6f6..48eb706 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,8 +13,41 @@ k3s_install_dir: /usr/local/bin # Are control hosts also worker nodes? k3s_control_workers: true -# Ensure Docker is installed on nodes -k3s_ensure_docker_installed: false +# HTTPS Listening port +k3s_https_port: 6443 -# Disable flannel +# Ensure Docker is installed on nodes +k3s_use_docker: false + +# Disable flannel, you will need to install your own CNI driver. k3s_no_flannel: false + +# Flannel backend ('none', 'vxlan', 'ipsec', or 'wireguard') +k3s_flannel_backend: vxlan + +# Disable CoreDNS, you will need to install your own DNS provider. +k3s_no_coredns: false + +# Cluster IP for CoreDNS service. Should be in your service-cidr range. +# Use `false` to use default +k3s_cluster_dns: false + +# Cluster Domain (default: "cluster.local") +k3s_cluster_domain: cluster.local + +# Disable Traefik +k3s_no_traefik: false + +# Disable Service Load Balancer, you will need to install your own +# load balancer, such as MetalLB. Must be disabled if using your own +# load balancer service. +k3s_no_servicelb: false + +# Disable default k3s scheduler +k3s_disable_scheduler: false + +# Disable k3s cloud controller +k3s_disable_cloud_controller: false + +# Disable k3s network policy controller +k3s_disable_network_policy: false diff --git a/handlers/main.yml b/handlers/main.yml index 69adce3..2c66746 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -3,18 +3,15 @@ - name: reload systemd systemd: daemon_reload: true - # when: molecule_is_test is not defined - name: restart k3s service: name: k3s state: restarted enabled: true - # when: molecule_is_test is not defined - name: restart docker service: name: docker state: restarted enabled: true - # when: molecule_is_test is not defined diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 7df57f1..8874dac 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -14,6 +14,8 @@ platforms: - /sys/fs/cgroup:/sys/fs/cgroup:ro privileged: true pre_build_image: true + networks: + - name: k3snet - name: node2 image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} @@ -21,6 +23,8 @@ platforms: - /sys/fs/cgroup:/sys/fs/cgroup:ro privileged: true pre_build_image: true + networks: + - name: k3snet - name: node3 image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} @@ -28,8 +32,12 @@ platforms: - /sys/fs/cgroup:/sys/fs/cgroup:ro privileged: true pre_build_image: true + networks: + - name: k3snet provisioner: name: ansible + options: + verbose: true lint: name: ansible-lint playbooks: diff --git a/molecule/default/playbook-docker-altport-wireguard.yml b/molecule/default/playbook-docker-altport-wireguard.yml new file mode 100644 index 0000000..1db8114 --- /dev/null +++ b/molecule/default/playbook-docker-altport-wireguard.yml @@ -0,0 +1,13 @@ +--- +- name: Converge + hosts: all + become: true + vars: + molecule_is_test: true + k3s_use_docker: true + k3s_https_port: 26443 + k3s_flannel_backend: wireguard + k3s_cluster_domain: examplecluster.local + k3s_control_workers: false + roles: + - role: xanmanning.k3s diff --git a/molecule/default/playbook-docker.yml b/molecule/default/playbook-docker.yml index 412eddc..a780b33 100644 --- a/molecule/default/playbook-docker.yml +++ b/molecule/default/playbook-docker.yml @@ -4,6 +4,6 @@ become: true vars: molecule_is_test: true - k3s_ensure_docker_installed: true + k3s_use_docker: true roles: - role: xanmanning.k3s diff --git a/molecule/default/playbook-no-deploy.yml b/molecule/default/playbook-no-deploy.yml new file mode 100644 index 0000000..6a2115b --- /dev/null +++ b/molecule/default/playbook-no-deploy.yml @@ -0,0 +1,21 @@ +--- +- name: Converge + hosts: all + become: true + vars: + molecule_is_test: true + k3s_no_flannel: true + k3s_no_coredns: true + k3s_no_traefik: true + k3s_no_servicelb: true + k3s_disable_scheduler: true + k3s_disable_cloud_controller: true + k3s_disable_network_policy: true + k3s_control_workers: false + k3s_node_labels: + - foo: bar + - hello: world + k3s_node_taints: + - key1: value1:NoExecute + roles: + - role: xanmanning.k3s diff --git a/tasks/configure-k3s-cluster.yml b/tasks/configure-k3s-cluster.yml index 4732432..ff42ae1 100644 --- a/tasks/configure-k3s-cluster.yml +++ b/tasks/configure-k3s-cluster.yml @@ -43,7 +43,7 @@ - name: Wait for control plane to be ready to accept connections wait_for: - port: 6443 + port: "{{ k3s_https_port }}" delay: 5 sleep: 5 timeout: 300 diff --git a/tasks/install-docker-amazon.yml b/tasks/install-docker-amazon.yml new file mode 100644 index 0000000..cd24d37 --- /dev/null +++ b/tasks/install-docker-amazon.yml @@ -0,0 +1,10 @@ +--- + +- name: Ensure docker is installed using amazon-linux-extras + command: amazon-linux-extras install docker + args: + creates: /etc/docker + notify: + - restart docker + +- meta: flush_handlers diff --git a/tasks/install-docker-prerequisites-redhat.yml b/tasks/install-docker-prerequisites-redhat.yml index 2bcd032..1089394 100644 --- a/tasks/install-docker-prerequisites-redhat.yml +++ b/tasks/install-docker-prerequisites-redhat.yml @@ -21,3 +21,4 @@ enabled: true gpgcheck: true state: present + when: ansible_distribution | lower not in ['amazon'] diff --git a/tasks/install-k3s.yml b/tasks/install-k3s.yml index 9455e8d..1f8c0b2 100644 --- a/tasks/install-k3s.yml +++ b/tasks/install-k3s.yml @@ -21,10 +21,11 @@ - k3s - kubectl - crictl + - ctr - name: Ensure k3s control plane is started service: name: k3s state: started enabled: true - when: k3s_control_node # and molecule_is_test is not defined + when: k3s_control_node diff --git a/tasks/main.yml b/tasks/main.yml index 2bc15c1..c7dbedf 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,17 +1,27 @@ --- - import_tasks: preconfigure-k3s.yml -- include_tasks: install-docker-prerequisites-{{ ansible_os_family|lower }}.yml - when: k3s_ensure_docker_installed + +- include_tasks: install-docker-prerequisites-{{ ansible_os_family | lower }}.yml + when: k3s_use_docker and ((k3s_control_workers) or (not k3s_control_workers and not k3s_control_node)) -- include_tasks: install-docker.yml - when: k3s_ensure_docker_installed + +- import_tasks: install-docker.yml + when: k3s_use_docker and ((k3s_control_workers) or (not k3s_control_workers and not k3s_control_node)) -- include_tasks: get-version.yml + and ansible_distribution | lower not in ['amazon'] + +- include_tasks: install-docker-{{ ansible_distribution | lower }}.yml + when: ansible_distribution | lower in ['amazon'] + +- import_tasks: get-version.yml when: k3s_release_version is not defined or not k3s_release_version + - import_tasks: download-k3s.yml + - import_tasks: install-k3s.yml -- include_tasks: configure-k3s-cluster.yml + +- import_tasks: configure-k3s-cluster.yml when: play_hosts | length > 1 diff --git a/templates/k3s.service.j2 b/templates/k3s.service.j2 index 137b0fc..191ee5e 100644 --- a/templates/k3s.service.j2 +++ b/templates/k3s.service.j2 @@ -7,11 +7,78 @@ After=network.target Type={{ 'notify' if k3s_control_node else 'exec' }} ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay +{% filter replace('\n', ' ') %} +ExecStart={{ k3s_install_dir }}/k3s {% if k3s_control_node %} -ExecStart={{ k3s_install_dir }}/k3s server{{ ' --disable-agent' if not k3s_control_workers else '' }}{{ ' --flannel-iface ' + k3s_flannel_interface if k3s_flannel_interface is defined and not k3s_no_flannel else '' }}{{ ' --no-flannel' if k3s_no_flannel else '' }}{{ ' --docker' if k3s_ensure_docker_installed else '' }} + server{{ ' --disable-agent' if not k3s_control_workers else '' }} + {% if k3s_https_port != 6443 %} + --https-listen-port {{ k3s_https_port }} + {% endif %} + {% if k3s_disable_scheduler %} + --disable-scheduler + {% endif %} + {% if k3s_disable_cloud_controller %} + --disable-cloud-controller + {% endif %} + {% if k3s_disable_network_policy %} + --disable-network-policy + {% endif %} + {% if k3s_no_flannel %} + --no-flannel + {% endif %} + {% if k3s_flannel_backend != "vxlan" and not k3s_no_flannel %} + --flannel-backend {{ k3s_flannel_backend }} + {% endif %} + {% if k3s_no_coredns is defined or k3s_no_traefik is defined or k3s_no_servicelb is defined %} + {% if k3s_no_coredns or k3s_no_traefik or k3s_no_servicelb %} + {{ ' --no-deploy coredns' if k3s_no_coredns else '' }}{{ ' --no-deploy servicelb' if k3s_no_servicelb else '' }}{{ ' --no-deploy traefik' if k3s_no_traefik else '' }} + {% endif %} + {% endif %} + {% if k3s_cluster_dns is defined and k3s_cluster_dns %} + --cluster-dns {{ k3s_cluster_dns }} + {% endif %} + {% if k3s_cluster_domain is defined and k3s_cluster_domain != "cluster.local" %} + --cluster-domain {{ k3s_cluster_domain }} + {% endif %} {% else %} -ExecStart={{ k3s_install_dir }}/k3s agent{{ ' --docker' if k3s_ensure_docker_installed else '' }}{{ ' --flannel-iface ' + k3s_flannel_interface if k3s_flannel_interface is defined and not k3s_no_flannel else '' }}{{ ' --no-flannel' if k3s_no_flannel else '' }} --server https://{{ k3s_control_node_address }}:6443 --token {{ k3s_control_token.content | b64decode }} + agent + --server https://{{ k3s_control_node_address }}:{{ k3s_https_port }} + --token {{ k3s_control_token.content | b64decode }} {% endif %} +{% if k3s_use_docker %} + --docker +{% endif %} +{% if k3s_flannel_interface is defined and not k3s_no_flannel %} + --flannel-iface {{ k3s_flannel_interface }} +{% endif %} +{% if k3s_bind_address is defined %} + --bind-address {{ k3s_bind_address }} +{% endif %} +{% if k3s_node_name is defined %} + --node-name {{ k3s_node_name }} +{% endif %} +{% if k3s_node_ip_address is defined %} + --node-ip {{ k3s_node_ip_address }} +{% endif %} +{% if k3s_node_external_address is defined %} + --node-external-ip {{ k3s_node_external_address }} +{% endif %} +{% if k3s_node_labels is defined and k3s_node_labels is iterable %} + {% for label in k3s_node_labels %} + {% for key, value in label.items() %} + --node-label {{ key }}={{ value }} + {% endfor %} + {% endfor %} +{% endif %} +{% if k3s_node_taints is defined and k3s_node_taints is iterable %} + {% for taint in k3s_node_taints %} + {% for key, value in taint.items() %} + --node-taint {{ key }}={{ value }} + {% endfor %} + {% endfor %} +{% endif %} +{% endfilter %} + KillMode=process Delegate=yes LimitNOFILE=infinity