diff --git a/README.md b/README.md index d076cc0..e9fc42d 100644 --- a/README.md +++ b/README.md @@ -43,53 +43,58 @@ my spare time so I cannot promise a speedy fix delivery. Below are variables that are set against all of the play hosts for environment consistency. -| Variable | Description | Default Value | -|----------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------| -| `k3s_cluster_state` | State of cluster: installed, started, stopped, restarted, downloaded, uninstalled. | installed | -| `k3s_release_version` | Use a specific version of k3s, eg. `v0.2.0`. Specify `false` for stable. | `false` | -| `k3s_build_cluster` | When multiple `play_hosts` are available, attempt to cluster. Read notes below. | `true` | -| `k3s_github_url` | Set the GitHub URL to install k3s from. | https://github.com/rancher/k3s | -| `k3s_install_dir` | Installation directory for k3s. | `/usr/local/bin` | -| `k3s_server_manifests_dir` | Path for place the `k3s_server_manifests_templates`. | `/var/lib/rancher/k3s/server/manifests` | -| `k3s_server_manifests_templates` | A list of Auto-Deploying Manifests Templates. | [] | -| `k3s_use_experimental` | Allow the use of experimental features in k3s. | `false` | -| `k3s_use_unsupported_config` | Allow the use of unsupported configurations in k3s. | `false` | -| `k3s_non_root` | Install k3s as non-root user. See notes below. | `false` | -| `k3s_control_workers` | Are control hosts also workers? | `true` | -| `k3s_cluster_cidr` | Network CIDR to use for pod IPs | 10.42.0.0/16 | -| `k3s_service_cidr` | Network CIDR to use for service IPs | 10.43.0.0/16 | -| `k3s_control_node_address` | Use a specific control node address. IP or FQDN. | _NULL_ | -| `k3s_control_token` | Use a specific control token, please read notes below. | _NULL_ | -| `k3s_https_port` | HTTPS port listening port. | 6443 | -| `k3s_use_docker` | Use Docker rather than Containerd? | `false` | -| `k3s_no_flannel` | Do not use Flannel | `false` | -| `k3s_flannel_backend` | Flannel backend ('none', 'vxlan', 'ipsec', 'host-gw' or 'wireguard') | vxlan | -| `k3s_no_coredns` | Do not use CoreDNS | `false` | -| `k3s_cluster_dns` | Cluster IP for CoreDNS service. Should be in your service-cidr range. | _NULL_ | -| `k3s_cluster_domain` | Cluster Domain. | cluster.local | -| `k3s_resolv_conf` | Kubelet resolv.conf file | _NULL_ | -| `k3s_no_traefik` | Do not use Traefik | `false` | -| `k3s_no_servicelb` | Do not use ServiceLB, necessary for using something like MetalLB. | `false` | -| `k3s_no_local_storage` | Do not use Local Storage | `false` | -| `k3s_default_local_storage_path` | Set Local Storage Path. Specify `false` for default. | `false` | -| `k3s_no_metrics_server` | Do not deploy metrics server | `false` | -| `k3s_disable_scheduler` | Disable Kubernetes default scheduler | `false` | -| `k3s_disable_cloud_controller` | Disable k3s default cloud controller manager. | `false` | -| `k3s_disable_network_policy` | Disable k3s default network policy controller. | `false` | -| `k3s_write_kubeconfig_mode` | Define the file mode from the generated KubeConfig, eg. `644` | _NULL_ | -| `k3s_datastore_endpoint` | Define the database or etcd cluster endpoint for HA. | _NULL_ | -| `k3s_datastore_cafile` | Define the database TLS CA file. | _NULL_ | -| `k3s_datastore_certfile` | Define the database TLS Cert file. | _NULL_ | -| `k3s_datastore_keyfile` | Define the database TLS Key file. | _NULL_ | -| `k3s_become_for_all` | Enable become for all (where value for `k3s_become_for_*` is _NULL_ | `false` | -| `k3s_become_for_systemd` | Enable become for systemd commands. | _NULL_ | -| `k3s_become_for_install_dir` | Enable become for writing to `k3s_install_dir`. | _NULL_ | -| `k3s_become_for_usr_local_bin` | Enable become for writing to `/usr/local/bin/`. | _NULL_ | -| `k3s_become_for_package_install` | Enable become for installing prerequisite packages. | _NULL_ | -| `k3s_become_for_kubectl` | Enable become for kubectl commands. | _NULL_ | -| `k3s_become_for_uninstall` | Enable become for running uninstall scripts. | _NULL_ | -| `k3s_dqlite_datastore` | Use DQLite as the database backend for HA. (EXPERIMENTAL) | `false` | -| `k3s_secrets_encryption` | Use secrets encryption at rest. (EXPERIMENTAL) | `false` | +| Variable | Description | Default Value | +|------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------| +| `k3s_cluster_state` | State of cluster: installed, started, stopped, restarted, downloaded, uninstalled. | installed | +| `k3s_release_version` | Use a specific version of k3s, eg. `v0.2.0`. Specify `false` for stable. | `false` | +| `k3s_build_cluster` | When multiple `play_hosts` are available, attempt to cluster. Read notes below. | `true` | +| `k3s_github_url` | Set the GitHub URL to install k3s from. | https://github.com/rancher/k3s | +| `k3s_install_dir` | Installation directory for k3s. | `/usr/local/bin` | +| `k3s_server_manifests_dir` | Path for place the `k3s_server_manifests_templates`. | `/var/lib/rancher/k3s/server/manifests` | +| `k3s_server_manifests_templates` | A list of Auto-Deploying Manifests Templates. | [] | +| `k3s_use_experimental` | Allow the use of experimental features in k3s. | `false` | +| `k3s_use_unsupported_config` | Allow the use of unsupported configurations in k3s. | `false` | +| `k3s_non_root` | Install k3s as non-root user. See notes below. | `false` | +| `k3s_control_workers` | Are control hosts also workers? | `true` | +| `k3s_cluster_cidr` | Network CIDR to use for pod IPs | 10.42.0.0/16 | +| `k3s_service_cidr` | Network CIDR to use for service IPs | 10.43.0.0/16 | +| `k3s_control_node_address` | Use a specific control node address. IP or FQDN. | _NULL_ | +| `k3s_control_token` | Use a specific control token, please read notes below. | _NULL_ | +| `k3s_private_registry` | Private registry configuration file (default: "/etc/rancher/k3s/registries.yaml") | _NULL_ | +| `k3s_https_port` | HTTPS port listening port. | 6443 | +| `k3s_use_docker` | Use Docker rather than Containerd? | `false` | +| `k3s_no_flannel` | Do not use Flannel | `false` | +| `k3s_flannel_backend` | Flannel backend ('none', 'vxlan', 'ipsec', 'host-gw' or 'wireguard') | vxlan | +| `k3s_no_coredns` | Do not use CoreDNS | `false` | +| `k3s_cluster_dns` | Cluster IP for CoreDNS service. Should be in your service-cidr range. | _NULL_ | +| `k3s_cluster_domain` | Cluster Domain. | cluster.local | +| `k3s_resolv_conf` | Kubelet resolv.conf file | _NULL_ | +| `k3s_no_traefik` | Do not use Traefik | `false` | +| `k3s_no_servicelb` | Do not use ServiceLB, necessary for using something like MetalLB. | `false` | +| `k3s_no_local_storage` | Do not use Local Storage | `false` | +| `k3s_default_local_storage_path` | Set Local Storage Path. Specify `false` for default. | `false` | +| `k3s_no_metrics_server` | Do not deploy metrics server | `false` | +| `k3s_kube_apiserver_args` | Customized flag for kube-apiserver process | [] | +| `k3s_kube_scheduler_args` | Customized flag for kube-scheduler process | [] | +| `k3s_kube_controller_manager_args` | Customized flag for kube-controller-manager process | [] | +| `k3s_kube_cloud_controller_manager_args` | Customized flag for kube-cloud-controller-manager process | [] | +| `k3s_disable_scheduler` | Disable Kubernetes default scheduler | `false` | +| `k3s_disable_cloud_controller` | Disable k3s default cloud controller manager. | `false` | +| `k3s_disable_network_policy` | Disable k3s default network policy controller. | `false` | +| `k3s_write_kubeconfig_mode` | Define the file mode from the generated KubeConfig, eg. `644` | _NULL_ | +| `k3s_datastore_endpoint` | Define the database or etcd cluster endpoint for HA. | _NULL_ | +| `k3s_datastore_cafile` | Define the database TLS CA file. | _NULL_ | +| `k3s_datastore_certfile` | Define the database TLS Cert file. | _NULL_ | +| `k3s_datastore_keyfile` | Define the database TLS Key file. | _NULL_ | +| `k3s_become_for_all` | Enable become for all (where value for `k3s_become_for_*` is _NULL_ | `false` | +| `k3s_become_for_systemd` | Enable become for systemd commands. | _NULL_ | +| `k3s_become_for_install_dir` | Enable become for writing to `k3s_install_dir`. | _NULL_ | +| `k3s_become_for_usr_local_bin` | Enable become for writing to `/usr/local/bin/`. | _NULL_ | +| `k3s_become_for_package_install` | Enable become for installing prerequisite packages. | _NULL_ | +| `k3s_become_for_kubectl` | Enable become for kubectl commands. | _NULL_ | +| `k3s_become_for_uninstall` | Enable become for running uninstall scripts. | _NULL_ | +| `k3s_dqlite_datastore` | Use DQLite as the database backend for HA. (EXPERIMENTAL) | `false` | +| `k3s_secrets_encryption` | Use secrets encryption at rest. (EXPERIMENTAL) | `false` | #### Important note about `k3s_release_version` @@ -153,20 +158,21 @@ Please note that this may potentially break setting up agents. Below are variables that are set against specific hosts in your inventory. -| Variable | Description | Default Value | -|----------------------------------|--------------------------------------------------------------------------|------------------------| -| `k3s_control_node` | Define the host as a control plane node, (True/False). | `false` | -| `k3s_node_name` | Define the name of this node. | `$(hostname)` | -| `k3s_node_id` | Define the ID of this node. | _NULL_ | -| `k3s_flannel_interface` | Define the flannel proxy interface for this node. | _NULL_ | -| `k3s_bind_address` | Define the bind address for this node. | localhost | -| `k3s_node_ip_address` | IP Address to advertise for this node. | _NULL_ | -| `k3s_node_external_address` | External IP Address to advertise for this node. | _NULL_ | -| `k3s_node_labels` | List of node labels. | _NULL_ | -| `k3s_kubelet_args` | A list of kubelet args to pass to the server. | [] | -| `k3s_node_taints` | List of node taints. | _NULL_ | -| `k3s_node_data_dir` | Folder to hold state. | `/var/lib/rancher/k3s` | -| `k3s_tls_san` | Add additional hosname or IP as Subject Alternate Name in the TLS cert. | _NULL_ | +| Variable | Description | Default Value | +|-----------------------------|--------------------------------------------------------------------------|------------------------| +| `k3s_control_node` | Define the host as a control plane node, (True/False). | `false` | +| `k3s_node_name` | Define the name of this node. | `$(hostname)` | +| `k3s_node_id` | Define the ID of this node. | _NULL_ | +| `k3s_flannel_interface` | Define the flannel proxy interface for this node. | _NULL_ | +| `k3s_bind_address` | Define the bind address for this node. | localhost | +| `k3s_node_ip_address` | IP Address to advertise for this node. | _NULL_ | +| `k3s_node_external_address` | External IP Address to advertise for this node. | _NULL_ | +| `k3s_node_labels` | List of node labels. | _NULL_ | +| `k3s_kubelet_args` | A list of kubelet args to pass to the server. | [] | +| `k3s_kube_proxy_args` | A list of kube proxy args to pass to the server. | [] | +| `k3s_node_taints` | List of node taints. | _NULL_ | +| `k3s_node_data_dir` | Folder to hold state. | `/var/lib/rancher/k3s` | +| `k3s_tls_san` | Add additional hosname or IP as Subject Alternate Name in the TLS cert. | _NULL_ | #### Important note about `k3s_control_node` and High Availability (HA) @@ -198,15 +204,21 @@ If you are running k3s on systems with multiple network interfaces, it is necessary to have the flannel interface on a network interface that is routable to the master node(s). -#### Notes about `k3s_kubelet_args` +#### Notes about `_args`, `_labels` and `_taints` variables -This parameter allows for assigning additional kubelet args to the server during -runtime. For instance, to use the Azure Cloud Controller, assign the following to +Affected variables: + + - `k3s_kube_apiserver_args` + - `k3s_kube_scheduler_args` + - `k3s_kube_controller_manager_args` + - `k3s_kube_cloud_controller_manager_args` + - `k3s_kubelet_args` + - `k3s_kube_proxy_args` + +These parameters allow for assigning additional args to K3s during runtime. +For instance, to use the Azure Cloud Controller, assign the below to the master node's configuration in your host file. -Note, when using an external cloud controller as below, ensure that the native k3s -cloud controller is disabled by setting the `k3s_disable_cloud_controller` to `true`. - **YAML**: ```yaml @@ -215,10 +227,12 @@ k3s_kubelet_args: - provider-id: azure ``` -#### Notes about `k3s_node_labels` and `k3s_node_taints` +_Note, when using an external cloud controller as above, ensure that the native +k3s cloud controller is disabled by setting the_ `k3s_disable_cloud_controller` +_to_ `true`. -Both these variables are lists that will be iterated on. The below example will -output the following: +Ultimately these variables are lists of key-value pairs that will be iterated +on. The below example will output the following: **YAML**: @@ -231,7 +245,7 @@ k3s_node_taints: - key1: value1:NoExecute ``` -**ARGS**: +**K3S ARGS**: ```text --node-label foo=bar \ diff --git a/tasks/validate/check-variables.yml b/tasks/validate/check-variables.yml index b9b26a7..ec689c6 100644 --- a/tasks/validate/check-variables.yml +++ b/tasks/validate/check-variables.yml @@ -126,6 +126,46 @@ fail_msg: "Kubelet args are not supported in {{ k3s_release_version }}" when: k3s_kubelet_args is defined and k3s_kubelet_args | length > 0 +- name: Check k3s_kube_proxy_args against k3s version + assert: + that: + - (k3s_release_version | replace('v', '')) is version_compare('0.4.0', '>=') + success_msg: "Kube proxy args supported in {{ k3s_release_version }}" + fail_msg: "Kube proxy args are not supported in {{ k3s_release_version }}" + when: k3s_kube_proxy_args is defined and k3s_kube_proxy_args | length > 0 + +- name: Check k3s_kube_apiserver_args against k3s version + assert: + that: + - (k3s_release_version | replace('v', '')) is version_compare('0.4.0', '>=') + success_msg: "Kube API Server supported in {{ k3s_release_version }}" + fail_msg: "Kube API Server args are not supported in {{ k3s_release_version }}" + when: k3s_kube_apiserver_args is defined and k3s_kube_apiserver_args | length > 0 + +- name: Check k3s_kube_scheduler_args against k3s version + assert: + that: + - (k3s_release_version | replace('v', '')) is version_compare('0.4.0', '>=') + success_msg: "Kube Scheduler supported in {{ k3s_release_version }}" + fail_msg: "Kube Scheduler args are not supported in {{ k3s_release_version }}" + when: k3s_kube_scheduler_args is defined and k3s_kube_scheduler_args | length > 0 + +- name: Check k3s_kube_controller_manager_args against k3s version + assert: + that: + - (k3s_release_version | replace('v', '')) is version_compare('0.4.0', '>=') + success_msg: "Kube Controller Manager supported in {{ k3s_release_version }}" + fail_msg: "Kube Controller Manager args are not supported in {{ k3s_release_version }}" + when: k3s_kube_controller_manager_args is defined and k3s_kube_controller_manager_args | length > 0 + +- name: Check k3s_kube_cloud_controller_manager_args against k3s version + assert: + that: + - (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=') + success_msg: "Kube Cloud Controller Manager supported in {{ k3s_release_version }}" + fail_msg: "Kube Cloud Controller Manager args are not supported in {{ k3s_release_version }}" + when: k3s_kube_cloud_controller_manager_args is defined and k3s_kube_cloud_controller_manager_args | length > 0 + - name: Check k3s_flannel_backend against k3s version assert: that: @@ -150,6 +190,14 @@ fail_msg: "--disable-network-policy not supported in {{ k3s_release_version }}" when: k3s_disable_network_policy is defined and k3s_disable_network_policy +- name: Check k3s_private_registry against k3s version + assert: + that: + - (k3s_release_version | replace('v', '')) is version_compare('0.10.0', '>=') + success_msg: "--private-registry supported in {{ k3s_release_version }}" + fail_msg: "--private-registry not supported in {{ k3s_release_version }}" + when: k3s_private_registry is defined and k3s_private_registry + - name: Check k3s_disable_cloud_controller against k3s version assert: that: diff --git a/templates/k3s.service.j2 b/templates/k3s.service.j2 index 334a531..04543d9 100644 --- a/templates/k3s.service.j2 +++ b/templates/k3s.service.j2 @@ -43,6 +43,9 @@ ExecStart={{ k3s_install_dir }}/k3s {% if k3s_flannel_backend is defined and not k3s_no_flannel %} --flannel-backend {{ k3s_flannel_backend }} {% endif %} + {% if k3s_private_registry is defined and k3s_private_registry %} + --private-registry {{ k3s_private_registry }} + {% endif %} {% if k3s_no_coredns is defined or k3s_no_traefik is defined or k3s_no_servicelb is defined %} {% if k3s_no_coredns or k3s_no_traefik or k3s_no_servicelb %} {{ ' --no-deploy coredns' if k3s_no_coredns else '' }}{{ ' --no-deploy servicelb' if k3s_no_servicelb else '' }}{{ ' --no-deploy traefik' if k3s_no_traefik else '' }}{{ ' --no-deploy local-storage' if k3s_no_local_storage else '' }}{{ ' --no-deploy metrics-server' if k3s_no_metrics_server else '' }} @@ -80,6 +83,34 @@ ExecStart={{ k3s_install_dir }}/k3s {% if k3s_secrets_encryption is defined and k3s_secrets_encryption %} --secrets-encryption {% endif %} + {% if k3s_kube_apiserver_args is defined and k3s_kube_apiserver_args is iterable %} + {% for arg in k3s_kube_apiserver_args %} + {% for key, value in arg.items() %} + --kube-apiserver-arg {{ key }}={{ value }} + {% endfor %} + {% endfor %} + {% endif %} + {% if k3s_kube_scheduler_args is defined and k3s_kube_scheduler_args is iterable %} + {% for arg in k3s_kube_scheduler_args %} + {% for key, value in arg.items() %} + --kube-scheduler-arg {{ key }}={{ value }} + {% endfor %} + {% endfor %} + {% endif %} + {% if k3s_kube_controller_manager_args is defined and k3s_kube_controller_manager_args is iterable %} + {% for arg in k3s_kube_controller_manager_args %} + {% for key, value in arg.items() %} + --kube-controller-manager-arg {{ key }}={{ value }} + {% endfor %} + {% endfor %} + {% endif %} + {% if k3s_kube_cloud_controller_manager_args is defined and k3s_kube_cloud_controller_manager_args is iterable %} + {% for arg in k3s_kube_cloud_controller_manager_args %} + {% for key, value in arg.items() %} + --kube-cloud-controller-manager-arg {{ key }}={{ value }} + {% endfor %} + {% endfor %} + {% endif %} {% else %} agent --server https://{{ k3s_control_node_address }}:{{ k3s_https_port }} @@ -139,6 +170,13 @@ ExecStart={{ k3s_install_dir }}/k3s {% endfor %} {% endfor %} {% endif %} +{% if k3s_kube_proxy_args is defined and k3s_kube_proxy_args is iterable %} + {% for arg in k3s_kube_proxy_args %} + {% for key, value in arg.items() %} + --kube-proxy-arg {{ key }}={{ value }} + {% endfor %} + {% endfor %} +{% endif %} {% endfilter %} {% endfilter %}