cluster up and running

.gitignore

@@ -3,4 +3,6 @@ venv
 .vscode
 .vagrant
 inventory.yml
-playbook/debug.yml
+playbook/debug.yml
+kube-config.yaml
+*.tfstate.*

enable-ethernet.yml

@@ -0,0 +1,18 @@
+- hosts: raspberry_pis
+  become: true
+  tasks:
+    - name: Configure eth0 using Netplan
+      copy:
+        dest: "/etc/netplan/01-netcfg.yaml"
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eth0:
+                dhcp4: true
+        owner: "root"
+        group: "root"
+        mode: "0644"
+    - name: Apply netplan configuration
+      ansible.builtin.command:
+        cmd: netplan apply

kube-vip/manifest.yaml

@@ -0,0 +1,122 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  name: kube-vip-ds
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: kube-vip-ds
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        name: kube-vip-ds
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+      containers:
+      - args:
+        - manager
+        env:
+        - name: vip_arp
+          value: "true"
+        - name: port
+          value: "6443"
+        - name: vip_interface
+          value: eth0
+        - name: vip_cidr
+          value: "32"
+        - name: cp_enable
+          value: "true"
+        - name: cp_namespace
+          value: kube-system
+        - name: vip_ddns
+          value: "false"
+        - name: svc_enable
+          value: "true"
+        - name: vip_leaderelection
+          value: "true"
+        - name: vip_leaseduration
+          value: "5"
+        - name: vip_renewdeadline
+          value: "3"
+        - name: vip_retryperiod
+          value: "1"
+        - name: address
+          value: 192.168.0.40
+        image: ghcr.io/kube-vip/kube-vip:v0.4.0
+        imagePullPolicy: Always
+        name: kube-vip
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_TIME
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+  updateStrategy: {}
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["services", "endpoints"]
+    verbs: ["list","get","watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list","get","watch", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["list", "get", "watch", "update", "create"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["list","get","watch", "update"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system

pi-inventory.yml

@@ -0,0 +1,6 @@
+[raspberry_pis]
+pi1 ansible_host=192.168.0.121
+pi2 ansible_host=192.168.0.146
+pi3 ansible_host=192.168.0.199
+pi4 ansible_host=192.168.0.229
+pi5 ansible_host=192.168.0.234

playbook/site.yml

@@ -5,8 +5,8 @@
   become: true
   roles:
     - role: prereq
-    - role: airgap
     - role: raspberrypi
+    - role: longhorn
 
 - name: Setup K3S server
   hosts: server
@@ -19,3 +19,4 @@
   become: true
   roles:
     - role: k3s_agent
+

roles/longhorn/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+
+- name: Install open-iscsi
+  ansible.builtin.package:
+    name: open-iscsi
+    state: present
+
+- name: Ensure iscsid service is started and enabled
+  ansible.builtin.service:
+    name: iscsid
+    state: started
+    enabled: yes

terraform/external-dns.tf

@@ -0,0 +1,54 @@
+
+data "google_dns_managed_zone" "env_dns_zone" {
+  name = "personal"
+}
+
+resource "google_service_account" "external_dns" {
+  account_id = "homelab-dns"
+}
+
+resource "google_dns_managed_zone_iam_member" "member" {
+  #   project = google_dns_managed_zone.default.project
+  managed_zone = data.google_dns_managed_zone.env_dns_zone.name
+  role         = "roles/dns.admin"
+  member       = "serviceAccount:${google_service_account.external_dns.email}"
+}
+
+resource "google_project_iam_member" "external-dns" {
+  project = "robertb724-personal"
+  role    = "roles/dns.admin"
+  member  = "serviceAccount:${google_service_account.external_dns.email}"
+}
+
+resource "google_service_account_key" "external_dns_creds" {
+  service_account_id = google_service_account.external_dns.name
+}
+
+resource "kubernetes_namespace" "external_dns" {
+  metadata {
+    name = "external-dns"
+  }
+}
+
+resource "kubernetes_secret" "external_dns_creds" {
+  metadata {
+    name      = "gcp-sa-key"
+    namespace = kubernetes_namespace.external_dns.metadata[0].name
+  }
+  data = {
+    "credentials.json" = base64decode(google_service_account_key.external_dns_creds.private_key)
+  }
+}
+
+resource "helm_release" "external_dns" {
+  name       = "external-dns"
+  repository = "https://kubernetes-sigs.github.io/external-dns/"
+  namespace  = kubernetes_namespace.external_dns.metadata[0].name
+  chart      = "external-dns"
+  values     = [file("${path.module}/values/external-dns.values.yaml")]
+  version    = "1.14.3"
+
+
+  depends_on = [kubernetes_secret.external_dns_creds]
+
+}

terraform/longhorn.tf

@@ -0,0 +1,10 @@
+resource "helm_release" "longhorn" {
+  name             = "longhorn"
+  repository       = "https://charts.longhorn.io"
+  namespace        = "longhorn-system"
+  chart            = "longhorn"
+  create_namespace = true
+  values           = [file("${path.module}/values/longhorn.values.yaml")]
+
+  #   version          = "0.14.3"
+}

terraform/manifests/metallb/ipaddresspool.yaml

@@ -0,0 +1,8 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: default
+  namespace: metallb-system
+spec:
+  addresses:
+  - 192.168.0.41/32

terraform/manifests/metallb/l2advertisement.yaml

@@ -0,0 +1,5 @@
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: default
+  namespace: metallb-system

terraform/monitoring.tf

@@ -0,0 +1,11 @@
+resource "helm_release" "kube-prometheus-stack" {
+  name             = "monitoring"
+  repository       = "https://prometheus-community.github.io/helm-charts"
+  namespace        = "monitoring"
+  chart            = "kube-prometheus-stack"
+  create_namespace = true
+#   version          = "4.9.1"
+  values = [
+    file("${path.module}/values/grafana.values.yaml")
+  ]
+}

terraform/networking.tf

@@ -0,0 +1,26 @@
+resource "helm_release" "metallb" {
+  name             = "metallb"
+  repository       = "https://metallb.github.io/metallb"
+  namespace        = "metallb-system"
+  chart            = "metallb"
+  create_namespace = true
+  version          = "0.14.3"
+}
+
+resource "kubernetes_manifest" "metallb_address_pool" {
+  manifest = yamldecode(file("./manifests/metallb/ipaddresspool.yaml"))
+}
+
+resource "kubernetes_manifest" "metallb-l2_advertisement" {
+  manifest = yamldecode(file("./manifests/metallb/l2advertisement.yaml"))
+}
+
+
+resource "helm_release" "nginx" {
+  name             = "nginx"
+  repository       = "https://kubernetes.github.io/ingress-nginx"
+  namespace        = "ingress"
+  chart            = "ingress-nginx"
+  create_namespace = true
+  version          = "4.9.1"
+}

terraform/provider.tf

@@ -0,0 +1,16 @@
+provider "helm" {
+  kubernetes {
+    config_path    = "~/.kube/config.new"
+    config_context = "k3s-ansible"
+  }
+}
+
+provider "kubernetes" {
+  config_path    = "~/.kube/config.new"
+  config_context = "k3s-ansible"
+}
+
+provider "google" {
+  project = "robertb724-personal"
+
+}

terraform/values/external-dns.values.yaml

@@ -0,0 +1,22 @@
+provider:
+  name: google
+
+domainFilters: [robertbatson.com]
+
+txtOwnerId: homelab-k3s
+
+extraArgs:
+- --google-project=robertb724-personal
+
+env:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /etc/secrets/service-account/credentials.json
+
+extraVolumeMounts:
+  - name: google-service-account
+    mountPath: /etc/secrets/service-account/
+
+extraVolumes:
+  - name: google-service-account
+    secret:
+      secretName: gcp-sa-key

terraform/values/grafana.values.yaml

@@ -0,0 +1,6 @@
+grafana:
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    hosts:
+      - grafana.robertbatson.com

terraform/values/longhorn.values.yaml

@@ -0,0 +1,85 @@
+image:
+  longhorn:
+    engine:
+      # -- Repository for the Longhorn Engine image.
+      repository: longhornio/longhorn-engine
+      # -- Specify Longhorn engine image tag
+      tag: v1.6.0-arm64
+    manager:
+      # -- Repository for the Longhorn Manager image.
+      repository: longhornio/longhorn-manager
+      # -- Specify Longhorn manager image tag
+      tag: v1.6.0-arm64
+    ui:
+      # -- Repository for the Longhorn UI image.
+      repository: longhornio/longhorn-ui
+      # -- Specify Longhorn ui image tag
+      tag: v1.6.0-arm64
+    instanceManager:
+      # -- Repository for the Longhorn Instance Manager image.
+      repository: longhornio/longhorn-instance-manager
+      # -- Specify Longhorn instance manager image tag
+      tag: v1.6.0-arm64
+    shareManager:
+      # -- Repository for the Longhorn Share Manager image.
+      repository: longhornio/longhorn-share-manager
+      # -- Specify Longhorn share manager image tag
+      tag: v1.6.0-arm64
+    backingImageManager:
+      # -- Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/backing-image-manager
+      # -- Specify Longhorn backing image manager image tag
+      tag: v1.6.0-arm64
+    supportBundleKit:
+      # -- Repository for the Longhorn Support Bundle Manager image.
+      repository: longhornio/support-bundle-kit
+      # -- Tag for the Longhorn Support Bundle Manager image.
+      tag: v0.0.33-arm64
+  csi:
+    attacher:
+      # -- Repository for the CSI attacher image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-attacher
+      # -- Tag for the CSI attacher image. When unspecified, Longhorn uses the default value.
+      tag: v4.4.2-arm64
+    provisioner:
+      # -- Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-provisioner
+      # -- Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value.
+      tag: v3.6.2-arm64
+    nodeDriverRegistrar:
+      # -- Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-node-driver-registrar
+      # -- Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value.
+      tag: v2.9.2-arm64
+    resizer:
+      # -- Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-resizer
+      # -- Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value.
+      tag: v1.9.2-arm64
+    snapshotter:
+      # -- Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/csi-snapshotter
+      # -- Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value.
+      tag: v6.3.2-arm64
+    livenessProbe:
+      # -- Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value.
+      repository: longhornio/livenessprobe
+      # -- Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value.
+      tag: v2.11.0-arm64
+
+
+
+longhornUI:
+  # -- Replica count for Longhorn UI.
+  replicas: 1
+
+ingress:
+  # -- Setting that allows Longhorn to generate ingress records for the Longhorn UI service.
+  enabled: true
+
+  # -- IngressClass resource that contains ingress configuration, including the name of the Ingress controller.
+  # ingressClassName can replace the kubernetes.io/ingress.class annotation used in earlier Kubernetes releases.
+  ingressClassName: "nginx"
+
+  # -- Hostname of the Layer 7 load balancer.
+  host: longhorn.robertbatson.com