🥉 Working K3s example

README.md

@@ -36,9 +36,9 @@ Planned features:
 - [x] Examples
   - [x] Basic
   - [x] Cloud Init
+  - [x] K3s
   - [ ] NixOS
   - [ ] Virtual Private Cloud
-  - [ ] Kubernetes
 
 ## Server parameters
 

examples/k3s/.gitignore

@@ -38,3 +38,4 @@ terraform.rc
 
 # .kube/config contains credentials
 .kube
+.id_ed25519*

examples/k3s/README.md

@@ -1,15 +1,18 @@
-# WIP - examples/k8s
+# examples/k3s
 
-This WIP example shows how to create a Kubernetes cluster on Binary Lane.
+This example shows how to create a K3s cluster on Binary Lane.
+
+After setting up the cluster by running `terraform apply`, the kubeconfig file will be downloaded to the `.kube/config` file in the current directory. You can then use `kubectl` to interact with the cluster, for example:
+
+The SSH key used to access the servers is stored in the `.id_ed25519` file. You can use it to SSH into the servers:
+
+```sh
+ssh -o "IdentitiesOnly=yes" -i ./.id_ed25519 root@neptune-stand.bnr.la
+```
 
 ## Prior art
 
 - [MartinHodges/create_k8s](https://github.com/MartinHodges/create_k8s) repo and the [medium article](https://medium.com/@martin.hodges/creating-a-kubernetes-cluster-from-scratch-in-1-hour-using-automation-a25e387be547) that goes with it.
 - https://github.com/inscapist/terraform-k3s-private-cloud
 - https://github.com/schnerring/schnerring.github.io/blob/07d06fb40e3d01f2483a739f516ac4d711a5742c/content/blog/use-terraform-to-deploy-an-azure-kubernetes-service-aks-cluster-traefik-2-cert-manager-and-lets-encrypt-certificates/index.md
-
-## Remaining TODOs
-
-- [ ] Consider rewriting module to use `microk8s` instead of `k3s`
-- [ ] Use `kustomize` instead of `helm`
-- [ ] Use the `binarylane_load_balancer` resource instead of `kubectl port-forward` (or use the load balancer in another example)
+- https://github.com/xunleii/terraform-module-k3s

examples/k3s/k3s-agent-join.sh

@@ -1,16 +0,0 @@
-#!/bin/bash -x
-exec >/tmp/k3s-agent-join-debug.log 2>&1
-
-export K3S_TOKEN="${cluster_token}"
-export K3S_URL="https://${cluster_server}:6443"
-
-curl -sfL https://get.k3s.io | sh -s - agent \
-  --node-name="$(hostname -f)"
-
-unset K3S_TOKEN
-unset K3S_URL
-
-# Disable firewall recommended
-ufw disable
-
-echo "K3s Node Join Completed"

examples/k3s/k3s-server-install.sh

@@ -1,25 +0,0 @@
-#!/bin/bash -x
-exec >/tmp/k3s-server-install-debug.log 2>&1
-
-export INSTALL_K3S_NAME="${cluster_id}"
-export K3S_TOKEN="${cluster_token}"
-export K3S_KUBECONFIG_MODE="644"
-
-curl -sfL https://get.k3s.io | sh -s - server \
-  --node-name="$(hostname -f)"
-# --disable-cloud-controller \
-# --disable servicelb \
-# --disable local-storage \
-# --disable traefik
-
-unset INSTALL_K3S_NAME
-unset K3S_TOKEN
-unset K3S_KUBECONFIG_MODE
-
-echo "Installing Helm"
-curl -sfL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
-
-# Disable firewall recommended
-ufw disable
-
-echo "K3s Setup Completed"

examples/k3s/k3s.tf

@@ -0,0 +1,50 @@
+module "k3s" {
+  # source = "xunleii/k3s/module"  # Uncomment after https://github.com/xunleii/terraform-module-k3s/pull/207 is merged
+  source = "github.com/dvdmuckle/terraform-module-k3s?ref=62dca7731b78f1120a141a2703e3ed14470276c0"
+
+  depends_on_    = binarylane_server.agents
+  k3s_version    = "latest"
+  cluster_domain = "cluster.local"
+  managed_fields = ["label"]
+
+  // These subnets are used by the Flannel network interface in K3s, and are NOT the same as the VPC address space
+  cidr = {
+    pods     = "10.42.0.0/16",
+    services = "10.43.0.0/16",
+  }
+
+  servers = {
+    for i in range(length(binarylane_server.servers)) :
+    binarylane_server.servers[i].name => {
+      ip   = binarylane_server.servers[i].private_ipv4_addresses[0]
+      name = binarylane_server.servers[i].name
+      connection = {
+        host        = binarylane_server.servers[i].permalink
+        private_key = trimspace(tls_private_key.ed25519_provisioning.private_key_pem)
+      }
+      labels = { "node.kubernetes.io/type" = "master" }
+      flags = [
+        "--tls-san ${binarylane_server.servers[0].private_ipv4_addresses[0]}",
+      ]
+    }
+  }
+
+  agents = {
+    for i in range(length(binarylane_server.agents)) :
+    binarylane_server.agents[i].name => {
+      ip   = binarylane_server.agents[i].private_ipv4_addresses[0]
+      name = binarylane_server.agents[i].name
+      connection = {
+        host        = binarylane_server.agents[i].permalink
+        private_key = trimspace(tls_private_key.ed25519_provisioning.private_key_pem)
+      }
+      labels = { "node.kubernetes.io/pool" = "general" }
+    }
+  }
+}
+
+resource "local_sensitive_file" "kube_config" {
+  content         = module.k3s.kube_config
+  filename        = ".kube/config"
+  file_permission = "0644"
+}

examples/k3s/main.tf

@@ -1,100 +1,60 @@
 locals {
-  cluster_id    = "tf-example-k8s"
-  custom_domain = "tf-example-k8s.internal"
+  cluster_id    = "tf-example-k3s"
+  custom_domain = "tf-example-k3s.internal"
 }
 
 resource "random_password" "binarylane" {
   length = 18
 }
 
-resource "random_password" "cluster_token" {
-  length  = 64
-  special = false
-}
-
 # SSH Key
 # -------
+resource "tls_private_key" "ed25519_provisioning" {
+  algorithm = "ED25519"
+}
+
+resource "local_sensitive_file" "ssh_private_key" {
+  content         = tls_private_key.ed25519_provisioning.private_key_openssh
+  filename        = ".id_ed25519"
+  file_permission = "0600"
+}
+
+resource "local_file" "ssh_public_key" {
+  content         = tls_private_key.ed25519_provisioning.public_key_openssh
+  filename        = ".id_ed25519.pub"
+  file_permission = "0644"
+}
+
 resource "binarylane_ssh_key" "example" {
-  name       = "tf-example-k8s"
-  public_key = file("~/.ssh/id_ed25519.pub")
+  name       = "tf-example-k3s"
+  public_key = "${trimspace(tls_private_key.ed25519_provisioning.public_key_openssh)} " # whitespace hack until #76 is fixed
 }
 
 # Virtual Private Cloud
 # ---------------------
 resource "binarylane_vpc" "example" {
   name     = local.cluster_id
-  ip_range = "10.240.0.0/16"
+  ip_range = "10.0.0.0/8"
 }
 
 # k3s Servers
 # -----------
-data "cloudinit_config" "server" {
-  gzip          = false
-  base64_encode = false
-
-  part {
-    content_type = "text/x-shellscript"
-    content = templatefile("${path.module}/k3s-server-install.sh", {
-      cluster_id    = local.cluster_id,
-      cluster_token = random_password.cluster_token.result,
-    })
-  }
-}
-
-resource "binarylane_server" "server" {
+resource "binarylane_server" "servers" {
   count = 1
 
   name              = "${local.cluster_id}-server-${count.index + 1}"
   region            = "per"
   image             = "ubuntu-24.04"
-  size              = "std-min"
+  size              = "std-1vcpu" # 2GB memory, 1 vCPU
   password          = random_password.binarylane.result
   ssh_keys          = [binarylane_ssh_key.example.id]
   vpc_id            = binarylane_vpc.example.id
   public_ipv4_count = 1
-  user_data         = sensitive(data.cloudinit_config.server.rendered)
-}
-
-data "external" "kubeconfig" {
-  program = [
-    "/usr/bin/ssh",
-    "-o UserKnownHostsFile=/dev/null",
-    "-o StrictHostKeyChecking=no",
-    "root@${binarylane_server.server.0.permalink}",
-    "echo '{\"kubeconfig\":\"'$(cat /etc/rancher/k3s/k3s.yaml | base64)'\"}'",
-  ]
-}
-
-resource "local_sensitive_file" "kubeconfig" {
-  depends_on = [binarylane_server.server]
-
-  content = replace(
-    base64decode(
-      replace(data.external.kubeconfig.result.kubeconfig, " ", "")
-    ),
-    "server: https://127.0.0.1:6443",
-    "server: https://${binarylane_server.server.0.permalink}:6443",
-  )
-  filename        = "${path.module}/.kube/config"
-  file_permission = "0600"
 }
 
 # k3s Agents
 # ----------
-data "cloudinit_config" "agent" {
-  gzip          = false
-  base64_encode = false
-
-  part {
-    content_type = "text/x-shellscript"
-    content = templatefile("${path.module}/k3s-agent-join.sh", {
-      cluster_token  = random_password.cluster_token.result,
-      cluster_server = binarylane_server.server.0.private_ipv4_addresses.0
-    })
-  }
-}
-
-resource "binarylane_server" "agent" {
+resource "binarylane_server" "agents" {
   count = 2
 
   name              = "${local.cluster_id}-agent-${count.index + 1}"
@@ -105,12 +65,11 @@ resource "binarylane_server" "agent" {
   ssh_keys          = [binarylane_ssh_key.example.id]
   vpc_id            = binarylane_vpc.example.id
   public_ipv4_count = 1
-  user_data         = sensitive(data.cloudinit_config.agent.rendered)
 }
 
 # Virtual Private Cloud Routing
 # -----------------------------
-resource "binarylane_vpc_route_entries" "example" {
+resource "binarylane_vpc_route_entries" "example" { # TODO
   vpc_id = binarylane_vpc.example.id
   route_entries = [
     # {
@@ -122,12 +81,12 @@ resource "binarylane_vpc_route_entries" "example" {
 }
 
 locals {
-  agent_ips  = flatten([for _, a in binarylane_server.agent : a.private_ipv4_addresses])
-  server_ips = flatten([for _, s in binarylane_server.server : s.private_ipv4_addresses])
+  agent_ips  = flatten([for _, a in binarylane_server.agents : a.private_ipv4_addresses])
+  server_ips = flatten([for _, s in binarylane_server.servers : s.private_ipv4_addresses])
 }
 
 resource "binarylane_server_firewall_rules" "example" {
-  for_each = { for server in concat(binarylane_server.server, binarylane_server.agent) : server.name => server }
+  for_each = { for server in concat(binarylane_server.servers, binarylane_server.agents) : server.name => server }
 
   server_id = each.value.id
 

examples/k3s/nginx.tf

@@ -1,64 +0,0 @@
-resource "kubectl_manifest" "hello" {
-  yaml_body = <<YAML
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: nginx-hello
-  labels:
-    app: nginx-hello
-spec:
-  selector:
-    matchLabels:
-      app: nginx-hello
-  template:
-    metadata:
-      labels:
-        app: nginx-hello
-    spec:
-      containers:
-        - image: nginxdemos/hello
-          name: nginx-hello
-          ports:
-            - name: http
-              containerPort: 8080
-              protocol: TCP
-YAML
-}
-
-resource "kubectl_manifest" "hello_service" {
-  yaml_body = <<YAML
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx-hello
-  labels:
-    app: nginx-hello
-spec:
-  type: ClusterIP
-  selector:
-    app: nginx-hello
-  ports:
-    - port: 80
-YAML
-}
-
-resource "kubectl_manifest" "hello_ingress" {
-  yaml_body = <<YAML
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nginx-hello
-spec:
-  rules:
-    - host: "*.bnr.la"
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: nginx-hello
-                port:
-                  number: 80
-  YAML
-}

examples/k3s/providers.tf

@@ -3,25 +3,7 @@ terraform {
     binarylane = {
       source = "oscarhermoso/binarylane"
     }
-    kubectl = {
-      source  = "alekc/kubectl"
-      version = ">= 2.0.0"
-    }
   }
 }
 
 provider "binarylane" {}
-
-provider "helm" {
-  kubernetes {
-    host        = "${binarylane_server.server.0.permalink}:6443"
-    config_path = local_sensitive_file.kubeconfig.filename
-    insecure    = true # TODO
-  }
-}
-
-provider "kubectl" {
-  host        = "${binarylane_server.server.0.permalink}:6443"
-  config_path = local_sensitive_file.kubeconfig.filename
-  insecure    = true # TODO
-}