diff --git a/README.md b/README.md index daf1ad2..63c4370 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,9 @@ Planned features: - [x] Examples - [x] Basic - [x] Cloud Init + - [x] K3s - [ ] NixOS - [ ] Virtual Private Cloud - - [ ] Kubernetes ## Server parameters diff --git a/examples/k3s/.gitignore b/examples/k3s/.gitignore index 533fc6d..bdfadb9 100644 --- a/examples/k3s/.gitignore +++ b/examples/k3s/.gitignore @@ -38,3 +38,4 @@ terraform.rc # .kube/config contains credentials .kube +.id_ed25519* diff --git a/examples/k3s/README.md b/examples/k3s/README.md index d641f31..339f67c 100644 --- a/examples/k3s/README.md +++ b/examples/k3s/README.md @@ -1,15 +1,18 @@ -# WIP - examples/k8s +# examples/k3s -This WIP example shows how to create a Kubernetes cluster on Binary Lane. +This example shows how to create a K3s cluster on Binary Lane. + +After setting up the cluster by running `terraform apply`, the kubeconfig file will be downloaded to the `.kube/config` file in the current directory. You can then use `kubectl` to interact with the cluster, for example: + +The SSH key used to access the servers is stored in the `.id_ed25519` file. You can use it to SSH into the servers: + +```sh +ssh -o "IdentitiesOnly=yes" -i ./.id_ed25519 root@neptune-stand.bnr.la +``` ## Prior art - [MartinHodges/create_k8s](https://github.com/MartinHodges/create_k8s) repo and the [medium article](https://medium.com/@martin.hodges/creating-a-kubernetes-cluster-from-scratch-in-1-hour-using-automation-a25e387be547) that goes with it. - https://github.com/inscapist/terraform-k3s-private-cloud - https://github.com/schnerring/schnerring.github.io/blob/07d06fb40e3d01f2483a739f516ac4d711a5742c/content/blog/use-terraform-to-deploy-an-azure-kubernetes-service-aks-cluster-traefik-2-cert-manager-and-lets-encrypt-certificates/index.md - -## Remaining TODOs - -- [ ] Consider rewriting module to use `microk8s` instead of `k3s` -- [ ] Use `kustomize` instead of `helm` -- [ ] Use the `binarylane_load_balancer` resource instead of `kubectl port-forward` (or use the load balancer in another example) +- https://github.com/xunleii/terraform-module-k3s diff --git a/examples/k3s/k3s-agent-join.sh b/examples/k3s/k3s-agent-join.sh deleted file mode 100644 index 521c1d7..0000000 --- a/examples/k3s/k3s-agent-join.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -x -exec >/tmp/k3s-agent-join-debug.log 2>&1 - -export K3S_TOKEN="${cluster_token}" -export K3S_URL="https://${cluster_server}:6443" - -curl -sfL https://get.k3s.io | sh -s - agent \ - --node-name="$(hostname -f)" - -unset K3S_TOKEN -unset K3S_URL - -# Disable firewall recommended -ufw disable - -echo "K3s Node Join Completed" diff --git a/examples/k3s/k3s-server-install.sh b/examples/k3s/k3s-server-install.sh deleted file mode 100644 index f3766d9..0000000 --- a/examples/k3s/k3s-server-install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -x -exec >/tmp/k3s-server-install-debug.log 2>&1 - -export INSTALL_K3S_NAME="${cluster_id}" -export K3S_TOKEN="${cluster_token}" -export K3S_KUBECONFIG_MODE="644" - -curl -sfL https://get.k3s.io | sh -s - server \ - --node-name="$(hostname -f)" -# --disable-cloud-controller \ -# --disable servicelb \ -# --disable local-storage \ -# --disable traefik - -unset INSTALL_K3S_NAME -unset K3S_TOKEN -unset K3S_KUBECONFIG_MODE - -echo "Installing Helm" -curl -sfL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash - -# Disable firewall recommended -ufw disable - -echo "K3s Setup Completed" diff --git a/examples/k3s/k3s.tf b/examples/k3s/k3s.tf new file mode 100644 index 0000000..bfca66d --- /dev/null +++ b/examples/k3s/k3s.tf @@ -0,0 +1,50 @@ +module "k3s" { + # source = "xunleii/k3s/module" # Uncomment after https://github.com/xunleii/terraform-module-k3s/pull/207 is merged + source = "github.com/dvdmuckle/terraform-module-k3s?ref=62dca7731b78f1120a141a2703e3ed14470276c0" + + depends_on_ = binarylane_server.agents + k3s_version = "latest" + cluster_domain = "cluster.local" + managed_fields = ["label"] + + // These subnets are used by the Flannel network interface in K3s, and are NOT the same as the VPC address space + cidr = { + pods = "10.42.0.0/16", + services = "10.43.0.0/16", + } + + servers = { + for i in range(length(binarylane_server.servers)) : + binarylane_server.servers[i].name => { + ip = binarylane_server.servers[i].private_ipv4_addresses[0] + name = binarylane_server.servers[i].name + connection = { + host = binarylane_server.servers[i].permalink + private_key = trimspace(tls_private_key.ed25519_provisioning.private_key_pem) + } + labels = { "node.kubernetes.io/type" = "master" } + flags = [ + "--tls-san ${binarylane_server.servers[0].private_ipv4_addresses[0]}", + ] + } + } + + agents = { + for i in range(length(binarylane_server.agents)) : + binarylane_server.agents[i].name => { + ip = binarylane_server.agents[i].private_ipv4_addresses[0] + name = binarylane_server.agents[i].name + connection = { + host = binarylane_server.agents[i].permalink + private_key = trimspace(tls_private_key.ed25519_provisioning.private_key_pem) + } + labels = { "node.kubernetes.io/pool" = "general" } + } + } +} + +resource "local_sensitive_file" "kube_config" { + content = module.k3s.kube_config + filename = ".kube/config" + file_permission = "0644" +} diff --git a/examples/k3s/main.tf b/examples/k3s/main.tf index 6341d87..e50509a 100644 --- a/examples/k3s/main.tf +++ b/examples/k3s/main.tf @@ -1,100 +1,60 @@ locals { - cluster_id = "tf-example-k8s" - custom_domain = "tf-example-k8s.internal" + cluster_id = "tf-example-k3s" + custom_domain = "tf-example-k3s.internal" } resource "random_password" "binarylane" { length = 18 } -resource "random_password" "cluster_token" { - length = 64 - special = false -} - # SSH Key # ------- +resource "tls_private_key" "ed25519_provisioning" { + algorithm = "ED25519" +} + +resource "local_sensitive_file" "ssh_private_key" { + content = tls_private_key.ed25519_provisioning.private_key_openssh + filename = ".id_ed25519" + file_permission = "0600" +} + +resource "local_file" "ssh_public_key" { + content = tls_private_key.ed25519_provisioning.public_key_openssh + filename = ".id_ed25519.pub" + file_permission = "0644" +} + resource "binarylane_ssh_key" "example" { - name = "tf-example-k8s" - public_key = file("~/.ssh/id_ed25519.pub") + name = "tf-example-k3s" + public_key = "${trimspace(tls_private_key.ed25519_provisioning.public_key_openssh)} " # whitespace hack until #76 is fixed } # Virtual Private Cloud # --------------------- resource "binarylane_vpc" "example" { name = local.cluster_id - ip_range = "10.240.0.0/16" + ip_range = "10.0.0.0/8" } # k3s Servers # ----------- -data "cloudinit_config" "server" { - gzip = false - base64_encode = false - - part { - content_type = "text/x-shellscript" - content = templatefile("${path.module}/k3s-server-install.sh", { - cluster_id = local.cluster_id, - cluster_token = random_password.cluster_token.result, - }) - } -} - -resource "binarylane_server" "server" { +resource "binarylane_server" "servers" { count = 1 name = "${local.cluster_id}-server-${count.index + 1}" region = "per" image = "ubuntu-24.04" - size = "std-min" + size = "std-1vcpu" # 2GB memory, 1 vCPU password = random_password.binarylane.result ssh_keys = [binarylane_ssh_key.example.id] vpc_id = binarylane_vpc.example.id public_ipv4_count = 1 - user_data = sensitive(data.cloudinit_config.server.rendered) -} - -data "external" "kubeconfig" { - program = [ - "/usr/bin/ssh", - "-o UserKnownHostsFile=/dev/null", - "-o StrictHostKeyChecking=no", - "root@${binarylane_server.server.0.permalink}", - "echo '{\"kubeconfig\":\"'$(cat /etc/rancher/k3s/k3s.yaml | base64)'\"}'", - ] -} - -resource "local_sensitive_file" "kubeconfig" { - depends_on = [binarylane_server.server] - - content = replace( - base64decode( - replace(data.external.kubeconfig.result.kubeconfig, " ", "") - ), - "server: https://127.0.0.1:6443", - "server: https://${binarylane_server.server.0.permalink}:6443", - ) - filename = "${path.module}/.kube/config" - file_permission = "0600" } # k3s Agents # ---------- -data "cloudinit_config" "agent" { - gzip = false - base64_encode = false - - part { - content_type = "text/x-shellscript" - content = templatefile("${path.module}/k3s-agent-join.sh", { - cluster_token = random_password.cluster_token.result, - cluster_server = binarylane_server.server.0.private_ipv4_addresses.0 - }) - } -} - -resource "binarylane_server" "agent" { +resource "binarylane_server" "agents" { count = 2 name = "${local.cluster_id}-agent-${count.index + 1}" @@ -105,12 +65,11 @@ resource "binarylane_server" "agent" { ssh_keys = [binarylane_ssh_key.example.id] vpc_id = binarylane_vpc.example.id public_ipv4_count = 1 - user_data = sensitive(data.cloudinit_config.agent.rendered) } # Virtual Private Cloud Routing # ----------------------------- -resource "binarylane_vpc_route_entries" "example" { +resource "binarylane_vpc_route_entries" "example" { # TODO vpc_id = binarylane_vpc.example.id route_entries = [ # { @@ -122,12 +81,12 @@ resource "binarylane_vpc_route_entries" "example" { } locals { - agent_ips = flatten([for _, a in binarylane_server.agent : a.private_ipv4_addresses]) - server_ips = flatten([for _, s in binarylane_server.server : s.private_ipv4_addresses]) + agent_ips = flatten([for _, a in binarylane_server.agents : a.private_ipv4_addresses]) + server_ips = flatten([for _, s in binarylane_server.servers : s.private_ipv4_addresses]) } resource "binarylane_server_firewall_rules" "example" { - for_each = { for server in concat(binarylane_server.server, binarylane_server.agent) : server.name => server } + for_each = { for server in concat(binarylane_server.servers, binarylane_server.agents) : server.name => server } server_id = each.value.id diff --git a/examples/k3s/nginx.tf b/examples/k3s/nginx.tf deleted file mode 100644 index 6fe3d97..0000000 --- a/examples/k3s/nginx.tf +++ /dev/null @@ -1,64 +0,0 @@ -resource "kubectl_manifest" "hello" { - yaml_body = <