Cluster provisions, now to add flux

.gitignore

@@ -1 +1,7 @@
-k3s-cluster-config.yaml
+k3s-cluster-config.kubeconfig.yaml
+*.agekey
+
+.terraform
+.terraform.tfstate*
+terraform.tfstate*
+.env*

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+- repo: https://github.com/k8s-at-home/sops-pre-commit
+  rev: v2.1.0
+  hooks:
+  - id: forbid-secrets

ansible/build-k3s-cluster.yml

@@ -1,5 +1,119 @@
-- name: setup k3s on k3s-nodes
+- name: debug stuff
+  hosts: all
+  tasks:
+    - name: set token fact for k3s-nodes hosts
+      run_once: true
+      with_items: '{{ groups["k3s-nodes"] }}'
+      delegate_to: '{{ item }}'
+      set_fact:
+        fake_hostname: '{{ hostvars[item]["inventory_hostname"] }} for {{ item }}'
+
+    - name: debug token fact for k3s-nodes hosts
+      run_once: true
+      with_items: '{{ groups["k3s-nodes"] }}'
+      delegate_to: '{{ item }}'
+      debug:
+        msg: '{{ hostvars[item]["fake_hostname"] }} for {{ inventory_hostname }}'
+
+- name: install k3s on k3s-nodes
   hosts: k3s-nodes
   gather_facts: yes
   roles:
     - role: k3s-node
+
+# TODO: ensure odd number of k3s-control-nodes hosts
+
+- name: intialize k3s cluster on first control plane node
+  hosts: k3s-control-nodes[0]
+  tasks:
+    - name: create config directory
+      file:
+        recurse: yes
+        path: /etc/rancher/k3s/
+        state: directory
+        mode: '0700'
+
+    - name: copy initial config
+      become: true
+      copy:
+        src: ./templates/k3s-initial-server-config.yaml
+        dest: /etc/rancher/k3s/config.yaml
+
+    - name: run initial k3s server
+      become: true
+      ansible.builtin.command: systemctl enable --now k3s
+
+    - name: fetch token
+      become: true
+      slurp: 
+        src: /var/lib/rancher/k3s/server/node-token
+      register: slurped_k3s_node_token
+
+    - name: set token fact
+      set_fact:
+        k3s_node_token: '{{ slurped_k3s_node_token.content | b64decode }}'
+
+- name: configure remaining nodes
+  hosts: k3s-nodes,!k3s-control-nodes[0]
+  gather_facts: no
+  tasks:
+    - name: debug info
+      ansible.builtin.debug:
+        msg: '{{ ansible_facts }}'
+
+    # - name: debug token fact for k3s-nodes hosts
+    #   run_once: true
+    #   debug:
+    #     msg: '{{ hostvars[groups["k3s-control-nodes"][0]]["k3s_node_token"] }}'
+
+    - name: set token fact
+      set_fact:
+        k3s_node_token: '{{ hostvars[groups["k3s-control-nodes"][0]]["k3s_node_token"] }}'
+
+    - name: configure k3s server with token
+      lineinfile:
+        path: /etc/systemd/system/k3s.service.env
+        state: present
+        line: K3S_TOKEN={{ k3s_node_token }}
+        create: yes
+
+    - name: configure k3s agent with token
+      lineinfile:
+        path: /etc/systemd/system/k3s-agent.service.env
+        state: present
+        line: K3S_TOKEN={{ k3s_node_token }}
+        create: yes
+
+    - name: create config directory
+      file:
+        recurse: yes
+        path: /etc/rancher/k3s/
+        state: directory
+        mode: '0700'
+
+    - name: configure k3s server endpoint
+      lineinfile:
+        path: /etc/rancher/k3s/config.yaml
+        state: present
+        line: 'server: https://{{ hostvars[groups["k3s-control-nodes"][0]]["ansible_default_ipv4"]["address"] }}:6443'
+        create: yes
+
+    # TODO: assert token is valid?
+    # - name: debug info2
+    #   ansible.builtin.debug:
+    #     msg: '{{ k3s_node_token }}'
+    # TODO: wait for initial control node to come up?
+
+- name: start remaining control nodes
+  hosts: k3s-control-nodes,!k3s-control-nodes[0]
+  tasks:
+    - name: run k3s
+      become: true
+      ansible.builtin.command: systemctl enable --now k3s
+
+- name: start remaining agent nodes
+  hosts: k3s-agent-nodes
+  tasks:
+    - name: run k3s
+      become: true
+      ansible.builtin.command: systemctl enable --now k3s-agent

ansible/inventory/README.md

@@ -0,0 +1 @@
+# [lyte.dev](https://lyte.dev)

ansible/inventory/hosts.yml

@@ -1,13 +0,0 @@
-k3s-nodes:
-  hosts:
-    "root@10.0.0.87":
-      control-plane-node: true
-    "root@10.0.0.138":
-      agent-node: true
-
-  vars:
-    ansible_python_interpreter: /usr/bin/python3.10
-    cluster_cidr: '192.168.0.0/16'
-    k3s:
-      version: v1.23.3+k3s1
-      master: 10.0.0.87

ansible/inventory/tags

@@ -0,0 +1,12 @@
+!_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
+!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
+!_TAG_OUTPUT_EXCMD	mixed	/number, pattern, mixed, or combineV2/
+!_TAG_OUTPUT_FILESEP	slash	/slash or backslash/
+!_TAG_OUTPUT_MODE	u-ctags	/u-ctags or e-ctags/
+!_TAG_PATTERN_LENGTH_LIMIT	96	/0 for no limit/
+!_TAG_PROC_CWD	/home/daniel/code/home-k8s-cluster/ansible/inventory/	//
+!_TAG_PROGRAM_AUTHOR	Universal Ctags Team	//
+!_TAG_PROGRAM_NAME	Universal Ctags	/Derived from Exuberant Ctags/
+!_TAG_PROGRAM_URL	https://ctags.io/	/official site/
+!_TAG_PROGRAM_VERSION	5.9.0	/p5.9.20210905.0/
+[lyte.dev](https://lyte.dev)	README.md	/^# [lyte.dev](https:\/\/lyte.dev)$/;"	c

ansible/k3s-node/tasks/create-aur-builder-user.yml

@@ -0,0 +1,9 @@
+# source: https://wiki.archlinux.org/title/Ansible#AUR
+- user: name=aur_builder
+
+- lineinfile:
+    path: /etc/sudoers.d/aur_builder-allow-to-sudo-pacman
+    state: present
+    line: "aur_builder ALL=(ALL) NOPASSWD: /usr/bin/pacman"
+    validate: /usr/sbin/visudo -cf %s
+    create: yes

ansible/k3s-node/tasks/create-aur-builder.yml

@@ -1,9 +0,0 @@
-# source: https://wiki.archlinux.org/title/Ansible#AUR
-- user: name=aur_builder
-
-- lineinfile:
-     path: /etc/sudoers.d/aur_builder-allow-to-sudo-pacman
-     state: present
-     line: "aur_builder ALL=(ALL) NOPASSWD: /usr/bin/pacman"
-     validate: /usr/sbin/visudo -cf %s
-     create: yes

ansible/k3s-node/tasks/install-aur-helper.yml

@@ -0,0 +1,15 @@
+- name: Check if AUR helper ({{ aur_helper_pkg }}) is already installed
+  shell: pacman -Q 2>&1 | grep '^{{ aur_helper_pkg }} ' >/dev/null 2>&1
+  register: is_aur_helper_exist
+  ignore_errors: yes
+
+# - name: debug is_aur_helper_exist
+#   debug:
+#     msg: '{{ is_aur_helper_exist }}'
+
+- name: install AUR helper ({{ aur_helper_pkg }})
+  include_tasks: aur.yml
+  when: is_aur_helper_exist.failed
+  vars:
+    makepkg_nonroot_user: aur_builder
+    pkg_name: '{{ aur_helper_pkg }}'

ansible/k3s-node/tasks/install-k3s.yml

@@ -1,3 +1,10 @@
-- name: install k3s
+- name: download k3s installer
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: /tmp/k3s-installer.sh
+    mode: '0777'
+
+- name: run k3s installer
+  become: true
   ansible.builtin.command: >
-    curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{{ k3s.version }}" sh -
+    env INSTALL_K3S_EXEC={{ k3s_command | default('server') }} INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION="{{ k3s.version }}" sh /tmp/k3s-installer.sh

ansible/k3s-node/tasks/install-yay-bin.yml

@@ -1,5 +0,0 @@
-- name: install yay-bin
-  include_tasks: aur.yml
-  vars:
-    makepkg_nonroot_user: aur_builder
-    pkg_name: yay-bin

ansible/k3s-node/tasks/main.yml

@@ -1,10 +1,9 @@
 - include_tasks: ./prepare-host.yml
 - include_tasks: ./install-k3s.yml
 
-# TODO: setup loadbalancer?
+# TODO: rpi4 loadbalancer? haproxy? metallb?
 
-- include_tasks: ./start-k3s-control-nodes.yml
-  # TODO: probably need some output here?
-
-- include_tasks: ./start-k3s-agents.yml
-  # TODO: probably need the output from control nodes
+# TODO: generate secret
+# TODO: start first control node
+# TODO: start remaining control nodes
+# TODO: start agent nodes

ansible/k3s-node/tasks/prepare-host.yml

@@ -1,19 +1,23 @@
 # arch linux
-- include_tasks: ./create-aur-builder.yml
+- include_tasks: ./create-aur-builder-user.yml
   when: ansible_facts.os_family == 'Archlinux'
 
-- include_tasks: ./install-yay-bin.yml
+- include_tasks: ./install-aur-helper.yml
   when: ansible_facts.os_family == 'Archlinux'
+  vars:
+    aur_helper_pkg: paru-bin
 
 # common
 - include_tasks: ./use-firewalld-with-iptables.yml
 
-- name: open http and https ports in firewalld
-  ansible.builtin.command: >
-    sudo firewall-cmd --add-port=443/tcp --permanent \
-      && sudo firewall-cmd --add-port=80/tcp --permanent \
-      && sudo firewall-cmd --reload
+# - name: open http and https ports in firewalld
+#   ansible.builtin.command: >
+#     sudo firewall-cmd --add-port=443/tcp --permanent \
+#       && sudo firewall-cmd --add-port=80/tcp --permanent \
+#       && sudo firewall-cmd --reload
 
-- include_tasks: ./update-all.yml
+- include_tasks: ./upgrade-all-packages.yml
+  vars:
+    aur_helper_command: paru
 
   # _todo: net.ipv4.ip_forward = 1

ansible/k3s-node/tasks/upgrade-all-packages.yml

@@ -1,7 +1,7 @@
 - name: upgrade all packages
   when: ansible_facts.os_family == 'Archlinux'
   community.general.pacman:
-    # executable: yay
+    executable: '{{ aur_helper_command }}'
     force: yes
     state: latest
     update_cache: yes

ansible/k3s-node/tasks/use-firewalld-with-iptables.yml

@@ -1,8 +1,9 @@
-- name: uninstall nftables, install firewalld, configure it to use iptables, and activate it
-  when: ansible_facts.os_family == 'Archlinux'
-  become: true
-  ansible.builtin.command: >
-    pacman -Rs nftables \
-      && pacman -Sy firewalld \
-      && sed -i 's/FirewallBackend=.*/FirewallBackend=iptables/g' /etc/firewalld/firewalld.conf \
-      && systemctl enable --now firewalld
+#- name: uninstall nftables, install firewalld, configure it to use iptables, and activate it
+#  when: ansible_facts.os_family == 'Archlinux'
+#  become: true
+#  ansible.builtin.command: >
+#    pacman -Rs nftables \
+#      && pacman -Sy firewalld \
+#      && sed -i 's/FirewallBackend=.*/FirewallBackend=iptables/g' /etc/firewalld/firewalld.conf \
+#      && systemctl enable firewalld \
+#      && systemctl restart firewalld

ansible/nuke-k3s-cluster.yml

@@ -1,7 +1,7 @@
 - name: purge k3s from k3s-nodes
   hosts: all
   become: true
-  gather_facts: yes
+  gather_facts: no
   any_errors_fatal: true
   tasks:
     - name: Kill k3s
@@ -24,3 +24,4 @@
         path: "{{ item.path }}"
         state: absent
       loop: "{{ files_to_delete.files }}"
+

ansible/templates/k3s-initial-server-config.yaml

@@ -0,0 +1 @@
+cluster-init: true

cluster/base/cluster-secrets.sops.yaml

@@ -0,0 +1,31 @@
+hello: ENC[AES256_GCM,data:K9I1dpze++/QMFBdZWk6y6nKC8oAhY1CAapUA0c6cIfpLzrc7iElJCYR/pS3ag==,iv:T/yuhqcW5eZYfhXXPFn8CrHMMDcI0ZfqMLkZ5fJ7Igc=,tag:n/hkY8p3oNrn9Ydddipqog==,type:str]
+example_key: ENC[AES256_GCM,data:n8rieIC5BVQsF3Dhqg==,iv:hiYkThvF1NdvN2iLhYXV8HEaBTKDxnoSUhZtouGVweY=,tag:hDyriQCq+XAGvi2Ec+y8cw==,type:str]
+#ENC[AES256_GCM,data:XC+U/PXcNkxqmJ3angsSwg==,iv:x/Yjgb5knFmOVWd2zEl0Pg/gI0crhFTZxlOkNyKE018=,tag:yeG1QkCjuHUWkUZc/XdBHA==,type:comment]
+example_array:
+    - ENC[AES256_GCM,data:PSrse6STeVIFo9PkXyU=,iv:tS9F+TpQBEHjfbqeIIBtjodOZigPu54XOTpq8+yxcxY=,tag:xj+mejjpKkDWr8WKldQbYw==,type:str]
+    - ENC[AES256_GCM,data:aY3qtnbxnW2oaajFNTs=,iv:imt+1DrFCz8RgT1bmooJH8lixfpvMDcECLnGnd63hfQ=,tag:68iFyD0I49aWt2+oYgAa/w==,type:str]
+example_number: ENC[AES256_GCM,data:ej7bNwZ3cxQxRw==,iv:LAh/cdJou71n++0EkQZzN+UDm60Cy3LzgalDStgCx1c=,tag:7Lls9fFlXq/VfmvDl3M1rQ==,type:float]
+example_booleans:
+    - ENC[AES256_GCM,data:22QmvQ==,iv:InUfL+AdUxhgmKo2i7LiOfOtBoRBXBuBpTsJuk9VuCk=,tag:KdEctBLG8KaLGEh8iLFIIw==,type:bool]
+    - ENC[AES256_GCM,data:rURSANQ=,iv:uHsCsINq4S3sQbWboqu0EibbAw4BUMOex6akgW2mnc0=,tag:U7VsVGaNLUk1TaTfxj0uDg==,type:bool]
+example_name: ENC[AES256_GCM,data:f7lJZchGCg==,iv:VLlISItMl0n3g6dJQ/tXSMficupxWtDc9KgjBTFG+P4=,tag:HbNRFL3l3pPyQoPJH8Te7g==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1stdue5q5teskee057ced6rh9pzzr93xsy66w4sc3zu49rgxl7cjshztt45
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVK0VGZThqZTRBeWtvOGtt
+            aFdicnVUVTBUSlJsc2piTVJ5djBrbE83aFJBCjhOYmpuNG96eXl0dFBzSzQ5RVNT
+            WUdDVmVRZnE0NFRvdWF0ZGROS3JhMWsKLS0tIDQrUzlCS211cHlsa3ZTcEd0TEJn
+            Y2wwV2grQWdBdWQ4MjYwZzZiZlNkWEEKmzf6mc3qeYk7EOC35h9CdIBeV77W0Iv8
+            7b0wRSGW0JQIL0ffR+mWl2RGNM7ykSnwz1mic1HMjsrd/exhhMLjEg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-02-08T03:00:15Z"
+    mac: ENC[AES256_GCM,data:biFMSADG1RFArTPfx7kO/jF6hFmW68kQJ1+BYaBt1wMTwpknCwLC60ls625pEbCYDx40Wt8nTHc5Px/0iGkTfA261c15k1o/ya3IH9U2mSui/4tAxHwHps5OC7eK2Cg10YoEXT1vute7rNbXxerLg0Vn5sPK2PAYmC/zkvTdJNc=,iv:ac69kwcLigbnrwgD1L/DFhV23QmY40CucbXFXs3f8ss=,tag:Qck4/u8HfEPJ2SoZnMLvtA==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1

readme.md

@@ -1,24 +1,64 @@
-# Provision Machines
+# home-k8s-cluster
 
-- My nodes are Arch Linux machines on bare metal
+This repository contains the configuration, scripts, and other goodies for
+building and managing my home cluster. I share the source with you so you can
+make exactly the same mistakes as I do.
+
+## Setup
+
+Setup the pre-commit hooks before you change anything!
+
+```
+pip install pre-commit
+pre-commit install --install-hooks
+pre-commit autoupdate
+```
+
+## Provision Machines
+
+Before we interact with the cluster, we have some manual work to do.
+
+### Manual Preparation
+
+- Currently, my nodes are Arch Linux machines on bare metal
   - Nodes must be ready to be controlled via Ansible
-    - Have `python3` installed for Ansible
-    - Need to be `ssh`-able from a workstation
-      - You can grab keys like so: `curl -L files.lyte.dev/key.pub >> ~/.ssh/authorized_keys`
-    - TODO: script this? maybe custom ISO+PXEBoot?
+    - Have `python3` installed
+    - Need to be `ssh`-able from a controller (my workstation)
+      - `curl -L files.lyte.dev/key.pub >> ~/.ssh/authorized_keys`
+
+**TODO**: script this? maybe custom ISO+PXEBoot? Talos+Sidero?
+
+### Automated Provisioning
+
 - Setup Ansible on the controller (from `./ansible`)
   - `ansible-galaxy install -r requirements.yml --force`
 - Verify Ansible can reach hosts (from `./ansible`)
   - `ansible all -i inventory/hosts.yml --list-hosts`
   - `ansible all -i inventory/hosts.yml -m ping`
-- Use Ansible to prepare hosts for k3s installation (from `./ansible`)
-  - `ansible-playbook -i inventory/hosts.yml ./provision.yml`
-- Use Ansible to install k3s as configured on all nodes (from `./ansible`)
-  - `ansible-playbook -i inventory/hosts.yml ./install-k3s.yml`
-    - You have to run this multiple times for the worker nodes to successfully
-      connect to the control plane nodes
-- Setup your local kubectl to work with the new cluster
-  - `ansible -i ansible/inventory/hosts.yml $REMOTE_HOST -m fetch -a "src=/etc/rancher/k3s/k3s.yaml dest=./k3s-cluster-config.yaml flat=yes"`
-  - Copy the cluster information from the `./k3s-cluster-config.yaml` file into
-    your existing `~/.kube/config` (or just copy it there if it doesn't exist)
-    - You will need to edit the host from `localhost`/`127.0.0.1` to the correct host
+- Use Ansible to build the cluster as configured on all nodes (from `./ansible`)
+  - `ansible-playbook -i inventory/hosts.yml ./build-k3s-cluster`
+
+And the cluster is up! If you want to interact with it from your controller,
+you can do this:
+
+```bash
+ansible -i ansible/inventory/hosts.yml $REMOTE_HOST -m fetch \
+  -a "src=/etc/rancher/k3s/k3s.yaml dest=./k3s-cluster-config.kubeconfig.yaml flat=yes"
+# TODO: this did not work for me
+# env KUBECONFIG="~/.kube/config:./k3s-cluster-config.kubeconfig.yaml" \
+# kubectl config view --flatten | sed "s/127.0.0.1/$REMOTE_HOST/" > ~/.kube/new-config
+```
+
+- Copy the cluster information from the `./k3s-cluster-config.kubeconfig.yaml` file into
+  your existing `~/.kube/config` (or just copy it there if it doesn't exist)
+  - You will need to edit the host from `localhost`/`127.0.0.1` to the correct host
+
+### Automated Teardown
+
+```bash
+ansible-playbook -i inventory/hosts.yml ./nuke-k3s-cluster
+```
+
+## Setting up Flux
+
+-