diff --git a/.gitignore b/.gitignore index b291ca4..ee03db6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ -k3s-cluster-config.yaml +k3s-cluster-config.kubeconfig.yaml +*.agekey + +.terraform +.terraform.tfstate* +terraform.tfstate* +.env* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..aa40202 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: +- repo: https://github.com/k8s-at-home/sops-pre-commit + rev: v2.1.0 + hooks: + - id: forbid-secrets diff --git a/ansible/build-k3s-cluster.yml b/ansible/build-k3s-cluster.yml index 1d3eae1..eda2d25 100644 --- a/ansible/build-k3s-cluster.yml +++ b/ansible/build-k3s-cluster.yml @@ -1,5 +1,119 @@ -- name: setup k3s on k3s-nodes +- name: debug stuff + hosts: all + tasks: + - name: set token fact for k3s-nodes hosts + run_once: true + with_items: '{{ groups["k3s-nodes"] }}' + delegate_to: '{{ item }}' + set_fact: + fake_hostname: '{{ hostvars[item]["inventory_hostname"] }} for {{ item }}' + + - name: debug token fact for k3s-nodes hosts + run_once: true + with_items: '{{ groups["k3s-nodes"] }}' + delegate_to: '{{ item }}' + debug: + msg: '{{ hostvars[item]["fake_hostname"] }} for {{ inventory_hostname }}' + +- name: install k3s on k3s-nodes hosts: k3s-nodes gather_facts: yes roles: - role: k3s-node + +# TODO: ensure odd number of k3s-control-nodes hosts + +- name: intialize k3s cluster on first control plane node + hosts: k3s-control-nodes[0] + tasks: + - name: create config directory + file: + recurse: yes + path: /etc/rancher/k3s/ + state: directory + mode: '0700' + + - name: copy initial config + become: true + copy: + src: ./templates/k3s-initial-server-config.yaml + dest: /etc/rancher/k3s/config.yaml + + - name: run initial k3s server + become: true + ansible.builtin.command: systemctl enable --now k3s + + - name: fetch token + become: true + slurp: + src: /var/lib/rancher/k3s/server/node-token + register: slurped_k3s_node_token + + - name: set token fact + set_fact: + k3s_node_token: '{{ slurped_k3s_node_token.content | b64decode }}' + +- name: configure remaining nodes + hosts: k3s-nodes,!k3s-control-nodes[0] + gather_facts: no + tasks: + - name: debug info + ansible.builtin.debug: + msg: '{{ ansible_facts }}' + + # - name: debug token fact for k3s-nodes hosts + # run_once: true + # debug: + # msg: '{{ hostvars[groups["k3s-control-nodes"][0]]["k3s_node_token"] }}' + + - name: set token fact + set_fact: + k3s_node_token: '{{ hostvars[groups["k3s-control-nodes"][0]]["k3s_node_token"] }}' + + - name: configure k3s server with token + lineinfile: + path: /etc/systemd/system/k3s.service.env + state: present + line: K3S_TOKEN={{ k3s_node_token }} + create: yes + + - name: configure k3s agent with token + lineinfile: + path: /etc/systemd/system/k3s-agent.service.env + state: present + line: K3S_TOKEN={{ k3s_node_token }} + create: yes + + - name: create config directory + file: + recurse: yes + path: /etc/rancher/k3s/ + state: directory + mode: '0700' + + - name: configure k3s server endpoint + lineinfile: + path: /etc/rancher/k3s/config.yaml + state: present + line: 'server: https://{{ hostvars[groups["k3s-control-nodes"][0]]["ansible_default_ipv4"]["address"] }}:6443' + create: yes + + # TODO: assert token is valid? + # - name: debug info2 + # ansible.builtin.debug: + # msg: '{{ k3s_node_token }}' + # TODO: wait for initial control node to come up? + +- name: start remaining control nodes + hosts: k3s-control-nodes,!k3s-control-nodes[0] + tasks: + - name: run k3s + become: true + ansible.builtin.command: systemctl enable --now k3s + +- name: start remaining agent nodes + hosts: k3s-agent-nodes + tasks: + - name: run k3s + become: true + ansible.builtin.command: systemctl enable --now k3s-agent diff --git a/ansible/inventory/README.md b/ansible/inventory/README.md new file mode 100644 index 0000000..3c68e8c --- /dev/null +++ b/ansible/inventory/README.md @@ -0,0 +1 @@ +# [lyte.dev](https://lyte.dev) diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml deleted file mode 100644 index 0ec04a2..0000000 --- a/ansible/inventory/hosts.yml +++ /dev/null @@ -1,13 +0,0 @@ -k3s-nodes: - hosts: - "root@10.0.0.87": - control-plane-node: true - "root@10.0.0.138": - agent-node: true - - vars: - ansible_python_interpreter: /usr/bin/python3.10 - cluster_cidr: '192.168.0.0/16' - k3s: - version: v1.23.3+k3s1 - master: 10.0.0.87 diff --git a/ansible/inventory/tags b/ansible/inventory/tags new file mode 100644 index 0000000..44a6742 --- /dev/null +++ b/ansible/inventory/tags @@ -0,0 +1,12 @@ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_OUTPUT_EXCMD mixed /number, pattern, mixed, or combineV2/ +!_TAG_OUTPUT_FILESEP slash /slash or backslash/ +!_TAG_OUTPUT_MODE u-ctags /u-ctags or e-ctags/ +!_TAG_PATTERN_LENGTH_LIMIT 96 /0 for no limit/ +!_TAG_PROC_CWD /home/daniel/code/home-k8s-cluster/ansible/inventory/ // +!_TAG_PROGRAM_AUTHOR Universal Ctags Team // +!_TAG_PROGRAM_NAME Universal Ctags /Derived from Exuberant Ctags/ +!_TAG_PROGRAM_URL https://ctags.io/ /official site/ +!_TAG_PROGRAM_VERSION 5.9.0 /p5.9.20210905.0/ +[lyte.dev](https://lyte.dev) README.md /^# [lyte.dev](https:\/\/lyte.dev)$/;" c diff --git a/ansible/k3s-node/tasks/create-aur-builder-user.yml b/ansible/k3s-node/tasks/create-aur-builder-user.yml new file mode 100644 index 0000000..0e2c4b2 --- /dev/null +++ b/ansible/k3s-node/tasks/create-aur-builder-user.yml @@ -0,0 +1,9 @@ +# source: https://wiki.archlinux.org/title/Ansible#AUR +- user: name=aur_builder + +- lineinfile: + path: /etc/sudoers.d/aur_builder-allow-to-sudo-pacman + state: present + line: "aur_builder ALL=(ALL) NOPASSWD: /usr/bin/pacman" + validate: /usr/sbin/visudo -cf %s + create: yes diff --git a/ansible/k3s-node/tasks/create-aur-builder.yml b/ansible/k3s-node/tasks/create-aur-builder.yml deleted file mode 100644 index 277ee71..0000000 --- a/ansible/k3s-node/tasks/create-aur-builder.yml +++ /dev/null @@ -1,9 +0,0 @@ -# source: https://wiki.archlinux.org/title/Ansible#AUR -- user: name=aur_builder - -- lineinfile: - path: /etc/sudoers.d/aur_builder-allow-to-sudo-pacman - state: present - line: "aur_builder ALL=(ALL) NOPASSWD: /usr/bin/pacman" - validate: /usr/sbin/visudo -cf %s - create: yes diff --git a/ansible/k3s-node/tasks/install-aur-helper.yml b/ansible/k3s-node/tasks/install-aur-helper.yml new file mode 100644 index 0000000..d275349 --- /dev/null +++ b/ansible/k3s-node/tasks/install-aur-helper.yml @@ -0,0 +1,15 @@ +- name: Check if AUR helper ({{ aur_helper_pkg }}) is already installed + shell: pacman -Q 2>&1 | grep '^{{ aur_helper_pkg }} ' >/dev/null 2>&1 + register: is_aur_helper_exist + ignore_errors: yes + +# - name: debug is_aur_helper_exist +# debug: +# msg: '{{ is_aur_helper_exist }}' + +- name: install AUR helper ({{ aur_helper_pkg }}) + include_tasks: aur.yml + when: is_aur_helper_exist.failed + vars: + makepkg_nonroot_user: aur_builder + pkg_name: '{{ aur_helper_pkg }}' diff --git a/ansible/k3s-node/tasks/install-k3s.yml b/ansible/k3s-node/tasks/install-k3s.yml index 257bba9..b99da94 100644 --- a/ansible/k3s-node/tasks/install-k3s.yml +++ b/ansible/k3s-node/tasks/install-k3s.yml @@ -1,3 +1,10 @@ -- name: install k3s +- name: download k3s installer + ansible.builtin.get_url: + url: https://get.k3s.io + dest: /tmp/k3s-installer.sh + mode: '0777' + +- name: run k3s installer + become: true ansible.builtin.command: > - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="{{ k3s.version }}" sh - + env INSTALL_K3S_EXEC={{ k3s_command | default('server') }} INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION="{{ k3s.version }}" sh /tmp/k3s-installer.sh diff --git a/ansible/k3s-node/tasks/install-yay-bin.yml b/ansible/k3s-node/tasks/install-yay-bin.yml deleted file mode 100644 index 590bdeb..0000000 --- a/ansible/k3s-node/tasks/install-yay-bin.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: install yay-bin - include_tasks: aur.yml - vars: - makepkg_nonroot_user: aur_builder - pkg_name: yay-bin diff --git a/ansible/k3s-node/tasks/main.yml b/ansible/k3s-node/tasks/main.yml index a05e2f3..590deb4 100644 --- a/ansible/k3s-node/tasks/main.yml +++ b/ansible/k3s-node/tasks/main.yml @@ -1,10 +1,9 @@ - include_tasks: ./prepare-host.yml - include_tasks: ./install-k3s.yml -# TODO: setup loadbalancer? +# TODO: rpi4 loadbalancer? haproxy? metallb? -- include_tasks: ./start-k3s-control-nodes.yml - # TODO: probably need some output here? - -- include_tasks: ./start-k3s-agents.yml - # TODO: probably need the output from control nodes +# TODO: generate secret +# TODO: start first control node +# TODO: start remaining control nodes +# TODO: start agent nodes diff --git a/ansible/k3s-node/tasks/prepare-host.yml b/ansible/k3s-node/tasks/prepare-host.yml index 99ee377..7a92a20 100644 --- a/ansible/k3s-node/tasks/prepare-host.yml +++ b/ansible/k3s-node/tasks/prepare-host.yml @@ -1,19 +1,23 @@ # arch linux -- include_tasks: ./create-aur-builder.yml +- include_tasks: ./create-aur-builder-user.yml when: ansible_facts.os_family == 'Archlinux' -- include_tasks: ./install-yay-bin.yml +- include_tasks: ./install-aur-helper.yml when: ansible_facts.os_family == 'Archlinux' + vars: + aur_helper_pkg: paru-bin # common - include_tasks: ./use-firewalld-with-iptables.yml -- name: open http and https ports in firewalld - ansible.builtin.command: > - sudo firewall-cmd --add-port=443/tcp --permanent \ - && sudo firewall-cmd --add-port=80/tcp --permanent \ - && sudo firewall-cmd --reload +# - name: open http and https ports in firewalld +# ansible.builtin.command: > +# sudo firewall-cmd --add-port=443/tcp --permanent \ +# && sudo firewall-cmd --add-port=80/tcp --permanent \ +# && sudo firewall-cmd --reload -- include_tasks: ./update-all.yml +- include_tasks: ./upgrade-all-packages.yml + vars: + aur_helper_command: paru # _todo: net.ipv4.ip_forward = 1 diff --git a/ansible/k3s-node/tasks/upgrade-all-packages.yml b/ansible/k3s-node/tasks/upgrade-all-packages.yml index 02530b4..b578ab7 100644 --- a/ansible/k3s-node/tasks/upgrade-all-packages.yml +++ b/ansible/k3s-node/tasks/upgrade-all-packages.yml @@ -1,7 +1,7 @@ - name: upgrade all packages when: ansible_facts.os_family == 'Archlinux' community.general.pacman: - # executable: yay + executable: '{{ aur_helper_command }}' force: yes state: latest update_cache: yes diff --git a/ansible/k3s-node/tasks/use-firewalld-with-iptables.yml b/ansible/k3s-node/tasks/use-firewalld-with-iptables.yml index a154c7f..de24883 100644 --- a/ansible/k3s-node/tasks/use-firewalld-with-iptables.yml +++ b/ansible/k3s-node/tasks/use-firewalld-with-iptables.yml @@ -1,8 +1,9 @@ -- name: uninstall nftables, install firewalld, configure it to use iptables, and activate it - when: ansible_facts.os_family == 'Archlinux' - become: true - ansible.builtin.command: > - pacman -Rs nftables \ - && pacman -Sy firewalld \ - && sed -i 's/FirewallBackend=.*/FirewallBackend=iptables/g' /etc/firewalld/firewalld.conf \ - && systemctl enable --now firewalld +#- name: uninstall nftables, install firewalld, configure it to use iptables, and activate it +# when: ansible_facts.os_family == 'Archlinux' +# become: true +# ansible.builtin.command: > +# pacman -Rs nftables \ +# && pacman -Sy firewalld \ +# && sed -i 's/FirewallBackend=.*/FirewallBackend=iptables/g' /etc/firewalld/firewalld.conf \ +# && systemctl enable firewalld \ +# && systemctl restart firewalld diff --git a/ansible/nuke-k3s-cluster.yml b/ansible/nuke-k3s-cluster.yml index dde990b..809535f 100644 --- a/ansible/nuke-k3s-cluster.yml +++ b/ansible/nuke-k3s-cluster.yml @@ -1,7 +1,7 @@ - name: purge k3s from k3s-nodes hosts: all become: true - gather_facts: yes + gather_facts: no any_errors_fatal: true tasks: - name: Kill k3s @@ -24,3 +24,4 @@ path: "{{ item.path }}" state: absent loop: "{{ files_to_delete.files }}" + diff --git a/ansible/templates/k3s-initial-server-config.yaml b/ansible/templates/k3s-initial-server-config.yaml new file mode 100644 index 0000000..55ecbc1 --- /dev/null +++ b/ansible/templates/k3s-initial-server-config.yaml @@ -0,0 +1 @@ +cluster-init: true diff --git a/cluster/base/cluster-secrets.sops.yaml b/cluster/base/cluster-secrets.sops.yaml new file mode 100644 index 0000000..14dc545 --- /dev/null +++ b/cluster/base/cluster-secrets.sops.yaml @@ -0,0 +1,31 @@ +hello: ENC[AES256_GCM,data:K9I1dpze++/QMFBdZWk6y6nKC8oAhY1CAapUA0c6cIfpLzrc7iElJCYR/pS3ag==,iv:T/yuhqcW5eZYfhXXPFn8CrHMMDcI0ZfqMLkZ5fJ7Igc=,tag:n/hkY8p3oNrn9Ydddipqog==,type:str] +example_key: ENC[AES256_GCM,data:n8rieIC5BVQsF3Dhqg==,iv:hiYkThvF1NdvN2iLhYXV8HEaBTKDxnoSUhZtouGVweY=,tag:hDyriQCq+XAGvi2Ec+y8cw==,type:str] +#ENC[AES256_GCM,data:XC+U/PXcNkxqmJ3angsSwg==,iv:x/Yjgb5knFmOVWd2zEl0Pg/gI0crhFTZxlOkNyKE018=,tag:yeG1QkCjuHUWkUZc/XdBHA==,type:comment] +example_array: + - ENC[AES256_GCM,data:PSrse6STeVIFo9PkXyU=,iv:tS9F+TpQBEHjfbqeIIBtjodOZigPu54XOTpq8+yxcxY=,tag:xj+mejjpKkDWr8WKldQbYw==,type:str] + - ENC[AES256_GCM,data:aY3qtnbxnW2oaajFNTs=,iv:imt+1DrFCz8RgT1bmooJH8lixfpvMDcECLnGnd63hfQ=,tag:68iFyD0I49aWt2+oYgAa/w==,type:str] +example_number: ENC[AES256_GCM,data:ej7bNwZ3cxQxRw==,iv:LAh/cdJou71n++0EkQZzN+UDm60Cy3LzgalDStgCx1c=,tag:7Lls9fFlXq/VfmvDl3M1rQ==,type:float] +example_booleans: + - ENC[AES256_GCM,data:22QmvQ==,iv:InUfL+AdUxhgmKo2i7LiOfOtBoRBXBuBpTsJuk9VuCk=,tag:KdEctBLG8KaLGEh8iLFIIw==,type:bool] + - ENC[AES256_GCM,data:rURSANQ=,iv:uHsCsINq4S3sQbWboqu0EibbAw4BUMOex6akgW2mnc0=,tag:U7VsVGaNLUk1TaTfxj0uDg==,type:bool] +example_name: ENC[AES256_GCM,data:f7lJZchGCg==,iv:VLlISItMl0n3g6dJQ/tXSMficupxWtDc9KgjBTFG+P4=,tag:HbNRFL3l3pPyQoPJH8Te7g==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1stdue5q5teskee057ced6rh9pzzr93xsy66w4sc3zu49rgxl7cjshztt45 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVK0VGZThqZTRBeWtvOGtt + aFdicnVUVTBUSlJsc2piTVJ5djBrbE83aFJBCjhOYmpuNG96eXl0dFBzSzQ5RVNT + WUdDVmVRZnE0NFRvdWF0ZGROS3JhMWsKLS0tIDQrUzlCS211cHlsa3ZTcEd0TEJn + Y2wwV2grQWdBdWQ4MjYwZzZiZlNkWEEKmzf6mc3qeYk7EOC35h9CdIBeV77W0Iv8 + 7b0wRSGW0JQIL0ffR+mWl2RGNM7ykSnwz1mic1HMjsrd/exhhMLjEg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2022-02-08T03:00:15Z" + mac: ENC[AES256_GCM,data:biFMSADG1RFArTPfx7kO/jF6hFmW68kQJ1+BYaBt1wMTwpknCwLC60ls625pEbCYDx40Wt8nTHc5Px/0iGkTfA261c15k1o/ya3IH9U2mSui/4tAxHwHps5OC7eK2Cg10YoEXT1vute7rNbXxerLg0Vn5sPK2PAYmC/zkvTdJNc=,iv:ac69kwcLigbnrwgD1L/DFhV23QmY40CucbXFXs3f8ss=,tag:Qck4/u8HfEPJ2SoZnMLvtA==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.1 diff --git a/readme.md b/readme.md index ad203ec..b7b0322 100644 --- a/readme.md +++ b/readme.md @@ -1,24 +1,64 @@ -# Provision Machines +# home-k8s-cluster -- My nodes are Arch Linux machines on bare metal +This repository contains the configuration, scripts, and other goodies for +building and managing my home cluster. I share the source with you so you can +make exactly the same mistakes as I do. + +## Setup + +Setup the pre-commit hooks before you change anything! + +``` +pip install pre-commit +pre-commit install --install-hooks +pre-commit autoupdate +``` + +## Provision Machines + +Before we interact with the cluster, we have some manual work to do. + +### Manual Preparation + +- Currently, my nodes are Arch Linux machines on bare metal - Nodes must be ready to be controlled via Ansible - - Have `python3` installed for Ansible - - Need to be `ssh`-able from a workstation - - You can grab keys like so: `curl -L files.lyte.dev/key.pub >> ~/.ssh/authorized_keys` - - TODO: script this? maybe custom ISO+PXEBoot? + - Have `python3` installed + - Need to be `ssh`-able from a controller (my workstation) + - `curl -L files.lyte.dev/key.pub >> ~/.ssh/authorized_keys` + +**TODO**: script this? maybe custom ISO+PXEBoot? Talos+Sidero? + +### Automated Provisioning + - Setup Ansible on the controller (from `./ansible`) - `ansible-galaxy install -r requirements.yml --force` - Verify Ansible can reach hosts (from `./ansible`) - `ansible all -i inventory/hosts.yml --list-hosts` - `ansible all -i inventory/hosts.yml -m ping` -- Use Ansible to prepare hosts for k3s installation (from `./ansible`) - - `ansible-playbook -i inventory/hosts.yml ./provision.yml` -- Use Ansible to install k3s as configured on all nodes (from `./ansible`) - - `ansible-playbook -i inventory/hosts.yml ./install-k3s.yml` - - You have to run this multiple times for the worker nodes to successfully - connect to the control plane nodes -- Setup your local kubectl to work with the new cluster - - `ansible -i ansible/inventory/hosts.yml $REMOTE_HOST -m fetch -a "src=/etc/rancher/k3s/k3s.yaml dest=./k3s-cluster-config.yaml flat=yes"` - - Copy the cluster information from the `./k3s-cluster-config.yaml` file into - your existing `~/.kube/config` (or just copy it there if it doesn't exist) - - You will need to edit the host from `localhost`/`127.0.0.1` to the correct host +- Use Ansible to build the cluster as configured on all nodes (from `./ansible`) + - `ansible-playbook -i inventory/hosts.yml ./build-k3s-cluster` + +And the cluster is up! If you want to interact with it from your controller, +you can do this: + +```bash +ansible -i ansible/inventory/hosts.yml $REMOTE_HOST -m fetch \ + -a "src=/etc/rancher/k3s/k3s.yaml dest=./k3s-cluster-config.kubeconfig.yaml flat=yes" +# TODO: this did not work for me +# env KUBECONFIG="~/.kube/config:./k3s-cluster-config.kubeconfig.yaml" \ +# kubectl config view --flatten | sed "s/127.0.0.1/$REMOTE_HOST/" > ~/.kube/new-config +``` + +- Copy the cluster information from the `./k3s-cluster-config.kubeconfig.yaml` file into + your existing `~/.kube/config` (or just copy it there if it doesn't exist) + - You will need to edit the host from `localhost`/`127.0.0.1` to the correct host + +### Automated Teardown + +```bash +ansible-playbook -i inventory/hosts.yml ./nuke-k3s-cluster +``` + +## Setting up Flux + +-