refactor!: remove old Ansible setup

Replaced by NixOS.
This commit is contained in:
Khue Doan
2025-10-03 16:12:08 +07:00
parent 3294e7adb4
commit cc9359451b
42 changed files with 8 additions and 650 deletions

View File

@@ -1,25 +0,0 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.3.0
hooks:
- id: check-added-large-files
- id: check-executables-have-shebangs
- id: check-merge-conflict
- id: check-shebang-scripts-are-executable
- id: detect-private-key
- id: end-of-file-fixer
- id: mixed-line-ending
- id: trailing-whitespace
- repo: https://github.com/adrienverge/yamllint
rev: v1.27.1
hooks:
- id: yamllint
- repo: https://github.com/gruntwork-io/pre-commit
rev: v0.1.24
hooks:
- id: helmlint
- id: shellcheck
- repo: https://github.com/tofuutils/pre-commit-opentofu
rev: v2.1.0
hooks:
- id: tofu_fmt

View File

@@ -1,9 +0,0 @@
ignore: |
templates/
mkdocs.yml
extends: default
rules:
document-start: disable
line-length: disable

View File

@@ -44,9 +44,6 @@ restore:
test:
make -C test
clean:
docker compose --project-directory ./metal/roles/pxe_server/files down
docs:
mkdocs serve

View File

@@ -19,6 +19,10 @@ They are not permanent, we can change them in the future if better alternatives
- CHANGEME
## Switching from Fedora and Ansible to NixOS
TODO
## Remove the Docker wrapper for Nix shell
**Context**

View File

@@ -27,48 +27,17 @@
{
devShells.default = pkgs.mkShell {
packages = with pkgs; [
ansible
ansible-lint
bmake
diffutils
docker
docker-compose
dyff
git
glibcLocales
gnumake
go
gotestsum
iproute2
jq
k9s
kanidm
kube3d
kubectl
kubernetes-helm
kustomize
libisoburn
neovim
nixfmt-tree
nixos-anywhere
nixos-rebuild
openssh
opentofu # Drop-in replacement for Terraform
p7zip
pre-commit
qrencode
shellcheck
wireguard-tools
yamllint
(python3.withPackages (
p: with p; [
jinja2
kubernetes
mkdocs-material
netaddr
pexpect
rich
]
))
opentofu
];
};
}

View File

@@ -1,23 +0,0 @@
.POSIX:
env ?= prod
export KUBECONFIG = $(shell pwd)/kubeconfig.yaml
default: boot cluster
~/.ssh/id_ed25519:
ssh-keygen -t ed25519 -P '' -f "$@"
boot: ~/.ssh/id_ed25519
ansible-playbook \
--inventory inventories/${env}.yml \
boot.yml
cluster:
ansible-playbook \
--inventory inventories/${env}.yml \
cluster.yml
console:
ansible-console \
--inventory inventories/${env}.yml

View File

@@ -1,5 +0,0 @@
[defaults]
host_key_checking=false
stdout_callback=debug
stderr_callback=debug
force_color=true

View File

@@ -1,10 +0,0 @@
- name: Start PXE server
hosts: localhost
roles:
- pxe_server
- name: Provision bare metal machines
hosts: metal
gather_facts: false
roles:
- wake

View File

@@ -1,11 +0,0 @@
- name: Create Kubernetes cluster
hosts: metal
roles:
- prerequisites
- k3s
- automatic_upgrade
- name: Install Kubernetes addons
hosts: localhost
roles:
- cilium

View File

@@ -13,6 +13,7 @@
disko.nixosModules.disko
./configuration.nix
{
# nix eval --raw .#nixosConfigurations.metal1.config.networking.hostName
networking.hostName = "metal1";
}
];

View File

@@ -1,4 +0,0 @@
ansible_user: root
ansible_ssh_private_key_file: ~/.ssh/id_ed25519
ssh_public_key: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
dns_server: "8.8.8.8"

View File

@@ -1,15 +0,0 @@
all:
vars:
control_plane_endpoint: 192.168.1.100
load_balancer_ip_pool:
- 192.168.1.224/27
metal:
children:
masters:
hosts:
metal0: {ansible_host: 192.168.1.110, mac: '00:23:24:d1:f5:69', disk: sda, network_interface: eno1}
metal1: {ansible_host: 192.168.1.111, mac: '00:23:24:d1:f3:f0', disk: sda, network_interface: eno1}
metal2: {ansible_host: 192.168.1.112, mac: '00:23:24:e7:04:60', disk: sda, network_interface: eno1}
workers:
hosts:
metal3: {ansible_host: 192.168.1.113, mac: '00:23:24:d1:f4:d6', disk: sda, network_interface: eno1}

View File

@@ -1,7 +0,0 @@
metal:
children:
masters:
hosts:
proxmox0: {ansible_host: 192.168.1.169, mac: 'c2:f5:cf:1f:3e:c0', disk: sda, network_interface: ens18}
workers:
hosts: {}

View File

@@ -1,23 +0,0 @@
apiVersion: k3d.io/v1alpha4
kind: Simple
metadata:
name: homelab-dev
image: docker.io/rancher/k3s:v1.27.1-k3s1
servers: 1
agents: 0
options:
k3s:
extraArgs:
- arg: --disable=traefik
nodeFilters:
- server:*
- arg: --disable-cloud-controller
nodeFilters:
- server:*
ports:
- port: 80:80
nodeFilters:
- loadbalancer
- port: 443:443
nodeFilters:
- loadbalancer

View File

@@ -1,3 +0,0 @@
[commands]
upgrade_type = default
apply_updates = yes

View File

@@ -1,17 +0,0 @@
- name: Install packages for automatic upgrade
ansible.builtin.dnf:
name:
- dnf-automatic
- dnf-utils
- name: Copy automatic upgrade config file
ansible.builtin.copy:
src: automatic.conf
dest: /etc/dnf/automatic.conf
mode: 0644
- name: Enable automatic upgrade service
ansible.builtin.systemd:
name: dnf-automatic.timer
state: started
enabled: true

View File

@@ -1,19 +0,0 @@
cilium_repo_url: https://helm.cilium.io
cilium_version: 1.16.1
cilium_namespace: kube-system
cilium_values:
operator:
replicas: 1
kubeProxyReplacement: true
l2announcements:
enabled: true
# TODO the host and port are k3s-specific, generic solution is in progress
# https://github.com/cilium/cilium/issues/19038
# https://github.com/cilium/cilium/pull/28741
k8sServiceHost: 127.0.0.1
k8sServicePort: 6444
hubble:
relay:
enabled: true
ui:
enabled: true

View File

@@ -1,27 +0,0 @@
- name: Install Cilium
kubernetes.core.helm:
name: cilium
chart_ref: cilium
chart_repo_url: "{{ cilium_repo_url }}"
chart_version: "{{ cilium_version }}"
release_namespace: "{{ cilium_namespace }}"
values: "{{ cilium_values }}"
- name: Wait for Cilium CRDs
kubernetes.core.k8s_info:
kind: CustomResourceDefinition
name: "{{ item }}"
loop:
- ciliuml2announcementpolicies.cilium.io
- ciliumloadbalancerippools.cilium.io
register: crd
until: crd.resources | length > 0
retries: 5
delay: 10
- name: Apply Cilium resources
kubernetes.core.k8s:
template: "{{ item }}"
loop:
- ciliuml2announcementpolicy.yaml
- ciliumloadbalancerippool.yaml

View File

@@ -1,7 +0,0 @@
apiVersion: cilium.io/v2alpha1
kind: CiliumL2AnnouncementPolicy
metadata:
name: default
spec:
externalIPs: true
loadBalancerIPs: true

View File

@@ -1,9 +0,0 @@
apiVersion: cilium.io/v2alpha1
kind: CiliumLoadBalancerIPPool
metadata:
name: default
spec:
blocks:
{% for cidr in load_balancer_ip_pool %}
- cidr: {{ cidr }}
{% endfor %}

View File

@@ -1,18 +0,0 @@
k3s_version: v1.30.4+k3s1
k3s_config_file: /etc/rancher/k3s/config.yaml
k3s_token_file: /etc/rancher/node/password
k3s_service_file: /etc/systemd/system/k3s.service
k3s_data_dir: /var/lib/rancher/k3s
k3s_kubeconfig_file: /etc/rancher/k3s/k3s.yaml
k3s_server_config:
tls-san:
- "{{ control_plane_endpoint }}"
disable:
- local-storage
- servicelb
- traefik
disable-helm-controller: true
disable-kube-proxy: true
disable-network-policy: true
flannel-backend: none
secrets-encryption: true

View File

@@ -1,2 +0,0 @@
*
!.gitignore

View File

@@ -1,97 +0,0 @@
- name: Download k3s binary
ansible.builtin.get_url:
url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s
checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt
dest: "{{ role_path }}/files/bin/k3s"
mode: 0755
delegate_to: localhost
run_once: true
register: k3s_binary
- name: Copy k3s binary to nodes
ansible.builtin.copy:
src: bin/k3s
dest: /usr/local/bin/k3s
owner: root
group: root
mode: 0755
- name: Ensure config directories exist
ansible.builtin.file:
path: "{{ item }}"
state: directory
mode: 0755
loop:
- /etc/rancher/k3s
- /etc/rancher/node
- "{{ k3s_data_dir }}/agent/pod-manifests"
- name: Check if k3s token file exists on the first node
run_once: true
ansible.builtin.stat:
path: "{{ k3s_token_file }}"
register: k3s_token_file_stat
- name: Generate k3s token file on the first node if not exist yet
run_once: true
when: not k3s_token_file_stat.stat.exists
ansible.builtin.copy:
content: "{{ lookup('community.general.random_string', length=32) }}"
dest: "{{ k3s_token_file }}"
mode: 0600
- name: Get k3s token from the first node
run_once: true
ansible.builtin.slurp:
src: "{{ k3s_token_file }}"
register: k3s_token_base64
- name: Ensure all nodes has the same token
ansible.builtin.copy:
content: "{{ k3s_token_base64.content | b64decode }}"
dest: "{{ k3s_token_file }}"
mode: 0600
- name: Copy k3s config files
ansible.builtin.template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: 0644
loop:
- src: config.yaml.j2
dest: "{{ k3s_config_file }}"
- src: k3s.service.j2
dest: "{{ k3s_service_file }}"
- name: Copy kube-vip manifests
when: "'masters' in group_names"
ansible.builtin.template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: 0644
loop:
- src: kube-vip.yaml.j2
dest: "{{ k3s_data_dir }}/agent/pod-manifests/kube-vip.yaml"
- name: Enable k3s service
ansible.builtin.systemd:
name: k3s
enabled: true
state: started
register: k3s_service
until: k3s_service is succeeded
retries: 5
- name: Get Kubernetes config file
run_once: true
ansible.builtin.slurp:
src: "{{ k3s_kubeconfig_file }}"
register: kubeconfig_base64
- name: Write Kubernetes config file with the correct cluster address
ansible.builtin.copy:
content: "{{ kubeconfig_base64.content | b64decode | replace('127.0.0.1', control_plane_endpoint) }}"
dest: "{{ playbook_dir }}/kubeconfig.yaml"
mode: 0600
delegate_to: localhost
run_once: true

View File

@@ -1,10 +0,0 @@
{% if inventory_hostname == groups['masters'][0] %}
cluster-init: true
{% else %}
server: https://{{ control_plane_endpoint }}:6443
{% endif %}
token-file: {{ k3s_token_file }}
{% if 'masters' in group_names %}
{{ k3s_server_config | to_nice_yaml }}
{% endif %}
snapshotter: stargz

View File

@@ -1,24 +0,0 @@
[Unit]
Description=Lightweight Kubernetes
Documentation=https://k3s.io
After=network-online.target
[Service]
Type=notify
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/k3s {{ 'server' if 'masters' in group_names else 'agent' }}
KillMode=process
Delegate=yes
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
TimeoutStartSec=0
Restart=always
RestartSec=5s
[Install]
WantedBy=multi-user.target

View File

@@ -1,39 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
spec:
containers:
- name: kube-vip
image: ghcr.io/kube-vip/kube-vip:v0.6.4
args:
- manager
env:
- name: address
value: {{ control_plane_endpoint }}
- name: vip_arp
value: "true"
- name: cp_enable
value: "true"
- name: vip_leaderelection
value: "true"
- name: lb_enable
value: "true"
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: {{ k3s_kubeconfig_file }}
name: kubeconfig

View File

@@ -1,8 +0,0 @@
- name: Adjust kernel parameters
ansible.posix.sysctl:
name: "{{ item.name }}"
value: "{{ item.value }}"
loop:
- {name: "fs.inotify.max_queued_events", value: 16384}
- {name: "fs.inotify.max_user_instances", value: 8192}
- {name: "fs.inotify.max_user_watches", value: 524288}

View File

@@ -1,4 +0,0 @@
iso_url: "https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/39/Server/x86_64/iso/Fedora-Server-dvd-x86_64-39-1.5.iso"
iso_checksum: "sha256:2755cdff6ac6365c75be60334bf1935ade838fc18de53d4c640a13d3e904f6e9"
timezone: Asia/Ho_Chi_Minh
dhcp_proxy: true

View File

@@ -1,2 +0,0 @@
*
!.gitignore

View File

@@ -1,2 +0,0 @@
*
!.gitignore

View File

@@ -1,2 +0,0 @@
*
!.gitignore

View File

@@ -1,2 +0,0 @@
*
!.gitignore

View File

@@ -1,5 +0,0 @@
FROM alpine:3.19
RUN apk --no-cache add dnsmasq
ENTRYPOINT ["dnsmasq", "-k"]

View File

@@ -1,20 +0,0 @@
services:
dnsmasq:
build: ./dnsmasq
volumes:
- ./data/pxe-config/dnsmasq.conf:/etc/dnsmasq.conf
- ./data/pxe-config/grub.cfg:/tftp/grub.cfg
- ./data/os/EFI/BOOT/grubx64.efi:/tftp/grubx64.efi
- ./data/os/images/pxeboot/initrd.img:/tftp/initrd.img
- ./data/os/images/pxeboot/vmlinuz:/tftp/vmlinuz
network_mode: host
cap_add:
- NET_ADMIN
http:
build: ./http
network_mode: host
volumes:
- ./data/os:/usr/share/nginx/html/os
- ./data/init-config/:/usr/share/nginx/html/init-config
environment:
NGINX_PORT: 80

View File

@@ -1 +0,0 @@
FROM nginx:1.25-alpine

View File

@@ -1,47 +0,0 @@
- name: Get Docker info
docker_host_info: {}
register: docker_info_result
- name: Ensure Docker is running on a supported operating system
fail:
msg: Docker host networking driver only works on Linux hosts, and is not supported on Docker Desktop for Mac or Windows (you can use a Linux VM with bridged networking instead)
when:
- docker_info_result.host_info.OperatingSystem == "Docker Desktop"
- name: Download boot image
ansible.builtin.get_url:
url: "{{ iso_url }}"
dest: "{{ role_path }}/files/data/iso/{{ iso_url | basename }}"
checksum: "{{ iso_checksum }}"
mode: 0644
register: iso
- name: Extract boot image
ansible.builtin.command:
cmd: "xorriso -osirrox on -indev {{ iso.dest }} -extract / {{ role_path }}/files/data/os"
creates: "{{ role_path }}/files/data/os/.treeinfo"
- name: Generate dnsmasq config
ansible.builtin.template:
src: dnsmasq.conf.j2
dest: "{{ role_path }}/files/data/pxe-config/dnsmasq.conf"
mode: 0644
- name: Generate GRUB config
ansible.builtin.template:
src: grub.cfg.j2
dest: "{{ role_path }}/files/data/pxe-config/grub.cfg"
mode: 0644
- name: Generate init config for each machine
ansible.builtin.template:
src: kickstart.ks.j2
dest: "{{ role_path }}/files/data/init-config/{{ hostvars[item]['mac'] }}.ks"
mode: 0644
loop: "{{ groups['metal'] }}"
- name: Start the ephemeral PXE server
community.docker.docker_compose_v2:
project_src: "{{ role_path }}/files"
state: present
build: always

View File

@@ -1,26 +0,0 @@
# Disable DNS Server.
port=0
{% if dhcp_proxy == true %}
# We're DHCP proxying on the network of the homelab host
dhcp-range={{ ansible_default_ipv4.address }},proxy
pxe-service=X86-64_EFI, "Boot From Network, (UEFI)", grubx64.efi
{% else %}
# We're DHCP configuring on this range
dhcp-range={{ ansible_default_ipv4.network | ansible.netcommon.ipmath(1) }},{{ ansible_default_ipv4.broadcast | ansible.netcommon.ipmath(-1) }},{{ ansible_default_ipv4.netmask }},12h
dhcp-option=3,{{ ansible_default_ipv4.gateway }}
# Match Arch Types efi x86 and x64
dhcp-match=set:efi-x86_64,option:client-arch,7
dhcp-match=set:efi-x86_64,option:client-arch,9
# Set the Boot file based on the tag from above
dhcp-boot=tag:efi-x86_64,grubx64.efi
{% endif %}
# Log DHCP queries to stdout
log-queries
log-dhcp
log-facility=-
# Enable TFTP server
enable-tftp
tftp-root=/tftp

View File

@@ -1,8 +0,0 @@
set timeout=1
menuentry '{{ iso_url | basename | splitext | first }} (PXE)' {
linux vmlinuz \
ip=dhcp \
inst.ks=http://{{ ansible_default_ipv4.address }}/init-config/${net_default_mac}.ks
initrd initrd.img
}

View File

@@ -1,67 +0,0 @@
#version=RHEL8
# Do not use graphical install
text
# Keyboard layouts
keyboard --xlayouts='us'
# System language
lang en_US.UTF-8
# Partition clearing information
clearpart --all --drives={{ hostvars[item]['disk'] }}
# Partitioning
ignoredisk --only-use={{ hostvars[item]['disk'] }}
partition /boot/efi --fstype=vfat --size=512
partition / --fstype=ext4 --size=65536
# Network information
network --bootproto=static --device={{ hostvars[item]['network_interface'] }} --ip={{ hostvars[item]['ansible_host'] }} --gateway={{ ansible_default_ipv4.gateway }} --nameserver={{ dns_server }} --netmask={{ ansible_default_ipv4.netmask }} --ipv6=auto --hostname={{ hostvars[item]['inventory_hostname'] }} --activate
# Use network installation
repo --name="Repository" --baseurl=http://{{ ansible_default_ipv4.address }}/os
url --url="http://{{ ansible_default_ipv4.address }}/os"
# Disable Setup Agent on first boot
firstboot --disable
# Do not configure the X Window System
skipx
# Enable NTP
services --enabled="chronyd"
# System timezone
timezone {{ timezone }} --utc
# Create user (locked by default)
user --groups=wheel --name=admin
# Add SSH key
sshkey --username=root "{{ ssh_public_key }}"
# Disable root password login
rootpw --lock
# Disable SELinux
selinux --disabled
# Disable firewall
firewall --disabled
%packages
@^custom-environment
openssh-server
%end
# Create a raw partition for Ceph using the remaining space
# Using a post script because there is no built-in feature in Kickstart
# The three empty lines are equivalent to pressing Enter to use the default values for:
# - Partition number
# - First sector
# - Last sector
%post
fdisk /dev/{{ hostvars[item]['disk'] }} << EOF
new
write
EOF
%end
reboot

View File

@@ -1,8 +0,0 @@
- name: Send Wake-on-LAN magic packets
community.general.wakeonlan:
mac: "{{ mac }}"
delegate_to: localhost
- name: Wait for the machines to come online
ansible.builtin.wait_for_connection:
timeout: 600

View File

@@ -1,7 +0,0 @@
#!/bin/sh
docker compose \
--project-directory ./metal/roles/pxe_server/files/ \
logs \
--follow \
"${@}"