refactor(k3s): enhance cluster setup and enable ArgoCD apps

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

roles/docker_host/tasks/20_installation.yml

@@ -26,6 +26,7 @@
     - curl
     - gnupg
     - lsb-release
+    - qemu-guest-agent
   become: true
 
 - name: Add Docker apt key.

roles/docker_host/tasks/main.yml

@@ -1,6 +1,7 @@
 ---
 - name: Setup VM
   ansible.builtin.include_tasks: 10_setup.yml
+
 - name: Install docker
   ansible.builtin.include_tasks: 20_installation.yml
 

roles/k3s_agent/tasks/installation.yml

@@ -1,4 +1,12 @@
 ---
+- name: Install dependencies for apt to use repositories over HTTPS
+  ansible.builtin.apt:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - qemu-guest-agent
+  become: true
+
 - name: See if k3s file exists
   ansible.builtin.stat:
     path: /usr/local/bin/k3s

roles/k3s_loadbalancer/tasks/configuration.yml

@@ -9,8 +9,6 @@
   become: true
   notify:
     - Restart nginx
-  vars:
-    k3s_server_ips: "{{ k3s_primary_server_ip }}"
 
 - name: Enable nginx
   ansible.builtin.systemd:

roles/k3s_loadbalancer/tasks/installation.yml

@@ -4,6 +4,14 @@
     update_cache: true
   become: true
 
+- name: Install dependencies for apt to use repositories over HTTPS
+  ansible.builtin.apt:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - qemu-guest-agent
+  become: true
+
 - name: Install Nginx
   ansible.builtin.apt:
     name:

roles/k3s_loadbalancer/tasks/main.yml

@@ -1,6 +1,7 @@
 ---
 - name: Installation
   ansible.builtin.include_tasks: installation.yml
+
 - name: Configure
   ansible.builtin.include_tasks: configuration.yml
 

roles/k3s_loadbalancer/templates/nginx.conf.j2

@@ -3,11 +3,10 @@ include /etc/nginx/modules-enabled/*.conf;
 events {}
 
 stream {
-# TCP Load Balancing for the K3s API
   upstream k3s_servers {
-    {% for ip in k3s_server_ips %}
+{% for ip in k3s_server_ips %}
     server {{ ip }}:{{ k3s.loadbalancer.default_port }};
-    {% endfor %}
+{% endfor %}
   }
 
   server {
@@ -15,10 +14,22 @@ stream {
     proxy_pass k3s_servers;
   }
 
+  upstream etcd_servers {
+{% for ip in k3s_server_ips %}
+    server {{ ip }}:2379;
+{% endfor %}
+  }
+
+  server {
+    listen 2379;
+    proxy_pass etcd_servers;
+  }
+
+
   upstream dns_servers {
-    {% for ip in k3s_server_ips %}
+{% for ip in k3s_server_ips %}
     server {{ ip }}:53;
-    {% endfor %}
+{% endfor %}
   }
 
   server {

roles/k3s_server/tasks/installation.yml

@@ -1,26 +0,0 @@
----
-- name: Download K3s install script to /tmp/
-  ansible.builtin.get_url:
-    url: https://get.k3s.io
-    dest: /tmp/k3s_install.sh
-    mode: "0755"
-
-- name: Install K3s server with node taint and TLS SAN
-  when: (ansible_default_ipv4.address == k3s_primary_server_ip)
-  ansible.builtin.command: |
-    /tmp/k3s_install.sh server \
-    --node-taint CriticalAddonsOnly=true:NoExecute \
-    --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}
-    --tls-san {{ k3s_server_name }}
-  become: true
-  register: k3s_primary_install
-
-- name: Install K3s on the secondary servers
-  when: (ansible_default_ipv4.address != k3s_primary_server_ip)
-  ansible.builtin.command: |
-    /tmp/k3s_install.sh server \
-    --node-taint CriticalAddonsOnly=true:NoExecute \
-    --tls-san {{ k3s.loadbalancer.ip }}
-  environment:
-    K3S_TOKEN: "{{ k3s_token }}"
-  become: true

roles/k3s_server/tasks/main.yml

@@ -1,21 +1,29 @@
 ---
+- name: Install dependencies for apt to use repositories over HTTPS
+  ansible.builtin.apt:
+    name: "{{ item }}"
+    state: present
+    update_cache: true
+  loop:
+    - qemu-guest-agent
+  become: true
+
 - name: See if k3s file exists
   ansible.builtin.stat:
     path: /usr/local/bin/k3s
   register: k3s_status
 
-- include_tasks: installation.yml
-  when: not k3s_status.stat.exists
-
-- include_tasks: create_kubeconfig.yml
+- name: Install primary k3s server
+  include_tasks: primary_installation.yml
   when: ansible_default_ipv4.address == k3s_primary_server_ip
 
-- name: Check if k3s token vault file already exists
-  ansible.builtin.stat:
-    path: "{{ playbook_dir }}/{{ k3s_server_token_vault_file }}"
-  register: k3s_vault_file_stat
-  delegate_to: localhost
-  run_once: true
+- name: Get token from primary k3s server
+  include_tasks: pull_token.yml
 
-- include_tasks: pull_token.yml
-  when: not k3s_vault_file_stat.stat.exists
+- name: Install seconary k3s servers
+  include_tasks: secondary_installation.yml
+  when: ansible_default_ipv4.address != k3s_primary_server_ip
+
+- name: Set kubeconfig on localhost
+  include_tasks: create_kubeconfig.yml
+  when: ansible_default_ipv4.address == k3s_primary_server_ip

roles/k3s_server/tasks/primary_installation.yml

@@ -0,0 +1,14 @@
+---
+- name: Download K3s install script to /tmp/
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: /tmp/k3s_install.sh
+    mode: "0755"
+
+- name: Install K3s server with and TLS SAN
+  ansible.builtin.command: |
+    /tmp/k3s_install.sh server \
+    --cluster-init
+    --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }} \
+    --tls-san {{ k3s_server_name }}
+  become: true

roles/k3s_server/tasks/pull_token.yml

@@ -1,6 +1,5 @@
 - name: Get K3s token from the first server
-  when:
-    - ansible_default_ipv4.address == k3s_primary_server_ip
+  when: ansible_default_ipv4.address == k3s_primary_server_ip
   ansible.builtin.slurp:
     src: /var/lib/rancher/k3s/server/node-token
   register: k3s_token
@@ -9,6 +8,8 @@
 - name: Set fact on k3s_primary_server_ip
   ansible.builtin.set_fact:
     k3s_token: "{{ k3s_token['content'] | b64decode | trim }}"
+  when:
+    - ansible_default_ipv4.address == k3s_primary_server_ip
 
 - name: Write K3s token to local file for encryption
   ansible.builtin.copy:
@@ -22,3 +23,4 @@
 - name: Encrypt k3s token
   ansible.builtin.shell: cd ../; ansible-vault encrypt "{{ playbook_dir }}/{{k3s_server_token_vault_file}}"
   delegate_to: localhost
+  run_once: true

roles/k3s_server/tasks/secondary_installation.yml

@@ -0,0 +1,21 @@
+---
+- name: Add token vault
+  ansible.builtin.include_vars:
+    file: "{{ playbook_dir }}/{{ k3s_server_token_vault_file }}"
+    name: k3s_token_vault
+
+- name: Download K3s install script to /tmp/
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: /tmp/k3s_install.sh
+    mode: "0755"
+
+- name: Install K3s on the secondary servers
+  ansible.builtin.command: |
+    /tmp/k3s_install.sh \
+    --server "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}" \
+    --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }} \
+    --tls-san {{ k3s_server_name }}
+  environment:
+    K3S_TOKEN: "{{ k3s_token_vault.k3s_token }}"
+  become: true

roles/kubernetes_cert_manager/tasks/main.yml

@@ -19,13 +19,6 @@
     name: cert-manager-webhook-netcup
     repo_url: https://aellwein.github.io/cert-manager-webhook-netcup/charts/
 
-- name: Install NetCup Webhook
-  kubernetes.core.helm:
-    name: my-cert-manager-webhook-netcup
-    chart_ref: cert-manager-webhook-netcup/cert-manager-webhook-netcup
-    release_namespace: cert-manager
-    create_namespace: true
-
 - name: Download cert-manager manifest
   ansible.builtin.get_url:
     url: "{{ cert_manager_manifest }}"
@@ -75,3 +68,10 @@
   tags:
     - cert_manager
     - certificate
+
+- name: Install NetCup Webhook
+  kubernetes.core.helm:
+    name: my-cert-manager-webhook-netcup
+    chart_ref: cert-manager-webhook-netcup/cert-manager-webhook-netcup
+    release_namespace: cert-manager
+    create_namespace: true

roles/kubernets_argo_apps/defaults/main.yml

@@ -0,0 +1,2 @@
+argocd_apps_repo_url: ssh://git@git.tudattr.dev/tudattr/argocd.git
+argocd_apps_target_revision: main

roles/kubernets_argo_apps/tasks/install_argo_app.yml

@@ -0,0 +1,10 @@
+- name: Render Argo CD Application YAML to a variable
+  ansible.builtin.set_fact:
+    argo_app_manifest: "{{ lookup('ansible.builtin.template', '../templates/argo_app.yaml.j2') }}"
+
+- name: Apply Argo CD Application to Kubernetes using k8s module
+  kubernetes.core.k8s:
+    state: present
+    definition: "{{ argo_app_manifest }}"
+  register: k8s_apply_result
+  delegate_to: localhost

roles/kubernets_argo_apps/tasks/main.yml

@@ -0,0 +1,5 @@
+- name: Install Argo Application
+  ansible.builtin.include_tasks: ./install_argo_app.yml
+  loop: argo_apps
+  loop_control:
+    loop_var: app

roles/kubernets_argo_apps/templates/argo-app.yaml.j2

@@ -0,0 +1,24 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ app.name }}
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: {{ argocd_apps_repo_url }}
+    targetRevision: {{ argocd_apps_target_revision | default("HEAD") }}
+    path: argocd/{{ app.name }}
+    directory:
+      recurse: true
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: {{ argocd_apps_target_namespace | default(app.name) }}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

roles/kubernets_argo_apps/templates/argo_repo.yaml.j2

@@ -0,0 +1,11 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Repository
+metadata:
+  name: {{ argocd_apps_repo_name }}
+  namespace: argocd
+spec:
+  url: {{ argocd_apps_repo_url }}
+  type: git
+  sshPrivateKeySecret:
+    name: {{ argocd_apps_ssh_private_key_secret_name }}
+    key: {{ argocd_apps_ssh_private_key_secret_key }}

roles/proxmox/vars/main.yml

@@ -19,8 +19,6 @@ proxmox_tags:
   - "{{ proxmox_creator }}"
 
 proxmox_node_dependencies:
-  - libguestfs-tools
-  - qemu-guest-agent
   - nmap
 
 proxmox_localhost_dependencies: []

roles/reverse_proxy/tasks/20_xcaddy_install.yml

@@ -25,7 +25,7 @@
   become: true
 
 - name: Build Custom Caddy with netcup
-  ansible.builtin.command: xcaddy build --with github.com/caddy-dns/netcup {{ reverse_proxy_caddy_version}}
+  ansible.builtin.command: xcaddy build --with github.com/caddy-dns/netcup
   environment:
     PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
   register: xcaddy_build

roles/reverse_proxy/templates/Caddyfile.j2

@@ -1,6 +1,14 @@
 {
-    email {{ caddy.admin_email | default('admin@example.com') }}
-    acme_ca {{ caddy.acme_ca | default('https://acme-v02.api.letsencrypt.org/directory') }}
+	acme_dns netcup {
+		customer_number {{ netcup_customer_id }}
+		api_key {{ netcup_api_key }}
+		api_password {{ netcup_api_password }}
+	}
+  email {{ caddy.admin_email | default('admin@example.com') }}
+  acme_ca {{ caddy.acme_ca | default('https://acme-v02.api.letsencrypt.org/directory') }}
+}
+
+*.{{ domain }} {
 }
 
 {% for service in services %}
@@ -16,17 +24,6 @@
       output file /var/log/caddy/{{ service.name }}.log
       format json
   }
-
-	tls {
-		dns netcup {
-			customer_number {{ vault_netcup.customer_number }}
-			api_key {{ vault_netcup.api_key }}
-			api_password {{ vault_netcup.api_password }}
-		}
-    propagation_timeout 900s
-		propagation_delay 600s
-		resolvers 1.1.1.1
-	}
 }
 
 {% endif %}

roles/reverse_proxy/vars/main.yml

@@ -1,4 +1,4 @@
-reverse_proxy_caddy_version: v2.9.1
+reverse_proxy_caddy_version: v1.0.0
 
 reverse_proxy_custom_caddy_source_path: "{{ ansible_env.HOME }}/caddy"
 reverse_proxy_default_caddy_path: "/usr/bin/caddy"