feat(kubernetes): stable kubernetes with argo

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

playbooks/kubernetes_setup.yml

@@ -0,0 +1,16 @@
+---
+- name: Setup Kubernetes Cluster
+  hosts: kubernetes
+  any_errors_fatal: true
+  gather_facts: false
+  vars:
+    is_localhost: "{{ inventory_hostname == '127.0.0.1' }}"
+  roles:
+    - role: kubernetes_traefik
+      when: is_localhost
+    - role: kubernetes_argocd
+      when: is_localhost
+    - role: kubernetes_metallb
+      when: is_localhost
+    - role: kubernetes_cert_manager
+      when: is_localhost

playbooks/traefik.yml

@@ -0,0 +1,8 @@
+---
+- name: "Enable traefik dashboard"
+  hosts: k3s_servers[0]
+  become: true
+  roles:
+    - role: kubernetes_traefik
+      vars:
+        traefik_dashboard_hostname: "{{ traefik_dashboard_hostname }}"

requirements.txt

@@ -1,7 +1,20 @@
+cachetools==5.5.2
 certifi==2025.1.31
 charset-normalizer==3.4.1
+durationpy==0.10
+google-auth==2.40.3
 idna==3.10
+kubernetes==33.1.0
 nc-dnsapi==0.1.3
+oauthlib==3.3.1
 proxmoxer==2.2.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+python-dateutil==2.9.0.post0
+PyYAML==6.0.2
 requests==2.32.3
+requests-oauthlib==2.0.0
+rsa==4.9.1
+six==1.17.0
 urllib3==2.3.0
+websocket-client==1.8.0

roles/k3s_loadbalancer/tasks/main.yml

@@ -6,9 +6,9 @@
 
 - name: Setup DNS on Netcup
   community.general.netcup_dns:
-    api_key: "{{ k3s_loadbalancer_netcup_api_key }}"
-    api_password: "{{ k3s_loadbalancer_netcup_api_password }}"
-    customer_id: "{{ k3s_loadbalancer_netcup_customer_id }}"
+    api_key: "{{ netcup_api_key }}"
+    api_password: "{{ netcup_api_password }}"
+    customer_id: "{{ netcup_customer_id }}"
     domain: "{{ domain }}"
     name: "k3s"
     type: "A"

roles/k3s_loadbalancer/vars/main.yml

@@ -1,7 +1,3 @@
 k3s_loadbalancer_nginx_config_path: "/etc/nginx/nginx.conf"
 
-k3s_loadbalancer_netcup_api_key: "{{ netcup_api_key }}"
-k3s_loadbalancer_netcup_api_password: "{{ netcup_api_password }}"
-k3s_loadbalancer_netcup_customer_id: "{{ netcup_customer_id }}"
-
 domain: "{{ internal_domain }}"

roles/kubernetes_argocd/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+argocd_version: stable
+argocd_namespace: argocd
+argocd_repo: "https://raw.githubusercontent.com/argoproj/argo-cd/refs/tags/{{ argocd_version }}/manifests/ha/install.yaml"

roles/kubernetes_argocd/files/argocd-cmd-params-cm.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-cmd-params-cm
+  labels:
+    app.kubernetes.io/name: argocd-cmd-params-cm
+    app.kubernetes.io/part-of: argocd
+data:
+  redis.server: "argocd-redis-ha-haproxy:6379"
+  server.insecure: "true"

roles/kubernetes_argocd/tasks/main.yml

@@ -0,0 +1,50 @@
+---
+- name: Install ArgoCD
+  block:
+    - name: Create ArgoCD namespace
+      kubernetes.core.k8s:
+        name: "{{ argocd_namespace }}"
+        api_version: v1
+        kind: Namespace
+        state: present
+
+    - name: Apply ArgoCD manifests
+      kubernetes.core.k8s:
+        src: "{{ argocd_repo }}"
+        state: present
+        namespace: "{{ argocd_namespace }}"
+      register: apply_manifests
+      until: apply_manifests is not failed
+      retries: 5
+      delay: 10
+
+    - name: Wait for ArgoCD server to be ready
+      kubernetes.core.k8s_info:
+        api_version: apps/v1
+        kind: Deployment
+        name: argocd-server
+        namespace: "{{ argocd_namespace }}"
+      register: rollout_status
+      until: rollout_status.resources[0].status.readyReplicas is defined and rollout_status.resources[0].status.readyReplicas == rollout_status.resources[0].spec.replicas
+      retries: 30
+      delay: 10
+
+- name: Apply ArgoCD Ingress
+  kubernetes.core.k8s:
+    definition: "{{ lookup('ansible.builtin.template', 'ingress.yml.j2') | from_yaml }}"
+    state: present
+    namespace: "{{ argocd_namespace }}"
+  register: apply_manifests
+  until: apply_manifests is not failed
+  retries: 5
+  delay: 10
+
+- name: Apply ArgoCD CM
+  kubernetes.core.k8s:
+    src: "files/argocd-cmd-params-cm.yaml"
+    state: present
+    namespace: "{{ argocd_namespace }}"
+  register: apply_manifests
+  until: apply_manifests is not failed
+  retries: 5
+  delay: 10

roles/kubernetes_argocd/templates/app.yaml.j2

@@ -0,0 +1,19 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: "{{ item.name }}"
+  namespace: "{{ argocd_namespace }}"
+spec:
+  project: default
+  source:
+    repoURL: "{{ item.repo_url }}"
+    targetRevision: "{{ item.target_revision }}"
+    path: "{{ item.path }}"
+  destination:
+    server: "{{ item.destination_server }}"
+    namespace: "{{ item.destination_namespace }}"
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

roles/kubernetes_argocd/templates/ingress.yml.j2

@@ -0,0 +1,27 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: argocd-ingress
+  namespace: argocd
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    cert-manager.io/cluster-issuer: "{{ argocd_cert_resolver }}"
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  rules:
+    - host: {{ argocd_hostname }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: argocd-server
+                port:
+                  number: 80
+  tls:
+    - hosts:
+      - {{ argocd_hostname }}
+      secretName: k3s-seyshiro-de-tls

roles/kubernetes_cert_manager/defaults/main.yml

@@ -0,0 +1,5 @@
+cert_manager_version: "v1.18.2"
+cert_manager_email: "mail@example.com"
+cert_manager_manifest: "https://github.com/cert-manager/cert-manager/releases/download/{{ cert_manager_version }}/cert-manager.yaml"
+cert_manager_issuer_name: "letsencrypt-prod"
+cert_manager_issuer_env: "staging"

roles/kubernetes_cert_manager/tasks/main.yml

@@ -0,0 +1,77 @@
+---
+- name: Ensure cert-manager namespace exists
+  kubernetes.core.k8s:
+    name: cert-manager
+    api_version: v1
+    kind: Namespace
+    state: present
+  tags:
+    - cert_manager
+    - namespace
+
+- name: Create netcup-secret
+  kubernetes.core.k8s:
+    namespace: cert-manager
+    definition: "{{ lookup('ansible.builtin.template', 'netcup.yml.j2') | from_yaml }}"
+
+- name: Add a repository
+  kubernetes.core.helm_repository:
+    name: cert-manager-webhook-netcup
+    repo_url: https://aellwein.github.io/cert-manager-webhook-netcup/charts/
+
+- name: Install NetCup Webhook
+  kubernetes.core.helm:
+    name: my-cert-manager-webhook-netcup
+    chart_ref: cert-manager-webhook-netcup/cert-manager-webhook-netcup
+    release_namespace: cert-manager
+    create_namespace: true
+
+- name: Download cert-manager manifest
+  ansible.builtin.get_url:
+    url: "{{ cert_manager_manifest }}"
+    dest: "/tmp/cert-manager.yaml"
+    mode: "0644"
+    validate_certs: true
+  tags:
+    - cert_manager
+    - download
+
+- name: Apply cert-manager core manifests
+  kubernetes.core.k8s:
+    src: "/tmp/cert-manager.yaml"
+    state: present
+  tags:
+    - cert_manager
+    - apply_manifest
+
+- name: Wait for cert-manager deployments to be ready
+  kubernetes.core.k8s_info:
+    api_version: apps/v1
+    kind: Deployment
+    namespace: cert-manager
+    name: "{{ item }}"
+    wait: true
+    wait_timeout: 300
+  loop:
+    - cert-manager
+    - cert-manager-cainjector
+    - cert-manager-webhook
+  tags:
+    - cert_manager
+    - wait_ready
+
+- name: Create Let's Encrypt ClusterIssuer
+  kubernetes.core.k8s:
+    state: present
+    definition: "{{ lookup('ansible.builtin.template', 'clusterissuer.yml.j2') | from_yaml }}"
+  tags:
+    - cert_manager
+    - cluster_issuer
+
+- name: Create Let's Encrypt Certificate
+  kubernetes.core.k8s:
+    state: present
+    definition: "{{ lookup('ansible.builtin.template', 'certificate.yml.j2') | from_yaml }}"
+  tags:
+    - cert_manager
+    - certificate

roles/kubernetes_cert_manager/templates/certificate.yml.j2

@@ -0,0 +1,16 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: k3s-seyshiro-de
+  namespace: cert-manager
+spec:
+  secretName: k3s-seyshiro-de-tls
+  issuerRef:
+    name: {{ cert_manager_issuer_name }}
+    kind: ClusterIssuer
+  commonName: "*.k3s.seyshiro.de"
+  dnsNames:
+    - "k3s.seyshiro.de"
+    - "*.k3s.seyshiro.de"
+

roles/kubernetes_cert_manager/templates/clusterissuer.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: "{{ cert_manager_issuer_name }}"
+spec:
+  acme:
+    server: "{% if cert_manager_issuer_env == 'production' %}https://acme-v02.api.letsencrypt.org/directory{% else %}https://acme-staging-v02.api.letsencrypt.org/directory{% endif %}"
+    email: "{{ cert_manager_email }}"
+    privateKeySecretRef:
+      name: "{{ cert_manager_issuer_name }}-account-key"
+    solvers:
+    - selector:
+        dnsZones:
+          - 'k3s.seyshiro.de'
+      dns01:
+        webhook:
+            groupName: com.netcup.webhook
+            solverName: netcup
+            config:
+                secretRef: netcup-secret
+                secretNamespace: cert-manager

roles/kubernetes_cert_manager/templates/netcup.yml.j2

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: netcup-secret
+type: Opaque
+data:
+  customer-number: {{ netcup_customer_id | b64encode }}
+  api-key: {{ netcup_api_key |b64encode }}
+  api-password: {{ netcup_api_password | b64encode }}
+

roles/kubernetes_metallb/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+metallb_version: v0.15.2
+metallb_ip_range: "192.168.178.200-192.168.178.220"
+metallb_manifest_url: "https://raw.githubusercontent.com/metallb/metallb/{{ metallb_version }}/config/manifests/metallb-native.yaml"

roles/kubernetes_metallb/tasks/main.yml

@@ -0,0 +1,62 @@
+---
+- name: Ensure metallb-system namespace exists
+  kubernetes.core.k8s:
+    name: metallb-system
+    api_version: v1
+    kind: Namespace
+    state: present
+  tags:
+    - metallb
+    - namespace
+
+- name: Download MetalLB manifest
+  ansible.builtin.get_url:
+    url: "{{ metallb_manifest_url }}"
+    dest: "/tmp/metallb.yaml"
+    mode: "0644"
+    validate_certs: true
+  run_once: true
+  tags:
+    - metallb
+    - download
+
+- name: Apply MetalLB core manifests
+  kubernetes.core.k8s:
+    src: "/tmp/metallb.yaml"
+    state: present
+    namespace: metallb-system
+  tags:
+    - metallb
+    - apply_manifest
+
+- name: Create IPAddressPool for MetalLB
+  kubernetes.core.k8s:
+    state: present
+    namespace: metallb-system
+    definition: "{{ lookup('ansible.builtin.template', 'ipaddresspool.yml.j2') | from_yaml }}"
+  tags:
+    - metallb
+    - ip_pool
+
+- name: Create L2Advertisement for MetalLB
+  kubernetes.core.k8s:
+    state: present
+    namespace: metallb-system
+    definition: "{{ lookup('ansible.builtin.template', 'l2advertisement.yml.j2') | from_yaml }}"
+  tags:
+    - metallb
+    - l2_advertisement
+
+- name: Setup DNS on Netcup
+  community.general.netcup_dns:
+    api_key: "{{ netcup_api_key }}"
+    api_password: "{{ netcup_api_password }}"
+    customer_id: "{{ netcup_customer_id }}"
+    domain: "{{ domain }}"
+    name: "{{ service.name }}.k3s"
+    type: "A"
+    value: "{{ service.ip }}"
+  loop: "{{ services }}"
+  loop_control:
+    loop_var: service
+  delegate_to: localhost

roles/kubernetes_metallb/templates/ipaddresspool.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: default-pool
+  namespace: metallb-system
+spec:
+  addresses:
+    - "{{ metallb_ip_range }}"

roles/kubernetes_metallb/templates/l2advertisement.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: default-l2advertisement
+  namespace: metallb-system
+spec:
+  ipAddressPools:
+    - default-pool

roles/kubernetes_nfs/defaults/main.yml

@@ -0,0 +1,6 @@
+kubernetes_nfs_helm_name: nfs-subdir-external-provisioner
+kubernetes_nfs_helm_url: https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
+kubernetes_nfs_helm_chart: nfs-subdir-external-provisioner/nfs-subdir-external-provisioner
+
+kubernetes_nfs_server_host: 192.168.20.1
+kubernetes_nfs_server_path: /nfs/

roles/kubernetes_nfs/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+# helm repo add
+
+- name: Add a repository
+  kubernetes.core.helm_repository:
+    name: "{{ kubernetes_nfs_helm_name }}"
+    repo_url: "{{ kubernetes_nfs_helm_url }}"
+
+- name: Install NetCup Webhook
+  kubernetes.core.helm:
+    name: "{{ kubernetes_nfs_helm_name }}"
+    chart_ref: "{{ kubernetes_nfs_helm_chart }}"
+    create_namespace: true
+    set_values:
+      - value: "nfs.server={{ kubernetes_nfs_server_host }}"
+      - value: "nfs.path={{ kubernetes_nfs_server_path }}"

roles/kubernetes_traefik/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+traefik_dashboard_hostname: "traefik.example.com"
+traefik_cert_resolver: "cert_resolver-prod"

roles/kubernetes_traefik/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+# roles/traefik/tasks/main.yml
+
+- name: "Traefik | Enable dashboard"
+  kubernetes.core.k8s:
+    template: "helmchartconfig.yaml.j2"
+    state: present
+
+- name: "Traefik | Create dashboard ingress"
+  kubernetes.core.k8s:
+    template: "ingress.yaml.j2"
+    state: present

roles/kubernetes_traefik/templates/helmchartconfig.yaml.j2

@@ -0,0 +1,17 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: traefik
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    logs:
+      access:
+        enabled: true
+    ingressRoute:
+      dashboard:
+        enabled: true
+    websecure:
+      tls:
+        enabled: true

roles/kubernetes_traefik/templates/ingress.yaml.j2

@@ -0,0 +1,27 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: traefik-dashboard
+  namespace: kube-system
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    cert-manager.io/cluster-issuer: {{ traefik_cert_resolver }}
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  rules:
+    - host: {{ traefik_dashboard_hostname }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: traefik
+                port:
+                  number: 8080
+  tls:
+    - hosts:
+      - {{ traefik_dashboard_hostname }}
+      secretName: k3s-seyshiro-de-tls

roles/reverse_proxy/tasks/50_netcup_dns.yml

@@ -1,9 +1,9 @@
 ---
 - name: Setup DNS on Netcup
   community.general.netcup_dns:
-    api_key: "{{ reverse_proxy_netcup_api_key }}"
-    api_password: "{{ reverse_proxy_netcup_api_password }}"
-    customer_id: "{{ reverse_proxy_netcup_customer_id }}"
+    api_key: "{{ netcup_api_key }}"
+    api_password: "{{ netcup_api_password }}"
+    customer_id: "{{ netcup_customer_id }}"
     domain: "{{ domain }}"
     name: "{{ service.name }}"
     type: "A"

roles/reverse_proxy/vars/main.yml

@@ -6,7 +6,3 @@ reverse_proxy_custom_caddy_dest_path: "/usr/bin/caddy.custom"
 reverse_proxy_diverted_caddy_path: "/usr/bin/caddy.default"
 reverse_proxy_alternatives_link: "/usr/bin/caddy"
 reverse_proxy_alternatives_name: "caddy"
-
-reverse_proxy_netcup_api_key: "{{ netcup_api_key }}"
-reverse_proxy_netcup_api_password: "{{ netcup_api_password }}"
-reverse_proxy_netcup_customer_id: "{{ netcup_customer_id }}"

scripts/create_secret_skeleton.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-SECRETS_FILE=$(fdfind "secrets.yml$")
-ansible-vault view "$SECRETS_FILE" | sed "s/: .\+/: ......../" >secrets.yml.skeleton

scripts/debian_seed.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-ssh $1 'mkdir .ssh && echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKqc9fnzfCz8fQDFzla+D8PBhvaMmFu2aF+TYkkZRxl9 tuan@genesis-2022-01-20" >> .ssh/authorized_keys'
-ssh $1 'su root -c "apt update && apt install sudo && /usr/sbin/usermod -a -G sudo tudattr"'

secrets.yml.skeleton

@@ -1,47 +0,0 @@
-vault:
-  k3s:
-    server00:
-      sudo: ........
-    server01:
-      sudo: ........
-    server02:
-      sudo: ........
-    agent00:
-      sudo: ........
-    agent01:
-      sudo: ........
-    agent02:
-      sudo: ........
-    longhorn00:
-      sudo: ........
-    longhorn01:
-      sudo: ........
-    longhorn02:
-      sudo: ........
-    loadbalancer:
-      sudo: ........
-    postgres:
-      sudo: ........
-      db:
-        password: ........
-  docker:
-    host00:
-      sudo: ........
-    host01:
-      sudo: ........
-    host02:
-      sudo: ........
-    lb:
-      sudo: ........
-    elk:
-      elastic: ........
-    paperless:
-      dbpass: ........
-    proton:
-      country: ........
-      openvpn_user: ........
-      openvpn_password: ........
-  netcup:
-    customer_number: ........
-    api_key: ........
-    api_password: ........

vars/group_vars/all/vars.yml

@@ -13,6 +13,8 @@ pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKqc9fnzfCz8fQDFzla+D8PBhvaMmFu2aF+
 public_domain: "tudattr.dev"
 internal_domain: "seyshiro.de"
 
+nfs_server: 192.168.20.12
+
 #
 # Packages
 #

vars/group_vars/kubernetes/services.yml

@@ -0,0 +1,5 @@
+services:
+  - name: argocd
+    ip: 192.168.20.240
+  - name: traefik
+    ip: 192.168.20.240

vars/group_vars/kubernetes/vars.yml

@@ -0,0 +1,18 @@
+domain: "{{ internal_domain }}"
+
+cert_resolver: "letsencrypt-prod"
+cert_manager_issuer_name: "{{ cert_resolver }}"
+cert_manager_email: "me+acme@tudattr.dev"
+
+traefik_cert_resolver: "{{ cert_resolver }}"
+traefik_dashboard_hostname: "traefik.k3s.{{ domain }}"
+
+argocd_cert_resolver: "{{ cert_resolver }}"
+argocd_hostname: "argocd.k3s.{{ domain }}"
+
+metallb_ip_range: "192.168.20.240-192.168.20.250"
+
+traefik_password: "{{ vault_kubernetes.traefik_password }}"
+
+kubernetes_nfs_server_host: "{{ nfs_server }}"
+kubernetes_nfs_server_path: /media/kubernetes

vars/kubernetes.ini

@@ -0,0 +1,2 @@
+[kubernetes]
+127.0.0.1 ansible_connection=local