refactor(k3s): streamline inventory and primary server IP handling

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

playbooks/k3s-agents.yml

@@ -1,17 +1,20 @@
 - name: Set up Agents
-  hosts: k3s_nodes
-  gather_facts: yes
+  hosts: k3s
+  gather_facts: true
+  vars:
+    k3s_primary_server_ip: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_host') | list | first }}"
   pre_tasks:
     - name: Get K3s token from the first server
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
       slurp:
         src: /var/lib/rancher/k3s/server/node-token
       register: k3s_token
       become: true
 
-    - name: Set fact on k3s.server.ips[0]
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
-      set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}"
+    - name: Set fact on k3s_primary_server_ip
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
+      set_fact:
+        k3s_token: "{{ k3s_token['content'] | b64decode | trim }}"
 
   roles:
     - role: common
@@ -20,7 +23,7 @@
         - common
     - role: k3s_agent
       when: inventory_hostname in groups["k3s_agent"]
-      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}"
+      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s_primary_server_ip ) | select() | first | items2dict).host.hostname].k3s_token }}"
       tags:
         - k3s_agent
     - role: node_exporter

playbooks/k3s-loadbalancer.yml

@@ -0,0 +1,17 @@
+---
+- name: Set up Servers
+  hosts: k3s
+  gather_facts: true
+  roles:
+    - role: common
+      tags:
+        - common
+      when: inventory_hostname in groups["k3s_loadbalancer"]
+    - role: k3s_loadbalancer
+      tags:
+        - k3s_loadbalancer
+      when: inventory_hostname in groups["k3s_loadbalancer"]
+    # - role: node_exporter
+    #   tags:
+    #     - node_exporter
+    #   when: inventory_hostname in groups["k3s_loadbalancer"]

playbooks/k3s-servers.yml

@@ -1,7 +1,7 @@
 ---
 - name: Set up Servers
-  hosts: k3s_server
-  gather_facts: yes
+  hosts: k3s
+  gather_facts: true
   roles:
     - role: common
       tags:
@@ -9,6 +9,7 @@
     - role: k3s_server
       tags:
         - k3s_server
+      when: inventory_hostname in groups["k3s_server"]
     - role: node_exporter
       tags:
         - node_exporter

playbooks/k3s-storage.yml

@@ -1,17 +1,20 @@
 - name: Set up storage
   hosts: k3s_nodes
   gather_facts: true
+  vars:
+    k3s_primary_server_ip: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_host') | list | first }}"
   pre_tasks:
     - name: Get K3s token from the first server
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
       slurp:
         src: /var/lib/rancher/k3s/server/node-token
       register: k3s_token
       become: true
 
-    - name: Set fact on k3s.server.ips[0]
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
-      set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}"
+    - name: Set fact on k3s_primary_server_ip
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
+      set_fact:
+        k3s_token: "{{ k3s_token['content'] | b64decode | trim }}"
 
   roles:
     - role: common
@@ -20,7 +23,7 @@
         - common
     - role: k3s_storage
       when: inventory_hostname in groups["k3s_storage"]
-      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}"
+      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s_primary_server_ip ) | select() | first | items2dict).host.hostname].k3s_token }}"
       tags:
         - k3s_storage
     - role: node_exporter

playbooks/loadbalancer.yml

@@ -1,14 +0,0 @@
----
-- name: Set up Servers
-  hosts: loadbalancer
-  gather_facts: yes
-  roles:
-    - role: common
-      tags:
-        - common
-    - role: loadbalancer
-      tags:
-        - loadbalancer
-    - role: node_exporter
-      tags:
-        - node_exporter

roles/k3s_agent/tasks/installation.yml

@@ -16,6 +16,6 @@
   ansible.builtin.command: |
     /tmp/k3s_install.sh
   environment:
-    K3S_URL: "https://{{ k3s.loadbalancer.ip }}:{{ k3s.loadbalancer.default_port }}"
+    K3S_URL: "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}"
     K3S_TOKEN: "{{ k3s_token }}"
   become: true

roles/k3s_loadbalancer/tasks/configuration.yml

@@ -2,7 +2,7 @@
 - name: Template the nginx config file with dynamic upstreams
   ansible.builtin.template:
     src: templates/nginx.conf.j2
-    dest: "{{ nginx_config_path }}"
+    dest: "{{ k3s_loadbalancer_nginx_config_path }}"
     owner: root
     group: root
     mode: "0644"
@@ -10,7 +10,7 @@
   notify:
     - Restart nginx
   vars:
-    k3s_server_ips: "{{ k3s.server.ips }}"
+    k3s_server_ips: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | unique | list }}"
 
 - name: Enable nginx
   ansible.builtin.systemd:

roles/k3s_loadbalancer/templates/nginx.conf.j2

@@ -0,0 +1,87 @@
+include /etc/nginx/modules-enabled/*.conf;
+
+events {}
+
+stream {
+# TCP Load Balancing for the K3s API
+  upstream k3s_servers {
+    {% for ip in k3s_server_ips %}
+    server {{ ip }}:{{ k3s.loadbalancer.default_port }};
+    {% endfor %}
+  }
+
+  server {
+    listen {{k3s.loadbalancer.default_port}};
+    proxy_pass k3s_servers;
+  }
+
+  upstream dns_servers {
+    {% for ip in k3s_server_ips %}
+    server {{ ip }}:53;
+    {% endfor %}
+  }
+
+  server {
+    listen 53 udp;
+    proxy_pass dns_servers;
+  }
+}
+
+# http {
+#   upstream k3s_servers_http {
+#     least_conn;
+#     {% for ip in k3s_server_ips %}
+#     server {{ ip }}:80;
+#     {% endfor %}
+#   }
+#
+#   upstream k3s_servers_https {
+#     least_conn;
+#     {% for ip in k3s_server_ips %}
+#     server {{ ip }}:443;
+#     {% endfor %}
+#   }
+#
+#   server {
+#     listen 80;
+#
+#     location / {
+#       proxy_pass http://k3s_servers_http;
+#       proxy_set_header Host $http_host;
+#       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#       proxy_set_header X-Forwarded-Proto http;
+#     }
+#   }
+#
+#   server {
+#     listen 443 ssl;
+#
+#     server_name staging.k3s.seyshiro.de *.staging.k3s.seyshiro.de;
+#
+#     ssl_certificate /etc/nginx/ssl/staging_tls.crt;
+#     ssl_certificate_key /etc/nginx/ssl/staging_tls.key;
+#
+#     location / {
+#       proxy_pass https://k3s_servers_https;
+#       proxy_set_header Host $host;
+#       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#       proxy_set_header X-Forwarded-Proto https;
+#     }
+#   }
+#
+#   server {
+#     listen 443 ssl;
+#
+#     server_name k3s.seyshiro.de *.k3s.seyshiro.de;
+#
+#     ssl_certificate /etc/nginx/ssl/production_tls.crt;
+#     ssl_certificate_key /etc/nginx/ssl/production_tls.key;
+#
+#     location / {
+#       proxy_pass https://k3s_servers_https;
+#       proxy_set_header Host $host;
+#       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#       proxy_set_header X-Forwarded-Proto https;
+#     }
+#   }
+# }

roles/k3s_loadbalancer/vars/main.yml

@@ -0,0 +1 @@
+k3s_loadbalancer_nginx_config_path: "/etc/nginx/nginx.conf"

roles/k3s_server/tasks/installation.yml

@@ -16,7 +16,7 @@
   ansible.builtin.command: |
     /tmp/k3s_install.sh server \
     --node-taint CriticalAddonsOnly=true:NoExecute \
-    --tls-san {{ k3s.loadbalancer.ip }}
+    --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}
   become: true
   async: 300
   poll: 0

roles/k3s_storage/tasks/installation.yml

@@ -18,6 +18,6 @@
     --node-taint storage=true:NoExecute \
     --node-label longhorn=true
   environment:
-    K3S_URL: "https://{{ k3s.loadbalancer.ip }}:{{ k3s.loadbalancer.default_port }}"
+    K3S_URL: "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}"
     K3S_TOKEN: "{{ k3s_token }}"
   become: true

roles/loadbalancer/templates/nginx.conf.j2

@@ -1,89 +0,0 @@
-include /etc/nginx/modules-enabled/*.conf;
-
-events {}
-
-stream {
-# TCP Load Balancing for the K3s API
-  upstream k3s_servers {
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:{{k3s.loadbalancer.default_port}};
-    {% endfor %}
-  }
-
-  server {
-    listen {{k3s.loadbalancer.default_port}};
-    proxy_pass k3s_servers;
-  }
-
-  upstream dns_servers {
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:53;
-    {% endfor %}
-  }
-
-  server {
-    listen 53 udp;
-    proxy_pass dns_servers;
-  }
-}
-
-http {
-  upstream k3s_servers_http {
-    least_conn;
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:80;
-    {% endfor %}
-  }
-
-  upstream k3s_servers_https {
-    least_conn;
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:443;
-    {% endfor %}
-  }
-
-  server {
-    listen 80;
-
-    location / {
-      proxy_pass http://k3s_servers_http;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto http;
-    }
-  }
-
-  server {
-    listen 443 ssl;
-
-    server_name staging.k3s.seyshiro.de *.staging.k3s.seyshiro.de;
-
-    ssl_certificate /etc/nginx/ssl/staging_tls.crt;
-    ssl_certificate_key /etc/nginx/ssl/staging_tls.key;
-
-    location / {
-      proxy_pass https://k3s_servers_https;
-      proxy_set_header Host $host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto https;
-    }
-  }
-
-  server {
-    listen 443 ssl;
-
-    server_name k3s.seyshiro.de *.k3s.seyshiro.de;
-
-    ssl_certificate /etc/nginx/ssl/production_tls.crt;
-    ssl_certificate_key /etc/nginx/ssl/production_tls.key;
-
-    location / {
-      proxy_pass https://k3s_servers_https;
-      proxy_set_header Host $host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto https;
-    }
-  }
-}
-
-

roles/loadbalancer/vars/main.yml

@@ -1 +0,0 @@
-nginx_config_path: "/etc/nginx/nginx.conf"

roles/proxmox/tasks/56_provision_new_vm.yml

@@ -78,11 +78,11 @@
         ProxyJump {{ vm.node }}
         StrictHostKeyChecking no
 
-- name: Add VM to homelab_vms group in production.ini
-  ansible.builtin.lineinfile:
-    path: "{{ inventory_file }}"
-    line: "{{ vm.name }}"
-    insertafter: '^\[vms\]'
-    create: true
-    state: present
-  delegate_to: localhost
+# - name: Add VM to homelab_vms group in production.ini
+#   ansible.builtin.lineinfile:
+#     path: "{{ inventory_file }}"
+#     line: "{{ vm.name }}"
+#     insertafter: '^\[vms\]'
+#     create: true
+#     state: present
+#   delegate_to: localhost

vars/group_vars/k3s/vars.yml

@@ -1,18 +1,7 @@
 k3s:
-  net: "192.168.20.0/24"
   server:
-    ips:
-      - 192.168.20.21
-      - 192.168.20.24
-      - 192.168.20.30
+    ips: []
   loadbalancer:
-    ip: 192.168.20.22
     default_port: 6443
-  db:
-    ip: 192.168.20.23
-    default_port: "5432"
   agent:
-    ips:
-      - 192.168.20.25
-      - 192.168.20.26
-      - 192.168.20.27
+    ips: []

vars/group_vars/proxmox/secrets_vm.yml

@@ -1,23 +1,34 @@
 $ANSIBLE_VAULT;1.1;AES256
-31643231626635633436363136386537616133326538323239663963346332383961396132316662
-3938393638646562306634333932666663363363353264620a613833666634383061343565613364
-38343537333930303563613839303265373339616463626133646365643630313339633765333231
-6236306463616565350a626235666164303737646338363232336363336539656439316462643332
-36346530306266616465643766333864356264386435383633356534663438376335643630613230
-64313663613332666534623433653539653234646661636230616134353336663631313661333661
-32666632363765613934353536343339306632666238626330663938313030633362316661656432
-64393863356336343261663935373530346162323665303632646531613530393432393332663963
-62303663613766613830383735643839353039663631333231343036636537643237643932656162
-32396632316263646637653562386438613930313331653261373363386134663835313762646136
-37623237636464613736353237313666656234303534623961666230393530386435393734376639
-39366636623132326230396635376136383634306664336332663535366230653632613935383135
-31383232386633666263666439306631373663613930623762343635376261316136656539323631
-64393062623461383733316231633335303535363763633737373933656563623234353930323262
-33623463613638306630653639646230396539383065336166643935346435363534353836626262
-62393262646262323433373561303132336564353062396331623264333464346534626633333561
-30636462646664656532393139636331376534643234663566633862373263306365643336343039
-64343236303139626164656139613438623030633735666130346335626530373636666534616233
-65626166386537333162393962666461613266366261316339356665643765376666393965613835
-65376561383865336165343662333236653537666563613730666461633233356166653964333164
-35646264653062396330373135363338346138353136626661643531323961316231356262363966
-3636356230396130663531353437653034396534313863336134
+64336139336538333337376465316164383766643666336666643166333134636338323562303364
+6235613337366634613532373933396230666137373562650a643633306165643331643464633762
+35336433626161393735353133343739353738653061613733393135313061643663616665316463
+6238376435633435650a306636303934383739656439383632313964356434353536373961646531
+35303533666633346363663936366535613039356164383362393736306338613236373138663731
+65666635353734353261333332393962636664653332313062336239313834653536363539306630
+61316431313631643637616434376334323232306232363936613139373762613862653938373461
+34366363643337326439633963303430613935323866343764326639663531303931396235643231
+36346463653866653137653931303439326433366231303530316632613033333761326536326335
+30343233333232333434303562396166386133313633323732636532376539633336613532633765
+66656663353964316364636236623133306533656465303833346563376461396639626262333133
+33663966393030653762636164653534363338613536636432663938393033313933323830336538
+61663865353466393836333539636466613137396430636566303135326565383764373831336532
+66626332383065643636663638616337316136623131333630613861353730646339366239633861
+31343133346138343637373039633930653731396537323438623237393436303063623862663965
+65353332393331623933323138633231363539323834333631643337613863643737306363323135
+61353663643563393539373839643462616339333762353962653065653134653063336466343431
+61313262616631343265386530653431356632616230633032363165656666333662636339306539
+37646634353961346165356565313038303333303564333862323766366238366434643562306262
+38656532333339643335386130356637353434393037636530363233393162663330663566663962
+34343333383631343330663962343639633464353961343933653764643666626631346434366365
+37303433626330346630353064613766303634386238636230346531663038653865393939663732
+37613461313738313766306663653264616563633966316362356539373239663464386430636464
+61373864313064626133623332643139336163643465376234373530666630656361616236336130
+37623962393237623135656534613839363831613165356563333039366462306230636432653636
+64333633393532313635323830333432666134373630666561626231666433303132663939633965
+61373137633865323564343661623039616331323164396133343165656263383865383861616262
+64636230336130356364333964336335656664303334326537303033613331353038353666646463
+63363631613238633831666136363833363964356432373434643131653531666166666233613861
+30306435306563303333343364333065616438383331383437353234323633393733653965313165
+30643539663330356630363833643136643265623966636466336539353738373136616265393265
+36613564653634313438666334313636653435336263393635656138343534336232346332356264
+33366232613832643862386532663264353735393033303864356230333864363366

vars/group_vars/proxmox/vms.yml

@@ -78,3 +78,14 @@ vms:
     ciuser: "{{ user }}"
     sshkeys: "{{ pubkey }}"
     disk_size: 64 # in Gb
+  - name: "k3s-loadbalancer"
+    node: "naruto01"
+    vmid: 150
+    cores: 1
+    memory: 2048 # in MiB
+    net:
+      net0: "virtio,bridge=vmbr0,firewall=1"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    ciuser: "{{ user }}"
+    sshkeys: "{{ pubkey }}"
+    disk_size: 32 # in Gb

vars/k3s.ini

@@ -4,7 +4,6 @@
 k3s_server
 k3s_agent
 k3s_storage
-k3s_storage
 k3s_loadbalancer
 
 [k3s_server]
@@ -13,9 +12,6 @@ k3s-server10
 [k3s_agent]
 k3s-agent[10:12]
 
-[k3s_storage]
-k3s-longhorn[10:12]
-
 [k3s_loadbalancer]
 k3s-loadbalancer