From f1b0cfad2cf07cbb8e1870a04e84b06ce2c1288a Mon Sep 17 00:00:00 2001 From: Tuan-Dat Tran Date: Sun, 13 Jul 2025 00:40:48 +0200 Subject: [PATCH] refactor(k3s): streamline inventory and primary server IP handling Signed-off-by: Tuan-Dat Tran --- playbooks/k3s-agents.yml | 17 ++-- playbooks/k3s-loadbalancer.yml | 17 ++++ playbooks/k3s-servers.yml | 5 +- playbooks/k3s-storage.yml | 13 +-- playbooks/loadbalancer.yml | 14 --- roles/k3s_agent/tasks/installation.yml | 2 +- .../handlers/main.yml | 0 .../tasks/configuration.yml | 4 +- .../tasks/installation.yml | 0 .../tasks/main.yml | 0 .../k3s_loadbalancer/templates/nginx.conf.j2 | 87 ++++++++++++++++++ roles/k3s_loadbalancer/vars/main.yml | 1 + roles/k3s_server/tasks/installation.yml | 2 +- roles/k3s_storage/tasks/installation.yml | 2 +- roles/loadbalancer/templates/nginx.conf.j2 | 89 ------------------- roles/loadbalancer/vars/main.yml | 1 - roles/proxmox/tasks/56_provision_new_vm.yml | 16 ++-- vars/group_vars/k3s/vars.yml | 15 +--- vars/group_vars/proxmox/secrets_vm.yml | 55 +++++++----- vars/group_vars/proxmox/vms.yml | 11 +++ vars/k3s.ini | 4 - 21 files changed, 185 insertions(+), 170 deletions(-) create mode 100644 playbooks/k3s-loadbalancer.yml delete mode 100644 playbooks/loadbalancer.yml rename roles/{loadbalancer => k3s_loadbalancer}/handlers/main.yml (100%) rename roles/{loadbalancer => k3s_loadbalancer}/tasks/configuration.yml (64%) rename roles/{loadbalancer => k3s_loadbalancer}/tasks/installation.yml (100%) rename roles/{loadbalancer => k3s_loadbalancer}/tasks/main.yml (100%) create mode 100644 roles/k3s_loadbalancer/templates/nginx.conf.j2 create mode 100644 roles/k3s_loadbalancer/vars/main.yml delete mode 100644 roles/loadbalancer/templates/nginx.conf.j2 delete mode 100644 roles/loadbalancer/vars/main.yml diff --git a/playbooks/k3s-agents.yml b/playbooks/k3s-agents.yml index 2d5f49f..e344a6c 100644 --- a/playbooks/k3s-agents.yml +++ b/playbooks/k3s-agents.yml @@ -1,17 +1,20 @@ - name: Set up Agents - hosts: k3s_nodes - gather_facts: yes + hosts: k3s + gather_facts: true + vars: + k3s_primary_server_ip: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_host') | list | first }}" pre_tasks: - name: Get K3s token from the first server - when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"] + when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"] slurp: src: /var/lib/rancher/k3s/server/node-token register: k3s_token become: true - - name: Set fact on k3s.server.ips[0] - when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"] - set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}" + - name: Set fact on k3s_primary_server_ip + when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"] + set_fact: + k3s_token: "{{ k3s_token['content'] | b64decode | trim }}" roles: - role: common @@ -20,7 +23,7 @@ - common - role: k3s_agent when: inventory_hostname in groups["k3s_agent"] - k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}" + k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s_primary_server_ip ) | select() | first | items2dict).host.hostname].k3s_token }}" tags: - k3s_agent - role: node_exporter diff --git a/playbooks/k3s-loadbalancer.yml b/playbooks/k3s-loadbalancer.yml new file mode 100644 index 0000000..5f0e46e --- /dev/null +++ b/playbooks/k3s-loadbalancer.yml @@ -0,0 +1,17 @@ +--- +- name: Set up Servers + hosts: k3s + gather_facts: true + roles: + - role: common + tags: + - common + when: inventory_hostname in groups["k3s_loadbalancer"] + - role: k3s_loadbalancer + tags: + - k3s_loadbalancer + when: inventory_hostname in groups["k3s_loadbalancer"] + # - role: node_exporter + # tags: + # - node_exporter + # when: inventory_hostname in groups["k3s_loadbalancer"] diff --git a/playbooks/k3s-servers.yml b/playbooks/k3s-servers.yml index 065ef2c..87f41d5 100644 --- a/playbooks/k3s-servers.yml +++ b/playbooks/k3s-servers.yml @@ -1,7 +1,7 @@ --- - name: Set up Servers - hosts: k3s_server - gather_facts: yes + hosts: k3s + gather_facts: true roles: - role: common tags: @@ -9,6 +9,7 @@ - role: k3s_server tags: - k3s_server + when: inventory_hostname in groups["k3s_server"] - role: node_exporter tags: - node_exporter diff --git a/playbooks/k3s-storage.yml b/playbooks/k3s-storage.yml index 35e29d4..4787ea0 100644 --- a/playbooks/k3s-storage.yml +++ b/playbooks/k3s-storage.yml @@ -1,17 +1,20 @@ - name: Set up storage hosts: k3s_nodes gather_facts: true + vars: + k3s_primary_server_ip: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_host') | list | first }}" pre_tasks: - name: Get K3s token from the first server - when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"] + when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"] slurp: src: /var/lib/rancher/k3s/server/node-token register: k3s_token become: true - - name: Set fact on k3s.server.ips[0] - when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"] - set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}" + - name: Set fact on k3s_primary_server_ip + when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"] + set_fact: + k3s_token: "{{ k3s_token['content'] | b64decode | trim }}" roles: - role: common @@ -20,7 +23,7 @@ - common - role: k3s_storage when: inventory_hostname in groups["k3s_storage"] - k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}" + k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s_primary_server_ip ) | select() | first | items2dict).host.hostname].k3s_token }}" tags: - k3s_storage - role: node_exporter diff --git a/playbooks/loadbalancer.yml b/playbooks/loadbalancer.yml deleted file mode 100644 index 9c6bc67..0000000 --- a/playbooks/loadbalancer.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Set up Servers - hosts: loadbalancer - gather_facts: yes - roles: - - role: common - tags: - - common - - role: loadbalancer - tags: - - loadbalancer - - role: node_exporter - tags: - - node_exporter diff --git a/roles/k3s_agent/tasks/installation.yml b/roles/k3s_agent/tasks/installation.yml index 17bfd05..ff0a69d 100644 --- a/roles/k3s_agent/tasks/installation.yml +++ b/roles/k3s_agent/tasks/installation.yml @@ -16,6 +16,6 @@ ansible.builtin.command: | /tmp/k3s_install.sh environment: - K3S_URL: "https://{{ k3s.loadbalancer.ip }}:{{ k3s.loadbalancer.default_port }}" + K3S_URL: "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}" K3S_TOKEN: "{{ k3s_token }}" become: true diff --git a/roles/loadbalancer/handlers/main.yml b/roles/k3s_loadbalancer/handlers/main.yml similarity index 100% rename from roles/loadbalancer/handlers/main.yml rename to roles/k3s_loadbalancer/handlers/main.yml diff --git a/roles/loadbalancer/tasks/configuration.yml b/roles/k3s_loadbalancer/tasks/configuration.yml similarity index 64% rename from roles/loadbalancer/tasks/configuration.yml rename to roles/k3s_loadbalancer/tasks/configuration.yml index 293b587..fee2ec8 100644 --- a/roles/loadbalancer/tasks/configuration.yml +++ b/roles/k3s_loadbalancer/tasks/configuration.yml @@ -2,7 +2,7 @@ - name: Template the nginx config file with dynamic upstreams ansible.builtin.template: src: templates/nginx.conf.j2 - dest: "{{ nginx_config_path }}" + dest: "{{ k3s_loadbalancer_nginx_config_path }}" owner: root group: root mode: "0644" @@ -10,7 +10,7 @@ notify: - Restart nginx vars: - k3s_server_ips: "{{ k3s.server.ips }}" + k3s_server_ips: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | unique | list }}" - name: Enable nginx ansible.builtin.systemd: diff --git a/roles/loadbalancer/tasks/installation.yml b/roles/k3s_loadbalancer/tasks/installation.yml similarity index 100% rename from roles/loadbalancer/tasks/installation.yml rename to roles/k3s_loadbalancer/tasks/installation.yml diff --git a/roles/loadbalancer/tasks/main.yml b/roles/k3s_loadbalancer/tasks/main.yml similarity index 100% rename from roles/loadbalancer/tasks/main.yml rename to roles/k3s_loadbalancer/tasks/main.yml diff --git a/roles/k3s_loadbalancer/templates/nginx.conf.j2 b/roles/k3s_loadbalancer/templates/nginx.conf.j2 new file mode 100644 index 0000000..bb59cab --- /dev/null +++ b/roles/k3s_loadbalancer/templates/nginx.conf.j2 @@ -0,0 +1,87 @@ +include /etc/nginx/modules-enabled/*.conf; + +events {} + +stream { +# TCP Load Balancing for the K3s API + upstream k3s_servers { + {% for ip in k3s_server_ips %} + server {{ ip }}:{{ k3s.loadbalancer.default_port }}; + {% endfor %} + } + + server { + listen {{k3s.loadbalancer.default_port}}; + proxy_pass k3s_servers; + } + + upstream dns_servers { + {% for ip in k3s_server_ips %} + server {{ ip }}:53; + {% endfor %} + } + + server { + listen 53 udp; + proxy_pass dns_servers; + } +} + +# http { +# upstream k3s_servers_http { +# least_conn; +# {% for ip in k3s_server_ips %} +# server {{ ip }}:80; +# {% endfor %} +# } +# +# upstream k3s_servers_https { +# least_conn; +# {% for ip in k3s_server_ips %} +# server {{ ip }}:443; +# {% endfor %} +# } +# +# server { +# listen 80; +# +# location / { +# proxy_pass http://k3s_servers_http; +# proxy_set_header Host $http_host; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-Forwarded-Proto http; +# } +# } +# +# server { +# listen 443 ssl; +# +# server_name staging.k3s.seyshiro.de *.staging.k3s.seyshiro.de; +# +# ssl_certificate /etc/nginx/ssl/staging_tls.crt; +# ssl_certificate_key /etc/nginx/ssl/staging_tls.key; +# +# location / { +# proxy_pass https://k3s_servers_https; +# proxy_set_header Host $host; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-Forwarded-Proto https; +# } +# } +# +# server { +# listen 443 ssl; +# +# server_name k3s.seyshiro.de *.k3s.seyshiro.de; +# +# ssl_certificate /etc/nginx/ssl/production_tls.crt; +# ssl_certificate_key /etc/nginx/ssl/production_tls.key; +# +# location / { +# proxy_pass https://k3s_servers_https; +# proxy_set_header Host $host; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-Forwarded-Proto https; +# } +# } +# } diff --git a/roles/k3s_loadbalancer/vars/main.yml b/roles/k3s_loadbalancer/vars/main.yml new file mode 100644 index 0000000..4df274c --- /dev/null +++ b/roles/k3s_loadbalancer/vars/main.yml @@ -0,0 +1 @@ +k3s_loadbalancer_nginx_config_path: "/etc/nginx/nginx.conf" diff --git a/roles/k3s_server/tasks/installation.yml b/roles/k3s_server/tasks/installation.yml index a833f3c..07d6b73 100644 --- a/roles/k3s_server/tasks/installation.yml +++ b/roles/k3s_server/tasks/installation.yml @@ -16,7 +16,7 @@ ansible.builtin.command: | /tmp/k3s_install.sh server \ --node-taint CriticalAddonsOnly=true:NoExecute \ - --tls-san {{ k3s.loadbalancer.ip }} + --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }} become: true async: 300 poll: 0 diff --git a/roles/k3s_storage/tasks/installation.yml b/roles/k3s_storage/tasks/installation.yml index e760dd3..41b9be8 100644 --- a/roles/k3s_storage/tasks/installation.yml +++ b/roles/k3s_storage/tasks/installation.yml @@ -18,6 +18,6 @@ --node-taint storage=true:NoExecute \ --node-label longhorn=true environment: - K3S_URL: "https://{{ k3s.loadbalancer.ip }}:{{ k3s.loadbalancer.default_port }}" + K3S_URL: "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}" K3S_TOKEN: "{{ k3s_token }}" become: true diff --git a/roles/loadbalancer/templates/nginx.conf.j2 b/roles/loadbalancer/templates/nginx.conf.j2 deleted file mode 100644 index 83438ab..0000000 --- a/roles/loadbalancer/templates/nginx.conf.j2 +++ /dev/null @@ -1,89 +0,0 @@ -include /etc/nginx/modules-enabled/*.conf; - -events {} - -stream { -# TCP Load Balancing for the K3s API - upstream k3s_servers { - {% for ip in k3s_server_ips %} - server {{ ip }}:{{k3s.loadbalancer.default_port}}; - {% endfor %} - } - - server { - listen {{k3s.loadbalancer.default_port}}; - proxy_pass k3s_servers; - } - - upstream dns_servers { - {% for ip in k3s_server_ips %} - server {{ ip }}:53; - {% endfor %} - } - - server { - listen 53 udp; - proxy_pass dns_servers; - } -} - -http { - upstream k3s_servers_http { - least_conn; - {% for ip in k3s_server_ips %} - server {{ ip }}:80; - {% endfor %} - } - - upstream k3s_servers_https { - least_conn; - {% for ip in k3s_server_ips %} - server {{ ip }}:443; - {% endfor %} - } - - server { - listen 80; - - location / { - proxy_pass http://k3s_servers_http; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto http; - } - } - - server { - listen 443 ssl; - - server_name staging.k3s.seyshiro.de *.staging.k3s.seyshiro.de; - - ssl_certificate /etc/nginx/ssl/staging_tls.crt; - ssl_certificate_key /etc/nginx/ssl/staging_tls.key; - - location / { - proxy_pass https://k3s_servers_https; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - } - } - - server { - listen 443 ssl; - - server_name k3s.seyshiro.de *.k3s.seyshiro.de; - - ssl_certificate /etc/nginx/ssl/production_tls.crt; - ssl_certificate_key /etc/nginx/ssl/production_tls.key; - - location / { - proxy_pass https://k3s_servers_https; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - } - } -} - - diff --git a/roles/loadbalancer/vars/main.yml b/roles/loadbalancer/vars/main.yml deleted file mode 100644 index cc55422..0000000 --- a/roles/loadbalancer/vars/main.yml +++ /dev/null @@ -1 +0,0 @@ -nginx_config_path: "/etc/nginx/nginx.conf" diff --git a/roles/proxmox/tasks/56_provision_new_vm.yml b/roles/proxmox/tasks/56_provision_new_vm.yml index c3fd284..2b2bb30 100644 --- a/roles/proxmox/tasks/56_provision_new_vm.yml +++ b/roles/proxmox/tasks/56_provision_new_vm.yml @@ -78,11 +78,11 @@ ProxyJump {{ vm.node }} StrictHostKeyChecking no -- name: Add VM to homelab_vms group in production.ini - ansible.builtin.lineinfile: - path: "{{ inventory_file }}" - line: "{{ vm.name }}" - insertafter: '^\[vms\]' - create: true - state: present - delegate_to: localhost +# - name: Add VM to homelab_vms group in production.ini +# ansible.builtin.lineinfile: +# path: "{{ inventory_file }}" +# line: "{{ vm.name }}" +# insertafter: '^\[vms\]' +# create: true +# state: present +# delegate_to: localhost diff --git a/vars/group_vars/k3s/vars.yml b/vars/group_vars/k3s/vars.yml index d8c994c..d813732 100644 --- a/vars/group_vars/k3s/vars.yml +++ b/vars/group_vars/k3s/vars.yml @@ -1,18 +1,7 @@ k3s: - net: "192.168.20.0/24" server: - ips: - - 192.168.20.21 - - 192.168.20.24 - - 192.168.20.30 + ips: [] loadbalancer: - ip: 192.168.20.22 default_port: 6443 - db: - ip: 192.168.20.23 - default_port: "5432" agent: - ips: - - 192.168.20.25 - - 192.168.20.26 - - 192.168.20.27 + ips: [] diff --git a/vars/group_vars/proxmox/secrets_vm.yml b/vars/group_vars/proxmox/secrets_vm.yml index e40960d..38b692a 100644 --- a/vars/group_vars/proxmox/secrets_vm.yml +++ b/vars/group_vars/proxmox/secrets_vm.yml @@ -1,23 +1,34 @@ $ANSIBLE_VAULT;1.1;AES256 -31643231626635633436363136386537616133326538323239663963346332383961396132316662 -3938393638646562306634333932666663363363353264620a613833666634383061343565613364 -38343537333930303563613839303265373339616463626133646365643630313339633765333231 -6236306463616565350a626235666164303737646338363232336363336539656439316462643332 -36346530306266616465643766333864356264386435383633356534663438376335643630613230 -64313663613332666534623433653539653234646661636230616134353336663631313661333661 -32666632363765613934353536343339306632666238626330663938313030633362316661656432 -64393863356336343261663935373530346162323665303632646531613530393432393332663963 -62303663613766613830383735643839353039663631333231343036636537643237643932656162 -32396632316263646637653562386438613930313331653261373363386134663835313762646136 -37623237636464613736353237313666656234303534623961666230393530386435393734376639 -39366636623132326230396635376136383634306664336332663535366230653632613935383135 -31383232386633666263666439306631373663613930623762343635376261316136656539323631 -64393062623461383733316231633335303535363763633737373933656563623234353930323262 -33623463613638306630653639646230396539383065336166643935346435363534353836626262 -62393262646262323433373561303132336564353062396331623264333464346534626633333561 -30636462646664656532393139636331376534643234663566633862373263306365643336343039 -64343236303139626164656139613438623030633735666130346335626530373636666534616233 -65626166386537333162393962666461613266366261316339356665643765376666393965613835 -65376561383865336165343662333236653537666563613730666461633233356166653964333164 -35646264653062396330373135363338346138353136626661643531323961316231356262363966 -3636356230396130663531353437653034396534313863336134 +64336139336538333337376465316164383766643666336666643166333134636338323562303364 +6235613337366634613532373933396230666137373562650a643633306165643331643464633762 +35336433626161393735353133343739353738653061613733393135313061643663616665316463 +6238376435633435650a306636303934383739656439383632313964356434353536373961646531 +35303533666633346363663936366535613039356164383362393736306338613236373138663731 +65666635353734353261333332393962636664653332313062336239313834653536363539306630 +61316431313631643637616434376334323232306232363936613139373762613862653938373461 +34366363643337326439633963303430613935323866343764326639663531303931396235643231 +36346463653866653137653931303439326433366231303530316632613033333761326536326335 +30343233333232333434303562396166386133313633323732636532376539633336613532633765 +66656663353964316364636236623133306533656465303833346563376461396639626262333133 +33663966393030653762636164653534363338613536636432663938393033313933323830336538 +61663865353466393836333539636466613137396430636566303135326565383764373831336532 +66626332383065643636663638616337316136623131333630613861353730646339366239633861 +31343133346138343637373039633930653731396537323438623237393436303063623862663965 +65353332393331623933323138633231363539323834333631643337613863643737306363323135 +61353663643563393539373839643462616339333762353962653065653134653063336466343431 +61313262616631343265386530653431356632616230633032363165656666333662636339306539 +37646634353961346165356565313038303333303564333862323766366238366434643562306262 +38656532333339643335386130356637353434393037636530363233393162663330663566663962 +34343333383631343330663962343639633464353961343933653764643666626631346434366365 +37303433626330346630353064613766303634386238636230346531663038653865393939663732 +37613461313738313766306663653264616563633966316362356539373239663464386430636464 +61373864313064626133623332643139336163643465376234373530666630656361616236336130 +37623962393237623135656534613839363831613165356563333039366462306230636432653636 +64333633393532313635323830333432666134373630666561626231666433303132663939633965 +61373137633865323564343661623039616331323164396133343165656263383865383861616262 +64636230336130356364333964336335656664303334326537303033613331353038353666646463 +63363631613238633831666136363833363964356432373434643131653531666166666233613861 +30306435306563303333343364333065616438383331383437353234323633393733653965313165 +30643539663330356630363833643136643265623966636466336539353738373136616265393265 +36613564653634313438666334313636653435336263393635656138343534336232346332356264 +33366232613832643862386532663264353735393033303864356230333864363366 diff --git a/vars/group_vars/proxmox/vms.yml b/vars/group_vars/proxmox/vms.yml index a556a85..c6c29ab 100644 --- a/vars/group_vars/proxmox/vms.yml +++ b/vars/group_vars/proxmox/vms.yml @@ -78,3 +78,14 @@ vms: ciuser: "{{ user }}" sshkeys: "{{ pubkey }}" disk_size: 64 # in Gb + - name: "k3s-loadbalancer" + node: "naruto01" + vmid: 150 + cores: 1 + memory: 2048 # in MiB + net: + net0: "virtio,bridge=vmbr0,firewall=1" + boot_image: "{{ proxmox_cloud_init_images.debian.name }}" + ciuser: "{{ user }}" + sshkeys: "{{ pubkey }}" + disk_size: 32 # in Gb diff --git a/vars/k3s.ini b/vars/k3s.ini index 67e9de6..d918471 100644 --- a/vars/k3s.ini +++ b/vars/k3s.ini @@ -4,7 +4,6 @@ k3s_server k3s_agent k3s_storage -k3s_storage k3s_loadbalancer [k3s_server] @@ -13,9 +12,6 @@ k3s-server10 [k3s_agent] k3s-agent[10:12] -[k3s_storage] -k3s-longhorn[10:12] - [k3s_loadbalancer] k3s-loadbalancer