refactor(k3s): streamline inventory and primary server IP handling

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

ansible.cfg

@@ -6,7 +6,7 @@ interpreter_python=python3
 roles_path=./roles
 
 # (pathlist) Comma separated list of Ansible inventory sources
-inventory=./inventory
+inventory=./vars/
 
 # (path) The vault password file to use. Equivalent to --vault-password-file or --vault-id
 # If executable, it will be run and the resulting stdout will be used as the password.
@@ -36,3 +36,6 @@ skip=dark gray
 [tags]
 # (list) default list of tags to skip in your plays, has precedence over Run Tags
 ;skip=
+
+[inventory]
+ignore_extensions={{(REJECT_EXTS + ('.orig', '.cfg', '.retry', '.bak'))}}

group_vars/k3s/vars.yml

@@ -1,18 +0,0 @@
-k3s:
-  net: "192.168.20.0/24"
-  server:
-    ips:
-      - 192.168.20.21
-      - 192.168.20.24
-      - 192.168.20.30
-  loadbalancer:
-    ip: 192.168.20.22
-    default_port: 6443
-  db:
-    ip: 192.168.20.23
-    default_port: "5432"
-  agent:
-    ips:
-      - 192.168.20.25
-      - 192.168.20.26
-      - 192.168.20.27

group_vars/proxmox/secrets.yml

@@ -1,15 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-35333866323538343132373761316430616539643436646637633131366232346566656438303438
-3539333661363964633834613161626134323533653737650a613832323436663739663162303066
-31333130646631306539356233346632636132346539343734393065353033613865363466646632
-6565343937666530330a326130393934326435643837323631653862313232363466643534306131
-62376132383137336230366538326364663362346137613930633161663834393835623935373164
-65623564633765653137623361376130623363613263313835366464313039613532323661363461
-37366438616566643537656639316665363339633737363539636364316335663639303364663366
-62653734343364663830633534643931656439313763366138323663373464303137323864313637
-65316135343464393031343166366338323839326631623533343931353833643232643339386231
-38623735386465383964653663346631376531376261353933346661666131353533633331353437
-63336366623333653732306130316264393865633338653238303861646535343837396232366134
-63343037636361323239376436326431623165326366383561323832323730636532623039383734
-66663139656262643038303435346666323762343661336234663131343531636161636536646465
-6530333864323262363536393562346362306161653162346132

group_vars/proxmox/secrets_vm.yml

@@ -1,20 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-35616266333838306161336339353538306634373132626132643732303066303163343630333630
-6338393762616262303038373334663230383464643836370a656538393531393134616463643239
-36383330653339393362353838313639333432643535643833396535653632376336613130646663
-3532646538363137630a363731613235653935316531616430346264643837306434386333373033
-33663135653931373963343734366562386263663939383536663439383537333264666233343233
-62626162666538333435396638393338393734656131303065616534613733353335643939333765
-38326237343337363064666530303664326563633262313432343030336266373437353837346461
-63333363626164316638346635666537613963383537313965373638303732353365623166363736
-31633239646262613539646637663664313337353465636366313338303439613638653530656631
-62396536316561623736633631623336313537646138383431633538303163303261323864383538
-38626338373332653561343036323236383337343037356366626230646432646538373836303063
-61346339376561626630653562346439306561643664666437386562356535303264646338326261
-33636536663161366635666264663539653037306339316233643662643134396636636162656333
-36666139376263646130333263653335333165356462363434373439313330383331356138333431
-31633362343639376436616339656561316433346532346533336261383433366366396261366134
-35363264373335616165643665653466613434386630373232386261393464376361313131386462
-33333531336334386562356338623233313862316232356562373561633364363263306465333439
-37386631626538636365376464653837333662363361653237366161316431653266643238346336
-363863376530613036313866323965326638

group_vars/proxmox/vars.yml

@@ -1,3 +0,0 @@
-proxmox_api_user: root
-proxmox_api_host: 192.168.20.12
-proxmox_api_password: "{{ vault.pve.aya01.root.sudo }}"

inventory/docker.ini

@@ -1,13 +0,0 @@
-[docker_host]
-docker-host01 ansible_become_pass: "{{ vault.docker.host01.sudo }}"
-docker-host10
-docker-host12
-
-[docker_lb]
-docker-lb ansible_become_pass: "{{ vault.docker.lb.sudo }}"
-
-[docker]
-
-[docker:children]
-docker_host
-docker_lb

playbooks/docker-host.yml

@@ -2,8 +2,6 @@
 - name: Set up Servers
   hosts: docker_host
   gather_facts: true
-  vars_files:
-    - secrets.yml
   roles:
     - role: common
       tags:

playbooks/docker-lb.yml

@@ -2,8 +2,6 @@
 - name: Set up reverse proxy for docker
   hosts: docker
   gather_facts: true
-  vars_files:
-    - secrets.yml
   roles:
     - role: common
       tags:

playbooks/k3s-agents.yml

@@ -1,19 +1,20 @@
 - name: Set up Agents
-  hosts: k3s_nodes
-  gather_facts: yes
-  vars_files:
-    - secrets.yml
+  hosts: k3s
+  gather_facts: true
+  vars:
+    k3s_primary_server_ip: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_host') | list | first }}"
   pre_tasks:
     - name: Get K3s token from the first server
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
       slurp:
         src: /var/lib/rancher/k3s/server/node-token
       register: k3s_token
       become: true
 
-    - name: Set fact on k3s.server.ips[0]
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
-      set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}"
+    - name: Set fact on k3s_primary_server_ip
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
+      set_fact:
+        k3s_token: "{{ k3s_token['content'] | b64decode | trim }}"
 
   roles:
     - role: common
@@ -22,7 +23,7 @@
         - common
     - role: k3s_agent
       when: inventory_hostname in groups["k3s_agent"]
-      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}"
+      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s_primary_server_ip ) | select() | first | items2dict).host.hostname].k3s_token }}"
       tags:
         - k3s_agent
     - role: node_exporter

playbooks/k3s-loadbalancer.yml

@@ -0,0 +1,17 @@
+---
+- name: Set up Servers
+  hosts: k3s
+  gather_facts: true
+  roles:
+    - role: common
+      tags:
+        - common
+      when: inventory_hostname in groups["k3s_loadbalancer"]
+    - role: k3s_loadbalancer
+      tags:
+        - k3s_loadbalancer
+      when: inventory_hostname in groups["k3s_loadbalancer"]
+    # - role: node_exporter
+    #   tags:
+    #     - node_exporter
+    #   when: inventory_hostname in groups["k3s_loadbalancer"]

playbooks/k3s-servers.yml

@@ -1,9 +1,7 @@
 ---
 - name: Set up Servers
-  hosts: k3s_server
-  gather_facts: yes
-  vars_files:
-    - secrets.yml
+  hosts: k3s
+  gather_facts: true
   roles:
     - role: common
       tags:
@@ -11,6 +9,7 @@
     - role: k3s_server
       tags:
         - k3s_server
+      when: inventory_hostname in groups["k3s_server"]
     - role: node_exporter
       tags:
         - node_exporter

playbooks/k3s-storage.yml

@@ -1,19 +1,20 @@
 - name: Set up storage
   hosts: k3s_nodes
-  gather_facts: yes
-  vars_files:
-    - secrets.yml
+  gather_facts: true
+  vars:
+    k3s_primary_server_ip: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_host') | list | first }}"
   pre_tasks:
     - name: Get K3s token from the first server
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
       slurp:
         src: /var/lib/rancher/k3s/server/node-token
       register: k3s_token
       become: true
 
-    - name: Set fact on k3s.server.ips[0]
-      when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"]
-      set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}"
+    - name: Set fact on k3s_primary_server_ip
+      when: host.ip == k3s_primary_server_ip and inventory_hostname in groups["k3s_server"]
+      set_fact:
+        k3s_token: "{{ k3s_token['content'] | b64decode | trim }}"
 
   roles:
     - role: common
@@ -22,7 +23,7 @@
         - common
     - role: k3s_storage
       when: inventory_hostname in groups["k3s_storage"]
-      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}"
+      k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s_primary_server_ip ) | select() | first | items2dict).host.hostname].k3s_token }}"
       tags:
         - k3s_storage
     - role: node_exporter

playbooks/loadbalancer.yml

@@ -1,16 +0,0 @@
----
-- name: Set up Servers
-  hosts: loadbalancer
-  gather_facts: yes
-  vars_files:
-    - secrets.yml
-  roles:
-    - role: common
-      tags:
-        - common
-    - role: loadbalancer
-      tags:
-        - loadbalancer
-    - role: node_exporter
-      tags:
-        - node_exporter

playbooks/proxmox.yml

@@ -2,8 +2,6 @@
 - name: Run proxmox vm playbook
   hosts: proxmox
   gather_facts: true
-  vars_files:
-    - secrets.yml
   vars:
     is_localhost: "{{ inventory_hostname == '127.0.0.1' }}"
     is_proxmox_node: "{{ 'proxmox_nodes' in group_names }}"

roles/common/tasks/time.yml

@@ -1,11 +1,11 @@
 ---
-- name: Set timezone to "{{ timezone }}"
+- name: Set timezone
   community.general.timezone:
     name: "{{ timezone }}"
   become: true
   when: ansible_user_id != "root"
 
-- name: Set timezone to "{{ timezone }}"
+- name: Set timezone
   community.general.timezone:
     name: "{{ timezone }}"
   when: ansible_user_id == "root"

roles/k3s_agent/tasks/installation.yml

@@ -16,6 +16,6 @@
   ansible.builtin.command: |
     /tmp/k3s_install.sh
   environment:
-    K3S_URL: "https://{{ k3s.loadbalancer.ip }}:{{ k3s.loadbalancer.default_port }}"
+    K3S_URL: "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}"
     K3S_TOKEN: "{{ k3s_token }}"
   become: true

roles/k3s_loadbalancer/tasks/configuration.yml

@@ -2,7 +2,7 @@
 - name: Template the nginx config file with dynamic upstreams
   ansible.builtin.template:
     src: templates/nginx.conf.j2
-    dest: "{{ nginx_config_path }}"
+    dest: "{{ k3s_loadbalancer_nginx_config_path }}"
     owner: root
     group: root
     mode: "0644"
@@ -10,7 +10,7 @@
   notify:
     - Restart nginx
   vars:
-    k3s_server_ips: "{{ k3s.server.ips }}"
+    k3s_server_ips: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | unique | list }}"
 
 - name: Enable nginx
   ansible.builtin.systemd:

roles/k3s_loadbalancer/templates/nginx.conf.j2

@@ -0,0 +1,87 @@
+include /etc/nginx/modules-enabled/*.conf;
+
+events {}
+
+stream {
+# TCP Load Balancing for the K3s API
+  upstream k3s_servers {
+    {% for ip in k3s_server_ips %}
+    server {{ ip }}:{{ k3s.loadbalancer.default_port }};
+    {% endfor %}
+  }
+
+  server {
+    listen {{k3s.loadbalancer.default_port}};
+    proxy_pass k3s_servers;
+  }
+
+  upstream dns_servers {
+    {% for ip in k3s_server_ips %}
+    server {{ ip }}:53;
+    {% endfor %}
+  }
+
+  server {
+    listen 53 udp;
+    proxy_pass dns_servers;
+  }
+}
+
+# http {
+#   upstream k3s_servers_http {
+#     least_conn;
+#     {% for ip in k3s_server_ips %}
+#     server {{ ip }}:80;
+#     {% endfor %}
+#   }
+#
+#   upstream k3s_servers_https {
+#     least_conn;
+#     {% for ip in k3s_server_ips %}
+#     server {{ ip }}:443;
+#     {% endfor %}
+#   }
+#
+#   server {
+#     listen 80;
+#
+#     location / {
+#       proxy_pass http://k3s_servers_http;
+#       proxy_set_header Host $http_host;
+#       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#       proxy_set_header X-Forwarded-Proto http;
+#     }
+#   }
+#
+#   server {
+#     listen 443 ssl;
+#
+#     server_name staging.k3s.seyshiro.de *.staging.k3s.seyshiro.de;
+#
+#     ssl_certificate /etc/nginx/ssl/staging_tls.crt;
+#     ssl_certificate_key /etc/nginx/ssl/staging_tls.key;
+#
+#     location / {
+#       proxy_pass https://k3s_servers_https;
+#       proxy_set_header Host $host;
+#       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#       proxy_set_header X-Forwarded-Proto https;
+#     }
+#   }
+#
+#   server {
+#     listen 443 ssl;
+#
+#     server_name k3s.seyshiro.de *.k3s.seyshiro.de;
+#
+#     ssl_certificate /etc/nginx/ssl/production_tls.crt;
+#     ssl_certificate_key /etc/nginx/ssl/production_tls.key;
+#
+#     location / {
+#       proxy_pass https://k3s_servers_https;
+#       proxy_set_header Host $host;
+#       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#       proxy_set_header X-Forwarded-Proto https;
+#     }
+#   }
+# }

roles/k3s_loadbalancer/vars/main.yml

@@ -0,0 +1 @@
+k3s_loadbalancer_nginx_config_path: "/etc/nginx/nginx.conf"

roles/k3s_server/tasks/installation.yml

@@ -16,7 +16,7 @@
   ansible.builtin.command: |
     /tmp/k3s_install.sh server \
     --node-taint CriticalAddonsOnly=true:NoExecute \
-    --tls-san {{ k3s.loadbalancer.ip }}
+    --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}
   become: true
   async: 300
   poll: 0

roles/k3s_storage/tasks/installation.yml

@@ -18,6 +18,6 @@
     --node-taint storage=true:NoExecute \
     --node-label longhorn=true
   environment:
-    K3S_URL: "https://{{ k3s.loadbalancer.ip }}:{{ k3s.loadbalancer.default_port }}"
+    K3S_URL: "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}"
     K3S_TOKEN: "{{ k3s_token }}"
   become: true

roles/loadbalancer/templates/nginx.conf.j2

@@ -1,89 +0,0 @@
-include /etc/nginx/modules-enabled/*.conf;
-
-events {}
-
-stream {
-# TCP Load Balancing for the K3s API
-  upstream k3s_servers {
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:{{k3s.loadbalancer.default_port}};
-    {% endfor %}
-  }
-
-  server {
-    listen {{k3s.loadbalancer.default_port}};
-    proxy_pass k3s_servers;
-  }
-
-  upstream dns_servers {
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:53;
-    {% endfor %}
-  }
-
-  server {
-    listen 53 udp;
-    proxy_pass dns_servers;
-  }
-}
-
-http {
-  upstream k3s_servers_http {
-    least_conn;
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:80;
-    {% endfor %}
-  }
-
-  upstream k3s_servers_https {
-    least_conn;
-    {% for ip in k3s_server_ips %}
-    server {{ ip }}:443;
-    {% endfor %}
-  }
-
-  server {
-    listen 80;
-
-    location / {
-      proxy_pass http://k3s_servers_http;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto http;
-    }
-  }
-
-  server {
-    listen 443 ssl;
-
-    server_name staging.k3s.seyshiro.de *.staging.k3s.seyshiro.de;
-
-    ssl_certificate /etc/nginx/ssl/staging_tls.crt;
-    ssl_certificate_key /etc/nginx/ssl/staging_tls.key;
-
-    location / {
-      proxy_pass https://k3s_servers_https;
-      proxy_set_header Host $host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto https;
-    }
-  }
-
-  server {
-    listen 443 ssl;
-
-    server_name k3s.seyshiro.de *.k3s.seyshiro.de;
-
-    ssl_certificate /etc/nginx/ssl/production_tls.crt;
-    ssl_certificate_key /etc/nginx/ssl/production_tls.key;
-
-    location / {
-      proxy_pass https://k3s_servers_https;
-      proxy_set_header Host $host;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto https;
-    }
-  }
-}
-
-

roles/loadbalancer/vars/main.yml

@@ -1 +0,0 @@
-nginx_config_path: "/etc/nginx/nginx.conf"

roles/proxmox/tasks/54_destroy_vm.yml

@@ -1,16 +1,18 @@
 ---
 - name: Gather info about VM
   community.general.proxmox_vm_info:
-    api_user: root@pam
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
+    api_user: "{{ proxmox_api_user }}@pam"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token_secret }}"
     api_host: "192.168.20.12"
     vmid: "{{ vm.vmid }}"
   register: vm_info
 
 - name: Stop VM
   community.general.proxmox_kvm:
-    api_user: root@pam
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
+    api_user: "{{ proxmox_api_user }}@pam"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token_secret }}"
     api_host: "192.168.20.12"
     node: "{{ vm.node }}"
     vmid: "{{ vm.vmid }}"
@@ -20,8 +22,9 @@
 
 - name: Destroy VM
   community.general.proxmox_kvm:
-    api_user: root@pam
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
+    api_user: "{{ proxmox_api_user }}@pam"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token_secret }}"
     api_host: "192.168.20.12"
     node: "{{ vm.node }}"
     vmid: "{{ vm.vmid }}"

roles/proxmox/tasks/55_create_vm.yml

@@ -2,7 +2,8 @@
 - name: Create VM
   community.general.proxmox_kvm:
     api_user: "{{ proxmox_api_user }}@pam"
-    api_password: "{{ proxmox_api_password }}"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token_secret }}"
     api_host: "{{ proxmox_api_host }}"
     agent: true
     name: "{{ vm.name }}"
@@ -11,7 +12,7 @@
     cores: "{{ vm.cores }}"
     memory: "{{ vm.memory }}"
     net: "{{ vm.net }}"
-    hostpci: "{{ vm.hostpci | default({})}}"
+    hostpci: "{{ vm.hostpci | default({}) }}"
     scsihw: "virtio-scsi-pci"
     ostype: "l26"
     tags: "{{ proxmox_tags }}"
@@ -19,8 +20,7 @@
     boot: "order=scsi0"
     cpu: "x86-64-v2-AES"
     ciuser: "{{ vm.ciuser }}"
-    # cipassword: "{{ vm_secrets[proxmox_secrets_prefix + '_' + vm.name.replace('-', '_')] }}"
-    cipassword: "flyff369"
+    cipassword: "{{ vm_secrets[proxmox_secrets_prefix + '_' + vm.name.replace('-', '_')] }}"
     ipconfig:
       ipconfig0: "ip=dhcp"
     sshkeys: "{{ vm.sshkeys }}"

roles/proxmox/tasks/56_provision_new_vm.yml

@@ -25,39 +25,26 @@
 
 - name: Start VM
   community.general.proxmox_kvm:
-    api_user: root@pam
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
+    api_user: "{{ proxmox_api_user }}@pam"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token_secret }}"
     api_host: "192.168.20.12"
     node: "{{ vm.node }}"
     vmid: "{{ vm.vmid }}"
     state: started
 
-- name: Stop VM
-  community.general.proxmox_kvm:
-    api_user: root@pam
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
-    api_host: "192.168.20.12"
-    node: "{{ vm.node }}"
-    vmid: "{{ vm.vmid }}"
-    state: stopped
-    force: true
+- name: Retry stopping VM
+  ansible.builtin.include_tasks: ./57_stop_and_verify_vm.yml
 
-- name: Wait until VM is fully stopped
-  community.general.proxmox_vm_info:
-    api_user: "root@pam"
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
-    api_host: "192.168.20.12"
-    node: "{{ vm.node }}"
-    vmid: "{{ vm.vmid }}"
-  register: vm_status_check
-  until: vm_status_check.proxmox_vms[0].status == "stopped"
-  retries: 24
-  delay: 5
+- name: Pause for 5 seconds for api
+  ansible.builtin.pause:
+    seconds: 5
 
 - name: Start VM
   community.general.proxmox_kvm:
-    api_user: root@pam
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
+    api_user: "{{ proxmox_api_user }}@pam"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token_secret }}"
     api_host: "192.168.20.12"
     node: "{{ vm.node }}"
     vmid: "{{ vm.vmid }}"
@@ -91,11 +78,11 @@
         ProxyJump {{ vm.node }}
         StrictHostKeyChecking no
 
-- name: Add VM to homelab_vms group in production.ini
-  ansible.builtin.lineinfile:
-    path: "{{ inventory_file }}"
-    line: "{{ vm.name }}"
-    insertafter: '^\[vms\]'
-    create: true
-    state: present
-  delegate_to: localhost
+# - name: Add VM to homelab_vms group in production.ini
+#   ansible.builtin.lineinfile:
+#     path: "{{ inventory_file }}"
+#     line: "{{ vm.name }}"
+#     insertafter: '^\[vms\]'
+#     create: true
+#     state: present
+#   delegate_to: localhost

roles/proxmox/tasks/57_stop_and_verify_vm.yml

@@ -0,0 +1,39 @@
+- name: "Wait until success"
+  block:
+    - name: Set the retry count
+      set_fact:
+        retry_count: "{{ 0 if retry_count is undefined else retry_count | int + 1 }}"
+
+    - name: Stop VM
+      community.general.proxmox_kvm:
+        api_user: "{{ proxmox_api_user }}@pam"
+        api_token_id: "{{ proxmox_api_token_id }}"
+        api_token_secret: "{{ proxmox_api_token_secret }}"
+        api_host: "192.168.20.12"
+        node: "{{ vm.node }}"
+        vmid: "{{ vm.vmid }}"
+        state: stopped
+        force: true
+
+    - name: Wait until VM is fully stopped
+      community.general.proxmox_vm_info:
+        api_user: "{{ proxmox_api_user }}@pam"
+        api_token_id: "{{ proxmox_api_token_id }}"
+        api_token_secret: "{{ proxmox_api_token_secret }}"
+        api_host: "192.168.20.12"
+        node: "{{ vm.node }}"
+        vmid: "{{ vm.vmid }}"
+      register: vm_status_check
+      failed_when: vm_status_check.proxmox_vms[0].status != "stopped"
+  rescue:
+    - name: Check for retry count
+      fail:
+        msg: Ended after 24 retries
+      when: retry_count|int == 24
+
+    - name: Wait 5s
+      ansible.builtin.pause:
+        seconds: 5
+
+    - name: "Failed to stop VM - Retrying..."
+      include_tasks: ./57_stop_and_verify_vm.yml

roles/proxmox/vars/main.yml

@@ -3,7 +3,7 @@ proxmox_creator: ansible
 
 proxmox_storage: proxmox
 
-proxmox_vault_file: ../group_vars/proxmox/secrets_vm.yml
+proxmox_vault_file: ../vars/group_vars/proxmox/secrets_vm.yml
 proxmox_secrets_prefix: secrets_vm
 proxmox_cloud_init_images:
   debian:

vars/docker.ini

@@ -0,0 +1,13 @@
+[docker_host]
+docker-host01 ansible_become_pass="{{ vault.docker.host01.sudo }}"
+docker-host10
+docker-host12
+
+[docker_lb]
+docker-lb ansible_become_pass="{{ vault.docker.lb.sudo }}"
+
+[docker]
+
+[docker:children]
+docker_host
+docker_lb

vars/group_vars/all/vars.yml

@@ -2,16 +2,16 @@
 # Essential
 #
 
-root: root
-user: tudattr
-timezone: Europe/Berlin
+root: "root"
+user: "tudattr"
+timezone: "Europe/Berlin"
 puid: "1000"
 pgid: "1000"
 pk_path: "/media/veracrypt1/genesis"
 pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKqc9fnzfCz8fQDFzla+D8PBhvaMmFu2aF+TYkkZRxl9 tuan@genesis-2022-01-20"
 
-public_domain: tudattr.dev
-internal_domain: seyshiro.de
+public_domain: "tudattr.dev"
+internal_domain: "seyshiro.de"
 
 #
 # Packages

vars/group_vars/k3s/vars.yml

@@ -0,0 +1,7 @@
+k3s:
+  server:
+    ips: []
+  loadbalancer:
+    default_port: 6443
+  agent:
+    ips: []

vars/group_vars/proxmox/secrets.yml

@@ -0,0 +1,16 @@
+$ANSIBLE_VAULT;1.1;AES256
+35336335313463633337373430646432306364613234666463373135306263383932323266303834
+3033643661303537303332316361326464336136623139350a373137396165623861623433303031
+37303264373362313534623966626665633339623464376236323436336563376261323739623033
+3066663137653562320a616130653165326530643562646531373736313064626164653661353535
+37633031626462663636366464323963653535333235633939636636376436646164333965326636
+62313164336265336539333261333732626562663966306537353763333339353030666133633064
+33336230646435616166346639363835373562313265306332346662636364326337616637346333
+39343063356138326536653933656164616264666662396132383865343630383139326531616464
+64333561313631616261303431336265623166386131613634646337396332653239323262343961
+66303938323337656662303562613736366366616663633639646566333737393765626365383963
+34616166336465376331366465303230666435626463383031653661376233626538353830356366
+34633239326532303931663435363365396535393733383637656139336164306663623761386135
+31313630383139376661343334616533316231393438663837383861313734313837623063366135
+64356334336133303164656338303339623631313461353139363838356337636462363862303436
+336363363733363436356663323962383030

vars/group_vars/proxmox/secrets_vm.yml

@@ -0,0 +1,34 @@
+$ANSIBLE_VAULT;1.1;AES256
+64336139336538333337376465316164383766643666336666643166333134636338323562303364
+6235613337366634613532373933396230666137373562650a643633306165643331643464633762
+35336433626161393735353133343739353738653061613733393135313061643663616665316463
+6238376435633435650a306636303934383739656439383632313964356434353536373961646531
+35303533666633346363663936366535613039356164383362393736306338613236373138663731
+65666635353734353261333332393962636664653332313062336239313834653536363539306630
+61316431313631643637616434376334323232306232363936613139373762613862653938373461
+34366363643337326439633963303430613935323866343764326639663531303931396235643231
+36346463653866653137653931303439326433366231303530316632613033333761326536326335
+30343233333232333434303562396166386133313633323732636532376539633336613532633765
+66656663353964316364636236623133306533656465303833346563376461396639626262333133
+33663966393030653762636164653534363338613536636432663938393033313933323830336538
+61663865353466393836333539636466613137396430636566303135326565383764373831336532
+66626332383065643636663638616337316136623131333630613861353730646339366239633861
+31343133346138343637373039633930653731396537323438623237393436303063623862663965
+65353332393331623933323138633231363539323834333631643337613863643737306363323135
+61353663643563393539373839643462616339333762353962653065653134653063336466343431
+61313262616631343265386530653431356632616230633032363165656666333662636339306539
+37646634353961346165356565313038303333303564333862323766366238366434643562306262
+38656532333339643335386130356637353434393037636530363233393162663330663566663962
+34343333383631343330663962343639633464353961343933653764643666626631346434366365
+37303433626330346630353064613766303634386238636230346531663038653865393939663732
+37613461313738313766306663653264616563633966316362356539373239663464386430636464
+61373864313064626133623332643139336163643465376234373530666630656361616236336130
+37623962393237623135656534613839363831613165356563333039366462306230636432653636
+64333633393532313635323830333432666134373630666561626231666433303132663939633965
+61373137633865323564343661623039616331323164396133343165656263383865383861616262
+64636230336130356364333964336335656664303334326537303033613331353038353666646463
+63363631613238633831666136363833363964356432373434643131653531666166666233613861
+30306435306563303333343364333065616438383331383437353234323633393733653965313165
+30643539663330356630363833643136643265623966636466336539353738373136616265393265
+36613564653634313438666334313636653435336263393635656138343534336232346332356264
+33366232613832643862386532663264353735393033303864356230333864363366

vars/group_vars/proxmox/vars.yml

@@ -0,0 +1,4 @@
+proxmox_api_host: 192.168.20.12
+proxmox_api_user: root
+proxmox_api_token_id: terraform
+proxmox_api_token_secret: "{{ vault.pve.api.token_secret }}"

vars/group_vars/proxmox/vms.yml

@@ -45,36 +45,47 @@ vms:
     ciuser: "{{ user }}"
     sshkeys: "{{ pubkey }}"
     disk_size: 64 # in Gb
-#  - name: "k3s-agent10"
-#    node: "naruto01"
-#    vmid: 210
-#    cores: 2
-#    memory: 4096 # in MiB
-#    net:
-#      net0: "virtio,bridge=vmbr0,firewall=1"
-#    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
-#    ciuser: "{{ user }}"
-#    sshkeys: "{{ pubkey }}"
-#    disk_size: 50 # in Gb
-#  - name: "k3s-agent11"
-#    node: "lulu"
-#    vmid: 211
-#    cores: 2
-#    memory: 4096 # in MiB
-#    net:
-#      net0: "virtio,bridge=vmbr0,firewall=1"
-#    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
-#    ciuser: "{{ user }}"
-#    sshkeys: "{{ pubkey }}"
-#    disk_size: 128 # in Gb
-#  - name: "k3s-agent12"
-#    node: "inko"
-#    vmid: 212
-#    cores: 2
-#    memory: 4096 # in MiB
-#    net:
-#      net0: "virtio,bridge=vmbr0,firewall=1"
-#    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
-#    ciuser: "{{ user }}"
-#    sshkeys: "{{ pubkey }}"
-#    disk_size: 128 # in Gb
+  - name: "k3s-agent10"
+    node: "naruto01"
+    vmid: 210
+    cores: 2
+    memory: 4096 # in MiB
+    net:
+      net0: "virtio,bridge=vmbr0,firewall=1"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    ciuser: "{{ user }}"
+    sshkeys: "{{ pubkey }}"
+    disk_size: 64 # in Gb
+  - name: "k3s-agent11"
+    node: "lulu"
+    vmid: 211
+    cores: 2
+    memory: 4096 # in MiB
+    net:
+      net0: "virtio,bridge=vmbr0,firewall=1"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    ciuser: "{{ user }}"
+    sshkeys: "{{ pubkey }}"
+    disk_size: 64 # in Gb
+  - name: "k3s-agent12"
+    node: "inko"
+    vmid: 212
+    cores: 2
+    memory: 4096 # in MiB
+    net:
+      net0: "virtio,bridge=vmbr0,firewall=1"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    ciuser: "{{ user }}"
+    sshkeys: "{{ pubkey }}"
+    disk_size: 64 # in Gb
+  - name: "k3s-loadbalancer"
+    node: "naruto01"
+    vmid: 150
+    cores: 1
+    memory: 2048 # in MiB
+    net:
+      net0: "virtio,bridge=vmbr0,firewall=1"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    ciuser: "{{ user }}"
+    sshkeys: "{{ pubkey }}"
+    disk_size: 32 # in Gb

vars/k3s.ini

@@ -4,7 +4,6 @@
 k3s_server
 k3s_agent
 k3s_storage
-k3s_storage
 k3s_loadbalancer
 
 [k3s_server]
@@ -13,9 +12,6 @@ k3s-server10
 [k3s_agent]
 k3s-agent[10:12]
 
-[k3s_storage]
-k3s-longhorn[10:12]
-
 [k3s_loadbalancer]
 k3s-loadbalancer