feat(proxmox): add k3s agents and refine VM provisioning

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

production.ini

@@ -1,63 +0,0 @@
-[proxmox]
-127.0.0.1 ansible_connection=local
-
-[proxmox:children]
-proxmox_nodes
-
-[proxmox_nodes]
-aya01
-lulu
-inko
-naruto01
-
-[k3s]
-k3s-postgres
-k3s-loadbalancer
-k3s-server[00:02]
-k3s-agent[00:02]
-k3s-longhorn[00:02]
-
-[vm]
-k3s-postgres
-k3s-loadbalancer
-k3s-agent[00:02]
-k3s-server[00:02]
-k3s-longhorn[00:02]
-# docker-host[00:01]
-
-[k3s_nodes]
-k3s-server[00:02]
-k3s-agent[00:02]
-k3s-longhorn[00:02]
-
-[docker]
-docker-host01
-docker-host10
-docker-host12
-docker-lb
-
-[vps]
-mii
-
-[k3s_server]
-k3s-server[00:02]
-
-[k3s_agent]
-k3s-agent[00:02]
-
-[k3s_storage]
-k3s-longhorn[00:02]
-
-[db]
-k3s-postgres
-
-[loadbalancer]
-k3s-loadbalancer
-
-[docker_host]
-docker-host01
-docker-host10
-docker-host12
-
-[docker_lb]
-docker-lb

roles/proxmox/tasks/55_create_vm.yml

@@ -20,8 +20,7 @@
     boot: "order=scsi0"
     cpu: "x86-64-v2-AES"
     ciuser: "{{ vm.ciuser }}"
-    # cipassword: "{{ vm_secrets[proxmox_secrets_prefix + '_' + vm.name.replace('-', '_')] }}"
+    cipassword: "{{ vm_secrets[proxmox_secrets_prefix + '_' + vm.name.replace('-', '_')] }}"
-    cipassword: "flyff369"
     ipconfig:
       ipconfig0: "ip=dhcp"
     sshkeys: "{{ vm.sshkeys }}"

roles/proxmox/tasks/56_provision_new_vm.yml

@@ -33,29 +33,12 @@
     vmid: "{{ vm.vmid }}"
     state: started
 
-- name: Stop VM
+- name: Retry stopping VM
-  community.general.proxmox_kvm:
+  ansible.builtin.include_tasks: ./57_stop_and_verify_vm.yml
-    api_user: "{{ proxmox_api_user }}@pam"
-    api_token_id: "{{ proxmox_api_token_id }}"
-    api_token_secret: "{{ proxmox_api_token_secret }}"
-    api_host: "192.168.20.12"
-    node: "{{ vm.node }}"
-    vmid: "{{ vm.vmid }}"
-    state: stopped
-    force: true
 
-- name: Wait until VM is fully stopped
+- name: Pause for 5 seconds for api
-  community.general.proxmox_vm_info:
+  ansible.builtin.pause:
-    api_user: "{{ proxmox_api_user }}@pam"
+    seconds: 5
-    api_token_id: "{{ proxmox_api_token_id }}"
-    api_token_secret: "{{ proxmox_api_token_secret }}"
-    api_host: "192.168.20.12"
-    node: "{{ vm.node }}"
-    vmid: "{{ vm.vmid }}"
-  register: vm_status_check
-  until: vm_status_check.proxmox_vms[0].status == "stopped"
-  retries: 24
-  delay: 5
 
 - name: Start VM
   community.general.proxmox_kvm:

roles/proxmox/tasks/57_stop_and_verify_vm.yml

@@ -0,0 +1,39 @@
+- name: "Wait until success"
+  block:
+    - name: Set the retry count
+      set_fact:
+        retry_count: "{{ 0 if retry_count is undefined else retry_count | int + 1 }}"
+
+    - name: Stop VM
+      community.general.proxmox_kvm:
+        api_user: "{{ proxmox_api_user }}@pam"
+        api_token_id: "{{ proxmox_api_token_id }}"
+        api_token_secret: "{{ proxmox_api_token_secret }}"
+        api_host: "192.168.20.12"
+        node: "{{ vm.node }}"
+        vmid: "{{ vm.vmid }}"
+        state: stopped
+        force: true
+
+    - name: Wait until VM is fully stopped
+      community.general.proxmox_vm_info:
+        api_user: "{{ proxmox_api_user }}@pam"
+        api_token_id: "{{ proxmox_api_token_id }}"
+        api_token_secret: "{{ proxmox_api_token_secret }}"
+        api_host: "192.168.20.12"
+        node: "{{ vm.node }}"
+        vmid: "{{ vm.vmid }}"
+      register: vm_status_check
+      failed_when: vm_status_check.proxmox_vms[0].status != "stopped"
+  rescue:
+    - name: Check for retry count
+      fail:
+        msg: Ended after 24 retries
+      when: retry_count|int == 24
+
+    - name: Wait 5s
+      ansible.builtin.pause:
+        seconds: 5
+
+    - name: "Failed to stop VM - Retrying..."
+      include_tasks: ./57_stop_and_verify_vm.yml

vars/group_vars/proxmox/secrets_vm.yml

@@ -1,20 +1,23 @@
 $ANSIBLE_VAULT;1.1;AES256
-35616266333838306161336339353538306634373132626132643732303066303163343630333630
+31643231626635633436363136386537616133326538323239663963346332383961396132316662
-6338393762616262303038373334663230383464643836370a656538393531393134616463643239
+3938393638646562306634333932666663363363353264620a613833666634383061343565613364
-36383330653339393362353838313639333432643535643833396535653632376336613130646663
+38343537333930303563613839303265373339616463626133646365643630313339633765333231
-3532646538363137630a363731613235653935316531616430346264643837306434386333373033
+6236306463616565350a626235666164303737646338363232336363336539656439316462643332
-33663135653931373963343734366562386263663939383536663439383537333264666233343233
+36346530306266616465643766333864356264386435383633356534663438376335643630613230
-62626162666538333435396638393338393734656131303065616534613733353335643939333765
+64313663613332666534623433653539653234646661636230616134353336663631313661333661
-38326237343337363064666530303664326563633262313432343030336266373437353837346461
+32666632363765613934353536343339306632666238626330663938313030633362316661656432
-63333363626164316638346635666537613963383537313965373638303732353365623166363736
+64393863356336343261663935373530346162323665303632646531613530393432393332663963
-31633239646262613539646637663664313337353465636366313338303439613638653530656631
+62303663613766613830383735643839353039663631333231343036636537643237643932656162
-62396536316561623736633631623336313537646138383431633538303163303261323864383538
+32396632316263646637653562386438613930313331653261373363386134663835313762646136
-38626338373332653561343036323236383337343037356366626230646432646538373836303063
+37623237636464613736353237313666656234303534623961666230393530386435393734376639
-61346339376561626630653562346439306561643664666437386562356535303264646338326261
+39366636623132326230396635376136383634306664336332663535366230653632613935383135
-33636536663161366635666264663539653037306339316233643662643134396636636162656333
+31383232386633666263666439306631373663613930623762343635376261316136656539323631
-36666139376263646130333263653335333165356462363434373439313330383331356138333431
+64393062623461383733316231633335303535363763633737373933656563623234353930323262
-31633362343639376436616339656561316433346532346533336261383433366366396261366134
+33623463613638306630653639646230396539383065336166643935346435363534353836626262
-35363264373335616165643665653466613434386630373232386261393464376361313131386462
+62393262646262323433373561303132336564353062396331623264333464346534626633333561
-33333531336334386562356338623233313862316232356562373561633364363263306465333439
+30636462646664656532393139636331376534643234663566633862373263306365643336343039
-37386631626538636365376464653837333662363361653237366161316431653266643238346336
+64343236303139626164656139613438623030633735666130346335626530373636666534616233
-363863376530613036313866323965326638
+65626166386537333162393962666461613266366261316339356665643765376666393965613835
+65376561383865336165343662333236653537666563613730666461633233356166653964333164
+35646264653062396330373135363338346138353136626661643531323961316231356262363966
+3636356230396130663531353437653034396534313863336134

vars/group_vars/proxmox/vars.yml

@@ -1,4 +1,4 @@
 proxmox_api_host: 192.168.20.12
 proxmox_api_user: root
-proxmox_api_token_id: root@pam!terraform
+proxmox_api_token_id: terraform
 proxmox_api_token_secret: "{{ vault.pve.api.token_secret }}"

vars/group_vars/proxmox/vms.yml

@@ -45,36 +45,36 @@ vms:
     ciuser: "{{ user }}"
     sshkeys: "{{ pubkey }}"
     disk_size: 64 # in Gb
-#  - name: "k3s-agent10"
+  - name: "k3s-agent10"
-#    node: "naruto01"
+    node: "naruto01"
-#    vmid: 210
+    vmid: 210
-#    cores: 2
+    cores: 2
-#    memory: 4096 # in MiB
+    memory: 4096 # in MiB
-#    net:
+    net:
-#      net0: "virtio,bridge=vmbr0,firewall=1"
+      net0: "virtio,bridge=vmbr0,firewall=1"
-#    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
-#    ciuser: "{{ user }}"
+    ciuser: "{{ user }}"
-#    sshkeys: "{{ pubkey }}"
+    sshkeys: "{{ pubkey }}"
-#    disk_size: 64 # in Gb
+    disk_size: 64 # in Gb
-#  - name: "k3s-agent11"
+  - name: "k3s-agent11"
-#    node: "lulu"
+    node: "lulu"
-#    vmid: 211
+    vmid: 211
-#    cores: 2
+    cores: 2
-#    memory: 4096 # in MiB
+    memory: 4096 # in MiB
-#    net:
+    net:
-#      net0: "virtio,bridge=vmbr0,firewall=1"
+      net0: "virtio,bridge=vmbr0,firewall=1"
-#    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
-#    ciuser: "{{ user }}"
+    ciuser: "{{ user }}"
-#    sshkeys: "{{ pubkey }}"
+    sshkeys: "{{ pubkey }}"
-#    disk_size: 64 # in Gb
+    disk_size: 64 # in Gb
-#  - name: "k3s-agent12"
+  - name: "k3s-agent12"
-#    node: "inko"
+    node: "inko"
-#    vmid: 212
+    vmid: 212
-#    cores: 2
+    cores: 2
-#    memory: 4096 # in MiB
+    memory: 4096 # in MiB
-#    net:
+    net:
-#      net0: "virtio,bridge=vmbr0,firewall=1"
+      net0: "virtio,bridge=vmbr0,firewall=1"
-#    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
+    boot_image: "{{ proxmox_cloud_init_images.debian.name }}"
-#    ciuser: "{{ user }}"
+    ciuser: "{{ user }}"
-#    sshkeys: "{{ pubkey }}"
+    sshkeys: "{{ pubkey }}"
-#    disk_size: 64 # in Gb
+    disk_size: 64 # in Gb