feat(proxmox): refactor vm provisioning and add pci passthrough config

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

group_vars/proxmox/secrets_vm.yml

@@ -1,8 +1,8 @@
 $ANSIBLE_VAULT;1.1;AES256
-33333937646463646566653162383830616434336437623065363665323739633331346266333763
+62653436363035633565383636383931353765663136646362366439306635306430313763323331
-3364663264306665626465666133666161626333323462650a353366303331303837316133326135
+3533346430316564356463613664366261336139636331320a636532633836303161396238663163
-33623862333036633438343538633161643333663632303362396438316638626338663935353337
+39643765613162346261643662333633323133373830313365326534626161326235363038383462
-3532323337663864640a333765653732393937396561373361393762386565353266343537306161
+6531643136646464610a383532316434383264326665613436623331333730633035316530663031
-62303539333837666365323630303836373065343437663433616664376432313135636266663764
+63343539393062383065396638363064613932363164346632366134333637343337353033346131
-36616132383330656165656264346231323039626131646432323935306233643866366439313962
+30613162303536313366656137306165303032636366376362656137343235313838356463306532
-353837396234643739346662316239356134
+653164653834613431633563633739313936

production.ini

@@ -56,6 +56,3 @@ docker-host[00:01]
 
 [docker_lb]
 docker-lb
-
-[vm:vars]
-ansible_ssh_common_args='-o ProxyCommand="ssh -p 22 -W %h:%p -q aya01"'

roles/docker_host/tasks/setup.yml

@@ -1,9 +1,8 @@
 ---
-- name: Enable HW accelerate for VM
+- name: Setup VM Packages
   ansible.builtin.apt:
     name: "{{ item }}"
     state: present
-  loop:
+    update_cache: true
-    - firmware-misc-nonfree
+  loop: "{{ docker_host_package_common_dependencies }}"
-    - nfs-common
   become: true

roles/docker_host/vars/main.yml

@@ -0,0 +1,2 @@
+docker_host_package_common_dependencies:
+  - nfs-common

roles/proxmox/handlers/node.yml

@@ -0,0 +1,6 @@
+---
+- name: Reboot Node
+  ansible.builtin.reboot:
+    connect_timeout: 5
+    reboot_timeout: 600
+    test_command: whoami

roles/proxmox/tasks/06_hardware_acceleration.yml

@@ -0,0 +1,25 @@
+---
+- name: Set GRUB_CMDLINE_LINUX_DEFAULT for PCI passthrough
+  ansible.builtin.lineinfile:
+    path: /etc/default/grub
+    regexp: "^GRUB_CMDLINE_LINUX_DEFAULT="
+    line: 'GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt pcie_acs_override=downstream,multifunction initcall_blacklist=sysfb_init video=simplefb:off video=vesafb:off video=efifb:off video=vesa:off disable_vga=1 vfio_iommu_type1.allow_unsafe_interrupts=1 kvm.ignore_msrs=1 modprobe.blacklist=radeon,nouveau,nvidia,nvidiafb,nvidia-gpu,snd_hda_intel,snd_hda_codec_hdmi,i915"'
+    backup: true
+  # notify:
+  #   - Reboot Node
+
+- name: Ensure VFIO modules are listed in /etc/modules
+  ansible.builtin.blockinfile:
+    path: /etc/modules
+    marker: "# {mark} VFIO Modules"
+    block: |
+      vfio
+      vfio_iommu_type1
+      vfio_pci
+      vfio_virqfd
+    create: true
+
+- name: Update initramfs
+  ansible.builtin.command: update-initramfs -u -k all
+  args:
+    warn: false

roles/proxmox/tasks/55_create_vm.yml

@@ -24,71 +24,6 @@
     sshkeys: "{{ vm.sshkeys }}"
   register: proxmox_deploy_info
 
-- name: Get MAC Address of new machine
+- name: Provision created VM
-  ansible.builtin.set_fact:
+  ansible.builtin.include_tasks: 56_provision_new_vm.yml
-    mac_address: "{{ proxmox_deploy_info.mac.net0 }}"
+  when: proxmox_deploy_info.changed
-
-- name: Import disk
-  ansible.builtin.shell: |
-    qm importdisk {{ vm.vmid }} {{ proxmox_dirs.isos }}/{{ vm.boot_image }} {{ proxmox_storage }}
-  delegate_to: "{{ vm.node }}"
-
-- name: Attach disk and cloud-init
-  ansible.builtin.shell: |
-    qm set {{ vm.vmid }} --scsi0 {{ proxmox_storage }}:{{ vm.vmid }}/vm-{{ vm.vmid }}-disk-0.raw --ide2 {{ proxmox_storage }}:cloudinit --boot order=scsi0
-  delegate_to: "{{ vm.node }}"
-
-- name: Resize scsi0 disk if needed
-  ansible.builtin.shell: |
-    qm resize {{ vm.vmid }} scsi0 {{ vm.disk_size }}G
-  delegate_to: "{{ vm.node }}"
-
-- name: Start VM
-  community.general.proxmox_kvm:
-    api_user: root@pam
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
-    api_host: "192.168.20.12"
-    node: "{{ vm.node }}"
-    vmid: "{{ vm.vmid }}"
-    state: started
-
-- name: Wait for VM to appear on network
-  ansible.builtin.shell: |
-    nmap -sn -n -PR 192.168.20.0/24 | grep -B2 "{{ mac_address }}" | grep "Nmap scan report for"
-  register: vm_nmap_scan
-  retries: 30
-  delay: 5
-  until: vm_nmap_scan.stdout != ""
-  delegate_to: "{{ vm.node }}"
-
-- name: Extract the IP address from Nmap output
-  ansible.builtin.set_fact:
-    vm_found_ip: "{{ vm_nmap_scan.stdout | regex_search('Nmap scan report for ([0-9\\.]+)', '\\1') | first }}"
-
-- name: Debug IP address
-  ansible.builtin.debug:
-    msg: "Found VM IP address: {{ vm_found_ip }}"
-
-- name: Define SSH config block
-  ansible.builtin.set_fact:
-    ssh_entry: |
-      Host {{ vm.name }}
-          HostName {{ vm_found_ip }}
-          Port 22
-          User tudattr
-          IdentityFile /media/veracrypt1/genesis
-          ProxyJump {{ vm.node }}
-
-- name: Append new VM to SSH config
-  ansible.builtin.blockinfile:
-    path: "{{ ansible_env.HOME }}/.ssh/config_homelab"
-    marker: "# {mark} HOMELAB VMS BLOCK"
-    block: |
-      {{ ssh_entry }}
-
-- name: Add the new VM to the proxmox_nodes group in production.ini
-  ansible.builtin.lineinfile:
-    path: "../inventory.ini"
-    line: "{{ proxmox_inventory_entry }}"
-    insertafter: "[proxmox_nodes]"
-    state: present

roles/proxmox/tasks/56_provision_new_vm.yml

@@ -0,0 +1,72 @@
+---
+- name: Debug proxmox_deploy_info
+  ansible.builtin.debug:
+    msg: "{{ proxmox_deploy_info }}"
+
+- name: Get MAC Address of new machine
+  ansible.builtin.set_fact:
+    mac_address: "{{ proxmox_deploy_info.mac.net0 }}"
+
+- name: Import disk
+  ansible.builtin.shell: |
+    qm importdisk {{ vm.vmid }} {{ proxmox_dirs.isos }}/{{ vm.boot_image }} {{ proxmox_storage }}
+  delegate_to: "{{ vm.node }}"
+  when: proxmox_deploy_info.changed
+
+- name: Attach disk and cloud-init
+  ansible.builtin.shell: |
+    qm set {{ vm.vmid }} --scsi0 {{ proxmox_storage }}:{{ vm.vmid }}/vm-{{ vm.vmid }}-disk-0.raw --ide2 {{ proxmox_storage }}:cloudinit --boot order=scsi0
+  delegate_to: "{{ vm.node }}"
+
+- name: Resize scsi0 disk if needed
+  ansible.builtin.shell: |
+    qm resize {{ vm.vmid }} scsi0 {{ vm.disk_size }}G
+  delegate_to: "{{ vm.node }}"
+
+- name: Start VM
+  community.general.proxmox_kvm:
+    api_user: root@pam
+    api_password: "{{ vault.pve.aya01.root.sudo }}"
+    api_host: "192.168.20.12"
+    node: "{{ vm.node }}"
+    vmid: "{{ vm.vmid }}"
+    state: started
+
+- name: Wait for VM to appear on network
+  ansible.builtin.shell: |
+    nmap -sn -n -PR 192.168.20.0/24 | grep -B2 "{{ mac_address }}" | grep "Nmap scan report for"
+  register: vm_nmap_scan
+  retries: 30
+  delay: 5
+  until: vm_nmap_scan.stdout != ""
+  delegate_to: "{{ vm.node }}"
+
+- name: Extract the IP address from Nmap output
+  ansible.builtin.set_fact:
+    vm_found_ip: "{{ vm_nmap_scan.stdout | regex_search('Nmap scan report for ([0-9\\.]+)', '\\1') | first }}"
+
+- name: Define SSH config block
+  ansible.builtin.set_fact:
+    ssh_entry: |
+      Host {{ vm.name }}
+          HostName {{ vm_found_ip }}
+          Port 22
+          User tudattr
+          IdentityFile /media/veracrypt1/genesis
+          ProxyJump {{ vm.node }}
+
+- name: Append new VM to SSH config
+  ansible.builtin.blockinfile:
+    path: "{{ ansible_env.HOME }}/.ssh/config_homelab"
+    marker: "# {mark} HOMELAB VMS BLOCK"
+    block: |
+      {{ ssh_entry }}
+
+- name: Add VM to homelab_vms group in production.ini
+  ansible.builtin.lineinfile:
+    path: "{{ inventory_file }}"
+    line: "{{ vm.name }}"
+    insertafter: '^\[vms\]'
+    create: true
+    state: present
+  delegate_to: localhost

roles/proxmox/vars/main.yml

@@ -21,5 +21,6 @@ proxmox_tags:
 proxmox_node_dependencies:
   - libguestfs-tools
   - nmap
+  - firmware-misc-nonfree
 
 proxmox_localhost_dependencies: []