refactor: reorganize proxmox roles, add hardware acceleration, and update common config tasks

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

group_vars/proxmox/containers.yml

@@ -0,0 +1,2 @@
+lxcs:
+  - name: "test-lxc-00"

group_vars/proxmox/secrets_vm.yml

@@ -1,8 +1,11 @@
 $ANSIBLE_VAULT;1.1;AES256
-34303365623966316461623032386163326137623233353933313536343633623339356430303738
+36383135343063353934336632616563666331386639393065633161353364623166393433636430
-6662353066663134356637633865396531616334636263360a313266393030623761626636333832
+3930633063366563306364643934353135333164383663660a346134626362396233626562313736
-39623262656664653430303162633739613761316536303865326465353333376230346632333737
+38353830376130343734323631363036363163326666356231373963643933633437613331643665
-6635333534343330610a393730396538333932393836323963376333393239666132616364323166
+3030356364626235340a366162343334653333643962393131303239386331653436393431373636
-30373933363131353339376333633530313263663830393661353966306162613666396465366437
+39356462393762626362653430376138633539663331396138663237363734613133613631356235
-37326630633463313864636239303030366633366635323266383664346466356166353433653661
+33323439646230613934373639346136663330626330383566636361616234363333613464376538
-313938346264623634366464363330313863
+35356565663032613463626133633164383538313564376362336135373732396332343835323038
+66656639616566613564396338623934623830333135343837363230646161323665316432646532
+36613338616334306532376237316566376635326538313730633938333436623333383866383264
+3438343462323536653130306333626132326231376231653465

group_vars/proxmox/vars.yml

@@ -1,19 +1,3 @@
 proxmox_api_user: root
 proxmox_api_host: 192.168.20.12
 proxmox_api_password: "{{ vault.pve.aya01.root.sudo }}"
-
-vms:
-  - name: "test-vm-00"
-    node: "inko"
-    vmid: 950
-    cores: 2
-    memory: 8192 # in MiB
-    net:
-      net0: "virtio,bridge=vmbr0,firewall=1"
-    boot_image: "{{ proxmox_cloud_init_images.ubuntu.name }}"
-    ciuser: "{{ user }}"
-    sshkeys: "{{ pubkey }}"
-    disk_size: 32 # in Gb
-
-lxcs:
-  - name: "test-lxc-00"

group_vars/proxmox/vms.yml

@@ -0,0 +1,25 @@
+vms:
+  - name: "docker-host11"
+    node: "inko"
+    vmid: 311
+    cores: 2
+    memory: 4096 # in MiB
+    net:
+      net0: "virtio,bridge=vmbr0,firewall=1"
+    boot_image: "{{ proxmox_cloud_init_images.ubuntu.name }}"
+    ciuser: "{{ user }}"
+    sshkeys: "{{ pubkey }}"
+    disk_size: 128 # in Gb
+  - name: "docker-host12"
+    node: "lulu"
+    vmid: 312
+    cores: 2
+    memory: 4096 # in MiB
+    net:
+      net0: "virtio,bridge=vmbr0,firewall=1"
+    boot_image: "{{ proxmox_cloud_init_images.ubuntu.name }}"
+    ciuser: "{{ user }}"
+    sshkeys: "{{ pubkey }}"
+    disk_size: 128 # in Gb
+    # hostpci:
+    #   hostpci0: "0000:00:02.0"

production.ini

@@ -23,7 +23,6 @@ k3s-agent[00:02]
 k3s-server[00:02]
 k3s-longhorn[00:02]
 docker-host[00:01]
-test-vm-00
 
 [k3s_nodes]
 k3s-server[00:02]
@@ -57,3 +56,5 @@ docker-host[00:01]
 
 [docker_lb]
 docker-lb
+docker-host11
+docker-host12

roles/common/files/ghostty/infocmp

@@ -0,0 +1,80 @@
+xterm-ghostty|ghostty|Ghostty,
+	am, bce, ccc, hs, km, mc5i, mir, msgr, npc, xenl, AX, Su, Tc, XT, fullkbd,
+	colors#0x100, cols#80, it#8, lines#24, pairs#0x7fff,
+	acsc=++\,\,--..00``aaffgghhiijjkkllmmnnooppqqrrssttuuvvwwxxyyzz{{||}}~~,
+	bel=^G, blink=\E[5m, bold=\E[1m, cbt=\E[Z, civis=\E[?25l,
+	clear=\E[H\E[2J, cnorm=\E[?12l\E[?25h, cr=\r,
+	csr=\E[%i%p1%d;%p2%dr, cub=\E[%p1%dD, cub1=^H,
+	cud=\E[%p1%dB, cud1=\n, cuf=\E[%p1%dC, cuf1=\E[C,
+	cup=\E[%i%p1%d;%p2%dH, cuu=\E[%p1%dA, cuu1=\E[A,
+	cvvis=\E[?12;25h, dch=\E[%p1%dP, dch1=\E[P, dim=\E[2m,
+	dl=\E[%p1%dM, dl1=\E[M, dsl=\E]2;\007, ech=\E[%p1%dX,
+	ed=\E[J, el=\E[K, el1=\E[1K, flash=\E[?5h$<100/>\E[?5l,
+	fsl=^G, home=\E[H, hpa=\E[%i%p1%dG, ht=^I, hts=\EH,
+	ich=\E[%p1%d@, ich1=\E[@, il=\E[%p1%dL, il1=\E[L, ind=\n,
+	indn=\E[%p1%dS,
+	initc=\E]4;%p1%d;rgb:%p2%{255}%*%{1000}%/%2.2X/%p3%{255}%*%{1000}%/%2.2X/%p4%{255}%*%{1000}%/%2.2X\E\\,
+	invis=\E[8m, kDC=\E[3;2~, kEND=\E[1;2F, kHOM=\E[1;2H,
+	kIC=\E[2;2~, kLFT=\E[1;2D, kNXT=\E[6;2~, kPRV=\E[5;2~,
+	kRIT=\E[1;2C, kbs=^?, kcbt=\E[Z, kcub1=\EOD, kcud1=\EOB,
+	kcuf1=\EOC, kcuu1=\EOA, kdch1=\E[3~, kend=\EOF, kent=\EOM,
+	kf1=\EOP, kf10=\E[21~, kf11=\E[23~, kf12=\E[24~,
+	kf13=\E[1;2P, kf14=\E[1;2Q, kf15=\E[1;2R, kf16=\E[1;2S,
+	kf17=\E[15;2~, kf18=\E[17;2~, kf19=\E[18;2~, kf2=\EOQ,
+	kf20=\E[19;2~, kf21=\E[20;2~, kf22=\E[21;2~,
+	kf23=\E[23;2~, kf24=\E[24;2~, kf25=\E[1;5P, kf26=\E[1;5Q,
+	kf27=\E[1;5R, kf28=\E[1;5S, kf29=\E[15;5~, kf3=\EOR,
+	kf30=\E[17;5~, kf31=\E[18;5~, kf32=\E[19;5~,
+	kf33=\E[20;5~, kf34=\E[21;5~, kf35=\E[23;5~,
+	kf36=\E[24;5~, kf37=\E[1;6P, kf38=\E[1;6Q, kf39=\E[1;6R,
+	kf4=\EOS, kf40=\E[1;6S, kf41=\E[15;6~, kf42=\E[17;6~,
+	kf43=\E[18;6~, kf44=\E[19;6~, kf45=\E[20;6~,
+	kf46=\E[21;6~, kf47=\E[23;6~, kf48=\E[24;6~,
+	kf49=\E[1;3P, kf5=\E[15~, kf50=\E[1;3Q, kf51=\E[1;3R,
+	kf52=\E[1;3S, kf53=\E[15;3~, kf54=\E[17;3~,
+	kf55=\E[18;3~, kf56=\E[19;3~, kf57=\E[20;3~,
+	kf58=\E[21;3~, kf59=\E[23;3~, kf6=\E[17~, kf60=\E[24;3~,
+	kf61=\E[1;4P, kf62=\E[1;4Q, kf63=\E[1;4R, kf7=\E[18~,
+	kf8=\E[19~, kf9=\E[20~, khome=\EOH, kich1=\E[2~,
+	kind=\E[1;2B, kmous=\E[<, knp=\E[6~, kpp=\E[5~,
+	kri=\E[1;2A, oc=\E]104\007, op=\E[39;49m, rc=\E8,
+	rep=%p1%c\E[%p2%{1}%-%db, rev=\E[7m, ri=\EM,
+	rin=\E[%p1%dT, ritm=\E[23m, rmacs=\E(B, rmam=\E[?7l,
+	rmcup=\E[?1049l, rmir=\E[4l, rmkx=\E[?1l\E>, rmso=\E[27m,
+	rmul=\E[24m, rs1=\E]\E\\\Ec, sc=\E7,
+	setab=\E[%?%p1%{8}%<%t4%p1%d%e%p1%{16}%<%t10%p1%{8}%-%d%e48;5;%p1%d%;m,
+	setaf=\E[%?%p1%{8}%<%t3%p1%d%e%p1%{16}%<%t9%p1%{8}%-%d%e38;5;%p1%d%;m,
+	sgr=%?%p9%t\E(0%e\E(B%;\E[0%?%p6%t;1%;%?%p2%t;4%;%?%p1%p3%|%t;7%;%?%p4%t;5%;%?%p7%t;8%;m,
+	sgr0=\E(B\E[m, sitm=\E[3m, smacs=\E(0, smam=\E[?7h,
+	smcup=\E[?1049h, smir=\E[4h, smkx=\E[?1h\E=, smso=\E[7m,
+	smul=\E[4m, tbc=\E[3g, tsl=\E]2;, u6=\E[%i%d;%dR, u7=\E[6n,
+	u8=\E[?%[;0123456789]c, u9=\E[c, vpa=\E[%i%p1%dd,
+	BD=\E[?2004l, BE=\E[?2004h, Clmg=\E[s,
+	Cmg=\E[%i%p1%d;%p2%ds, Dsmg=\E[?69l, E3=\E[3J,
+	Enmg=\E[?69h, Ms=\E]52;%p1%s;%p2%s\007, PE=\E[201~,
+	PS=\E[200~, RV=\E[>c, Se=\E[2 q,
+	Setulc=\E[58:2::%p1%{65536}%/%d:%p1%{256}%/%{255}%&%d:%p1%{255}%&%d%;m,
+	Smulx=\E[4:%p1%dm, Ss=\E[%p1%d q,
+	Sync=\E[?2026%?%p1%{1}%-%tl%eh%;,
+	XM=\E[?1006;1000%?%p1%{1}%=%th%el%;, XR=\E[>0q,
+	fd=\E[?1004l, fe=\E[?1004h, kDC3=\E[3;3~, kDC4=\E[3;4~,
+	kDC5=\E[3;5~, kDC6=\E[3;6~, kDC7=\E[3;7~, kDN=\E[1;2B,
+	kDN3=\E[1;3B, kDN4=\E[1;4B, kDN5=\E[1;5B, kDN6=\E[1;6B,
+	kDN7=\E[1;7B, kEND3=\E[1;3F, kEND4=\E[1;4F,
+	kEND5=\E[1;5F, kEND6=\E[1;6F, kEND7=\E[1;7F,
+	kHOM3=\E[1;3H, kHOM4=\E[1;4H, kHOM5=\E[1;5H,
+	kHOM6=\E[1;6H, kHOM7=\E[1;7H, kIC3=\E[2;3~, kIC4=\E[2;4~,
+	kIC5=\E[2;5~, kIC6=\E[2;6~, kIC7=\E[2;7~, kLFT3=\E[1;3D,
+	kLFT4=\E[1;4D, kLFT5=\E[1;5D, kLFT6=\E[1;6D,
+	kLFT7=\E[1;7D, kNXT3=\E[6;3~, kNXT4=\E[6;4~,
+	kNXT5=\E[6;5~, kNXT6=\E[6;6~, kNXT7=\E[6;7~,
+	kPRV3=\E[5;3~, kPRV4=\E[5;4~, kPRV5=\E[5;5~,
+	kPRV6=\E[5;6~, kPRV7=\E[5;7~, kRIT3=\E[1;3C,
+	kRIT4=\E[1;4C, kRIT5=\E[1;5C, kRIT6=\E[1;6C,
+	kRIT7=\E[1;7C, kUP=\E[1;2A, kUP3=\E[1;3A, kUP4=\E[1;4A,
+	kUP5=\E[1;5A, kUP6=\E[1;6A, kUP7=\E[1;7A, kxIN=\E[I,
+	kxOUT=\E[O, rmxx=\E[29m, rv=\E\\[[0-9]+;[0-9]+;[0-9]+c,
+	setrgbb=\E[48:2:%p1%d:%p2%d:%p3%dm,
+	setrgbf=\E[38:2:%p1%d:%p2%d:%p3%dm, smxx=\E[9m,
+	xm=\E[<%i%p3%d;%p1%d;%p2%d;%?%p4%tM%em%;,
+	xr=\EP>\\|[ -~]+a\E\\,

roles/common/files/ssh/root/sshd_config

@@ -0,0 +1,19 @@
+Protocol 2
+PermitRootLogin yes
+MaxAuthTries 3
+PubkeyAuthentication yes
+PasswordAuthentication no
+PermitEmptyPasswords no
+ChallengeResponseAuthentication no
+UsePAM yes
+AllowAgentForwarding no
+AllowTcpForwarding yes
+X11Forwarding no
+PrintMotd no
+TCPKeepAlive no
+ClientAliveCountMax 2
+TrustedUserCAKeys /etc/ssh/vault-ca.pub
+UseDNS yes
+AcceptEnv LANG LC_*
+Subsystem	sftp	/usr/lib/openssh/sftp-server
+

roles/common/tasks/bash.yml

@@ -9,4 +9,16 @@
   loop:
     - bashrc
     - bash_aliases
-  become: true
+
+- name: Copy ghostty infocmp
+  ansible.builtin.copy:
+    src: files/ghostty/infocmp
+    dest: "{{ ansible_env.HOME }}/ghostty"
+    owner: "{{ ansible_user_id }}"
+    group: "{{ ansible_user_id }}"
+    mode: "0644"
+  register: ghostty_terminfo
+
+- name: Compile ghostty terminalinfo
+  ansible.builtin.command: "tic -x {{ ansible_env.HOME }}/ghostty"
+  when: ghostty_terminfo.changed

roles/common/tasks/sshd.yml

@@ -1,12 +1,24 @@
 ---
-- name: Copy sshd_config
+- name: Copy user sshd_config
   ansible.builtin.template:
-    src: templates/ssh/sshd_config
+    src: files/ssh/user/sshd_config
     dest: /etc/ssh/sshd_config
     mode: "644"
+    backup: true
   notify:
     - Restart sshd
   become: true
+  when: ansible_user_id != "root"
+
+- name: Copy root sshd_config
+  ansible.builtin.template:
+    src: files/ssh/root/sshd_config
+    dest: /etc/ssh/sshd_config
+    mode: "644"
+    backup: true
+  notify:
+    - Restart sshd
+  when: ansible_user_id == "root"
 
 - name: Copy pubkey
   ansible.builtin.copy:

roles/docker_host/tasks/main.yml

@@ -1,21 +1,21 @@
 ---
 - name: Setup VM
-  ansible.builtin.include_tasks: setup.yml
+  ansible.builtin.include_tasks: 10_setup.yml
 
 - name: Install docker
-  ansible.builtin.include_tasks: installation.yml
+  ansible.builtin.include_tasks: 20_installation.yml
 
 - name: Setup user and group for docker
-  ansible.builtin.include_tasks: user_group_setup.yml
+  ansible.builtin.include_tasks: 30_user_group_setup.yml
 
 - name: Setup directory structure for docker
-  ansible.builtin.include_tasks: directory_setup.yml
+  ansible.builtin.include_tasks: 40_directory_setup.yml
 
 - name: Deploy configs
-  ansible.builtin.include_tasks: provision.yml
+  ansible.builtin.include_tasks: 50_provision.yml
 
 - name: Deploy docker compose
-  ansible.builtin.include_tasks: deploy_compose.yml
+  ansible.builtin.include_tasks: 60_deploy_compose.yml
 
 - name: Publish metrics
-  ansible.builtin.include_tasks: export.yml
+  ansible.builtin.include_tasks: 70_export.yml

roles/proxmox/tasks/05_setup_node.yml

@@ -5,3 +5,6 @@
     update_cache: true
     state: present
   loop: "{{ proxmox_node_dependencies }}"
+
+- name: Ensure Harware Acceleration on node
+  ansible.builtin.include_tasks: 06_hardware_acceleration.yml

roles/proxmox/tasks/06_hardware_acceleration.yml

@@ -5,8 +5,7 @@
     regexp: "^GRUB_CMDLINE_LINUX_DEFAULT="
     line: 'GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt pcie_acs_override=downstream,multifunction initcall_blacklist=sysfb_init video=simplefb:off video=vesafb:off video=efifb:off video=vesa:off disable_vga=1 vfio_iommu_type1.allow_unsafe_interrupts=1 kvm.ignore_msrs=1 modprobe.blacklist=radeon,nouveau,nvidia,nvidiafb,nvidia-gpu,snd_hda_intel,snd_hda_codec_hdmi,i915"'
     backup: true
-  # notify:
+  register: iommu_result
-  #   - Reboot Node
 
 - name: Ensure VFIO modules are listed in /etc/modules
   ansible.builtin.blockinfile:
@@ -18,8 +17,21 @@
       vfio_pci
       vfio_virqfd
     create: true
+    backup: true
+  register: vfio_result
 
 - name: Update initramfs
   ansible.builtin.command: update-initramfs -u -k all
   args:
     warn: false
+  when: iommu_result.changed or vfio_result.changed
+  # notify:
+  #   - Reboot Node
+
+- name: update grub configuration
+  ansible.builtin.command: update-grub
+  args:
+    warn: false
+  when: iommu_result.changed or vfio_result.changed
+  # notify:
+  #   - Reboot Node

roles/proxmox/tasks/10_create_secrets.yml

@@ -5,29 +5,8 @@
     state: touch
     mode: "0600"
 
-- name: Decrypt vm vault file
-  ansible.builtin.shell: cd ../; ansible-vault decrypt "./playbooks/{{ proxmox_vault_file }}"
-  ignore_errors: true
-  no_log: true
-
-- name: Load existing vault content
-  ansible.builtin.slurp:
-    src: "{{ proxmox_vault_file }}"
-  register: vault_content
-  no_log: true
-
-- name: Parse vault content as YAML
-  ansible.builtin.set_fact:
-    vault_data: "{{ (vault_content['content'] | b64decode | from_yaml) if (vault_content['content'] | length > 0) else {} }}"
-  no_log: true
-
 - name: Update Vault data
   ansible.builtin.include_tasks: 15_create_secret.yml
   loop: "{{ vms | map(attribute='name') }}"
   loop_control:
     loop_var: "vm_name"
-
-- name: Encrypt vm vault file
-  ansible.builtin.shell: cd ../; ansible-vault encrypt "./playbooks/{{ proxmox_vault_file }}"
-  ignore_errors: true
-  no_log: true

roles/proxmox/tasks/15_create_secret.yml

@@ -1,4 +1,20 @@
 ---
+- name: Decrypt vm vault file
+  ansible.builtin.shell: cd ../; ansible-vault decrypt "./playbooks/{{ proxmox_vault_file }}"
+  ignore_errors: true
+  no_log: true
+
+- name: Load existing vault content
+  ansible.builtin.slurp:
+    src: "{{ proxmox_vault_file }}"
+  register: vault_content
+  no_log: true
+
+- name: Parse vault content as YAML
+  ansible.builtin.set_fact:
+    vault_data: "{{ (vault_content['content'] | b64decode | from_yaml) if (vault_content['content'] | length > 0) else {} }}"
+  no_log: true
+
 - name: Setup secret name
   ansible.builtin.set_fact:
     vm_name_secret: "{{ proxmox_secrets_prefix }}_{{ vm_name | replace('-','_') }}"
@@ -24,3 +40,8 @@
     mode: "0600"
   when: not variable_exists
   no_log: true
+
+- name: Encrypt vm vault file
+  ansible.builtin.shell: cd ../; ansible-vault encrypt "./playbooks/{{ proxmox_vault_file }}"
+  ignore_errors: true
+  no_log: true

roles/proxmox/tasks/55_create_vm.yml

@@ -1,9 +1,9 @@
 ---
 - name: Create VM
   community.general.proxmox_kvm:
-    api_user: root@pam
+    api_user: "{{ proxmox_api_user }}@pam"
-    api_password: "{{ vault.pve.aya01.root.sudo }}"
+    api_password: "{{ proxmox_api_password }}"
-    api_host: "192.168.20.12"
+    api_host: "{{ proxmox_api_host }}"
     agent: true
     name: "{{ vm.name }}"
     vmid: "{{ vm.vmid }}"
@@ -11,6 +11,7 @@
     cores: "{{ vm.cores }}"
     memory: "{{ vm.memory }}"
     net: "{{ vm.net }}"
+    hostpci: "{{ vm.hostpci | default({})}}"
     scsihw: "virtio-scsi-pci"
     ostype: "l26"
     tags: "{{ proxmox_tags }}"
@@ -18,7 +19,8 @@
     boot: "order=scsi0"
     cpu: "x86-64-v2-AES"
     ciuser: "{{ vm.ciuser }}"
-    cipassword: "{{ vm_secrets[proxmox_secrets_prefix + '_' + vm.name.replace('-', '_')] }}"
+    # cipassword: "{{ vm_secrets[proxmox_secrets_prefix + '_' + vm.name.replace('-', '_')] }}"
+    cipassword: "flyff369"
     ipconfig:
       ipconfig0: "ip=dhcp"
     sshkeys: "{{ vm.sshkeys }}"

roles/proxmox/tasks/56_provision_new_vm.yml

@@ -45,9 +45,11 @@
   ansible.builtin.set_fact:
     vm_found_ip: "{{ vm_nmap_scan.stdout | regex_search('Nmap scan report for ([0-9\\.]+)', '\\1') | first }}"
 
-- name: Define SSH config block
+- name: Append new VM to SSH config "{{ vm.name }}"
-  ansible.builtin.set_fact:
+  ansible.builtin.blockinfile:
-    ssh_entry: |
+    path: "{{ ansible_env.HOME }}/.ssh/config_homelab"
+    marker: "# {mark} HOMELAB VM: {{ vm.name }} BLOCK"
+    block: |
       Host {{ vm.name }}
         HostName {{ vm_found_ip }}
         Port 22
@@ -58,13 +60,6 @@
         ProxyJump {{ vm.node }}
         StrictHostKeyChecking no
 
-- name: Append new VM to SSH config
-  ansible.builtin.blockinfile:
-    path: "{{ ansible_env.HOME }}/.ssh/config_homelab"
-    marker: "# {mark} HOMELAB VMS BLOCK"
-    block: |
-      {{ ssh_entry }}
-
 - name: Add VM to homelab_vms group in production.ini
   ansible.builtin.lineinfile:
     path: "{{ inventory_file }}"

roles/proxmox/tasks/main.yml

@@ -13,6 +13,7 @@
 - name: Create VMs
   ansible.builtin.include_tasks: 50_create_vms.yml
   when: is_localhost
+
 - name: Create LXC containers
   ansible.builtin.include_tasks: 60_create_containers.yml
   when: is_localhost