feat(proxmox): per-node CPU type based on hardware capabilities

Add proxmox_node_cpu map — aya01 (Celeron N5105, no AVX2) stays at
x86-64-v2-AES; inko01/lulu/mii01/naruto01 (all AVX2-capable) use x86-64-v3.
Task looks up cpu type by vm.node with x86-64-v2-AES as fallback.

docs/runbooks/arr-cleanup/cleanup-orphans.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""
+Delete download entries from /media/downloads/sonarr that are NOT in Sonarr,
+logging every action (size, path, timestamp, outcome) to cleanup.log.
+
+Runs in two passes:
+  1. Tries hard to match each orphan against Sonarr (title + romaji + partial).
+     Anything that matches is skipped — only true non-matches are deleted.
+  2. For each confirmed non-match, checks whether a directory with that show
+     name exists in /media/series (belt-and-suspenders). If it does, skips.
+  3. Deletes remaining entries and logs every outcome.
+
+Usage:
+  python3 cleanup-orphans.py --dry-run     # show what would be deleted
+  python3 cleanup-orphans.py --yes         # delete without confirmation
+"""
+
+import urllib.request
+import json
+import subprocess
+import re
+import os
+import sys
+import argparse
+from datetime import datetime, timezone
+
+SONARR_URL    = "http://localhost:8989/api/v3"
+SSH_HOST      = "aya01"
+DL_ROOT       = "/media/downloads/sonarr"
+SERIES_ROOT   = "/media/series"
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+LOG_FILE   = os.path.join(script_dir, "cleanup.log")
+
+with open(os.path.join(script_dir, '../../../..', 'sonarr.api.env')) as f:
+    SONARR_KEY = f.read().strip()
+
+
+def api_get(url):
+    with urllib.request.urlopen(url, timeout=30) as r:
+        return json.load(r)
+
+
+def norm(s):
+    return re.sub(r'[^a-z0-9]', '', s.lower())
+
+
+def ssh_run(cmd):
+    r = subprocess.run(['ssh', SSH_HOST, cmd], capture_output=True, text=True)
+    return r.stdout.strip()
+
+
+def ssh_exists(path):
+    return ssh_run(f'[ -e {json.dumps(path)} ] && echo yes || echo no') == 'yes'
+
+
+def ssh_size(path):
+    """Return size in bytes, or 0 if path doesn't exist."""
+    out = ssh_run(f'du -sb {json.dumps(path)} 2>/dev/null | cut -f1')
+    try:
+        return int(out)
+    except ValueError:
+        return 0
+
+
+def ssh_delete(path):
+    r = subprocess.run(['ssh', SSH_HOST, f'rm -rf {json.dumps(path)}'],
+                       capture_output=True, text=True)
+    return r.returncode == 0, r.stderr.strip()
+
+
+def log(line):
+    ts = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+    entry = f"[{ts}] {line}"
+    print(entry)
+    with open(LOG_FILE, 'a') as f:
+        f.write(entry + '\n')
+
+
+def extract_title(name):
+    """Strip season/episode/quality tags to recover a bare show title."""
+    name = re.sub(r'\.(mkv|mp4|ts|avi)$', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'^\[.*?\]\s*', '', name)            # [Group] prefix
+    name = re.sub(r'\s*\[.*?\]\s*', ' ', name)         # inline [tags]
+    name = re.sub(r'[\.\s_\-]?[Ss]\d{1,2}[Ee]\d{1,2}.*$', '', name)
+    name = re.sub(r'[\.\s_\-]?[Ss]\d{1,2}[\.\s_\-].*$', '', name)
+    name = re.sub(r'[\.\s_\-]?[Ss]\d{2}$', '', name)
+    name = re.sub(r'[\.\s_\-]?(19|20)\d{2}.*$', '', name)
+    name = re.sub(r'[\.\s_\-]?\d{3,4}p.*$', '', name) # 1080p etc
+    name = re.sub(r'[\.\-_]+', ' ', name).strip()
+    return name
+
+
+def build_sonarr_index(series):
+    idx = {}
+    for s in series:
+        for title_variant in [s['title'], s.get('titleSlug', ''), s.get('sortTitle', '')]:
+            if title_variant:
+                idx[norm(title_variant)] = s
+        # Also index alternate titles if present
+        for alt in s.get('alternateTitles', []):
+            t = alt.get('title', '')
+            if t:
+                idx[norm(t)] = s
+    return idx
+
+
+def find_in_sonarr(dl_name, idx):
+    title = extract_title(dl_name)
+    tn = norm(title)
+    if tn in idx:
+        return idx[tn], title
+    # Partial: dl title starts with series title (or vice versa), min 6 chars
+    for k, rec in idx.items():
+        if k and len(k) >= 6 and len(tn) >= 6:
+            if tn.startswith(k) or k.startswith(tn):
+                return rec, title
+    return None, title
+
+
+def confirm(prompt):
+    answer = input(f"{prompt} [y/N] ").strip().lower()
+    return answer == 'y'
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dry-run', action='store_true')
+    parser.add_argument('--yes', '-y', action='store_true')
+    args = parser.parse_args()
+
+    if args.dry_run:
+        print("DRY-RUN — nothing will be deleted\n")
+
+    log("=" * 60)
+    log(f"cleanup-orphans.py started (dry_run={args.dry_run})")
+
+    print("Fetching Sonarr series (including alternate titles)...")
+    series = api_get(f"{SONARR_URL}/series?apikey={SONARR_KEY}")
+    print(f"  {len(series)} series")
+    idx = build_sonarr_index(series)
+
+    # Collect series dirs on disk for secondary check
+    # Strip years, imdb tags, and punctuation so "Bleach (2004) {imdb-...}" matches "Bleach"
+    print("Fetching /media/series directory listing...")
+    series_on_disk_raw = ssh_run(f'ls {json.dumps(SERIES_ROOT)}/').splitlines()
+    def norm_dir(d):
+        d = re.sub(r'\{.*?\}', '', d)          # remove {imdb-...}
+        d = re.sub(r'\(?\d{4}\)?', '', d)      # remove years
+        d = re.sub(r'[^a-z0-9]', '', d.lower())
+        return d
+    series_on_disk_norm = {norm_dir(d) for d in series_on_disk_raw if d.strip()}
+
+    print("Fetching download listing...")
+    dl_entries = ssh_run(f'ls {json.dumps(DL_ROOT)}/').splitlines()
+    dl_entries = [e.strip() for e in dl_entries if e.strip()]
+    print(f"  {len(dl_entries)} entries in {DL_ROOT}")
+
+    # --- First pass: match against Sonarr ---
+    not_in_sonarr = []
+    in_sonarr     = []
+
+    for dl in dl_entries:
+        rec, extracted_title = find_in_sonarr(dl, idx)
+        if rec:
+            in_sonarr.append((dl, rec['title']))
+        else:
+            not_in_sonarr.append((dl, extracted_title))
+
+    print(f"\n  Matched to Sonarr:   {len(in_sonarr)}")
+    print(f"  NOT in Sonarr:       {len(not_in_sonarr)}")
+
+    # --- Second pass: check if series dir exists on disk anyway ---
+    skip_has_series_dir = []
+    to_delete = []
+
+    for dl, title in not_in_sonarr:
+        title_n = norm(title)
+        # Check if any series dir on disk has a similar name
+        has_dir = any(
+            d and len(d) >= 6 and (title_n.startswith(d) or d.startswith(title_n))
+            for d in series_on_disk_norm
+        )
+        # Also check the full download path exists
+        dl_path = f"{DL_ROOT}/{dl}"
+        if has_dir:
+            skip_has_series_dir.append((dl, title, dl_path))
+        else:
+            to_delete.append((dl, title, dl_path))
+
+    if skip_has_series_dir:
+        print(f"\n  SKIPPED (series dir found on disk, needs manual review): {len(skip_has_series_dir)}")
+        for dl, title, _ in skip_has_series_dir:
+            print(f"    {title:40s} ← {dl[:60]}")
+
+    print(f"\n{'='*60}")
+    print(f"TO DELETE ({len(to_delete)} entries — not in Sonarr, no series dir on disk)")
+    print(f"{'='*60}")
+
+    # Get sizes in parallel
+    print("\nMeasuring sizes...")
+    size_cmd = ' && '.join(
+        f'du -sb {json.dumps(f"{DL_ROOT}/{dl}")} 2>/dev/null | cut -f1'
+        for dl, _, _ in to_delete
+    )
+    if to_delete:
+        size_out = ssh_run(f'bash -c {json.dumps(size_cmd)}').splitlines()
+    else:
+        size_out = []
+
+    sizes = {}
+    for i, (dl, title, path) in enumerate(to_delete):
+        try:
+            sizes[dl] = int(size_out[i]) if i < len(size_out) else 0
+        except (ValueError, IndexError):
+            sizes[dl] = 0
+
+    total_bytes = sum(sizes.values())
+    for dl, title, path in sorted(to_delete, key=lambda x: x[1]):
+        sz = sizes.get(dl, 0)
+        print(f"  {sz/1e9:6.1f}G  {title:40s} ← {dl[:60]}")
+
+    print(f"\n  Total: {total_bytes/1e9:.1f}G across {len(to_delete)} entries")
+
+    if not to_delete:
+        log("Nothing to delete.")
+        return
+
+    if not args.dry_run and not args.yes:
+        if not confirm(f"\nDelete {len(to_delete)} entries?"):
+            log("Aborted by user.")
+            return
+
+    # --- Delete with logging ---
+    deleted_count = 0
+    deleted_bytes = 0
+    failed_count  = 0
+
+    for dl, title, path in sorted(to_delete, key=lambda x: x[1]):
+        sz = sizes.get(dl, 0)
+        if args.dry_run:
+            log(f"DRY-RUN | {sz/1e9:.2f}G | {title} | {path}")
+            deleted_count += 1
+            deleted_bytes += sz
+        else:
+            ok, err = ssh_delete(path)
+            if ok:
+                log(f"DELETED | {sz/1e9:.2f}G | {title} | {path}")
+                deleted_count += 1
+                deleted_bytes += sz
+            else:
+                log(f"FAILED  | {sz/1e9:.2f}G | {title} | {path} | {err}")
+                failed_count += 1
+
+    log(f"DONE | deleted={deleted_count} | freed={deleted_bytes/1e9:.1f}G | failed={failed_count}")
+
+
+if __name__ == '__main__':
+    main()

docs/runbooks/arr-cleanup/cleanup.log

@@ -0,0 +1,160 @@
+[2026-04-22T21:18:32Z] ============================================================
+[2026-04-22T21:18:32Z] cleanup-orphans.py started (dry_run=True)
+[2026-04-22T21:18:55Z] DRY-RUN | 14.62G | BLEACH Thousand Year Blood War | /media/downloads/sonarr/BLEACH.Thousand-Year.Blood.War.S01.JAPANESE.1080p.DSNP.WEBRip.AAC2.0.x264-NTb[rartv]
+[2026-04-22T21:18:55Z] DRY-RUN | 1971.45G | Bleach USBD Remux TL | /media/downloads/sonarr/Bleach USBD Remux TL
+[2026-04-22T21:18:55Z] DRY-RUN | 0.52G | Gachiakuta   09 | /media/downloads/sonarr/[KiyoshiiSubs] Gachiakuta - 09 [1080p][H.265 - 10Bit].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.44G | Gachiakuta   19 ( | /media/downloads/sonarr/[SubsPlease] Gachiakuta - 19 (1080p) [019A6A50].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 24.39G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S01.1080p.MAX.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:18:55Z] DRY-RUN | 36.73G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S02.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:18:55Z] DRY-RUN | 37.52G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S03.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:18:55Z] DRY-RUN | 36.83G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S04.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:18:55Z] DRY-RUN | 37.77G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S05.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:18:55Z] DRY-RUN | 36.07G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S06.1080p.HMAX.WEB-DL.DD.5.1.H.264-GNOME
+[2026-04-22T21:18:55Z] DRY-RUN | 29.48G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S07.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:18:55Z] DRY-RUN | 28.71G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S08.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:18:55Z] DRY-RUN | 4.58G | Grimgar Of Fantasy And Ash ( | /media/downloads/sonarr/Grimgar Of Fantasy And Ash (2016) S01 1080p BluRay 10bit EAC3 2 0 x265-iVy
+[2026-04-22T21:18:55Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 01 (1080p) [4CA94F81]
+[2026-04-22T21:18:55Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 05 (1080p) [A0556FA8].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 06 (1080p) [982D7547].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 07 (1080p) [247CFB44].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.52G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 10 (1080p) [ABE1B90A]
+[2026-04-22T21:18:55Z] DRY-RUN | 1.52G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 13 (1080p) [230618C3].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 0.52G | Hikikomari Kyuuketsuki no Monmon   07 ( | /media/downloads/sonarr/[SubsPlease] Hikikomari Kyuuketsuki no Monmon - 07 (1080p) [B07BA1C7]
+[2026-04-22T21:18:55Z] DRY-RUN | 10.49G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S01.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:18:55Z] DRY-RUN | 8.97G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S01.1080p.NF.WEB-DL.DDP5.1.x264-NTG
+[2026-04-22T21:18:55Z] DRY-RUN | 4.23G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S02.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:18:55Z] DRY-RUN | 4.97G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S02.1080p.NF.WEB-DL.DDP5.1.Atmos.x264-Telly
+[2026-04-22T21:18:55Z] DRY-RUN | 5.99G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S03.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:18:55Z] DRY-RUN | 5.26G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S03.1080p.NF.WEBRip.DDP5.1.Atmos.x264-SMURF
+[2026-04-22T21:18:55Z] DRY-RUN | 4.44G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S04.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:18:55Z] DRY-RUN | 0.88G | SANDA | /media/downloads/sonarr/SANDA.S01E02.1080p.WEB.H264-SENSEI
+[2026-04-22T21:18:55Z] DRY-RUN | 0.70G | Senpai Is An Otokonoko | /media/downloads/sonarr/Senpai.Is.An.Otokonoko.S01E05.720p.WEB.H264-SKYANiME
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E01.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E02.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E03.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E04.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E07.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E08.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E10.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E12.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 31.17G | Sex Education | /media/downloads/sonarr/Sex.Education.S01.1080p.NF.WEB.DDP5.1.x264-DEFLATE
+[2026-04-22T21:18:55Z] DRY-RUN | 41.53G | Sex Education | /media/downloads/sonarr/Sex.Education.S02.1080p.NF.WEB.DDP5.1.x264-NTb
+[2026-04-22T21:18:55Z] DRY-RUN | 15.56G | Sex Education | /media/downloads/sonarr/Sex.Education.S03.1080p.NF.WEB-DL.DDP5.1.H.264-FLUX
+[2026-04-22T21:18:55Z] DRY-RUN | 21.49G | Sex Education | /media/downloads/sonarr/Sex.Education.S04.1080p.NF.WEB-DL.DDP5.1.H.264-Archie
+[2026-04-22T21:18:55Z] DRY-RUN | 1.49G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E02.THE.HERO.OF.MY.DREAMS.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.49G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E03.THE.MAN.WHO.STANDS.AT.THE.TOP.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.48G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E04.CLASH.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 0.34G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E07.A.Fight.He.Cant.Lose.1080p.B-Global.WEB-DL.JPN.AAC2.0.H.264.MSubs-ToonsHub.mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 0.26G | Wind Breaker | /media/downloads/sonarr/Wind Breaker - S01E12 - 1080p WEB HEVC -NanDesuKa (B-Global).mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.46G | Wind Breaker   01 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 01 (1080p) [5D5071F6].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.46G | Wind Breaker   05 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 05 (1080p) [B6649F46].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 1.46G | Wind Breaker   06 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 06 (1080p) [1C13E5BC].mkv
+[2026-04-22T21:18:55Z] DRY-RUN | 0.74G | Wistoria Wand And Sword | /media/downloads/sonarr/Wistoria.Wand.And.Sword.S01E01.720p.WEB.H264-SKYANiME
+[2026-04-22T21:18:55Z] DRY-RUN | 1.45G | Wistoria Wand and Sword | /media/downloads/sonarr/Wistoria.Wand.and.Sword.S01E02.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 1.44G | Wistoria Wand and Sword | /media/downloads/sonarr/Wistoria.Wand.and.Sword.S01E03.1080p.WEB.H264-KAWAII
+[2026-04-22T21:18:55Z] DRY-RUN | 0.00G | www UIndex org         Severance | /media/downloads/sonarr/www.UIndex.org    -    Severance S02E10 Cold Harbor 1080p ATVP WEB-DL DDP5 1 Atmos H 264-Kitsune
+[2026-04-22T21:18:55Z] DONE | deleted=53 | freed=2449.6G | failed=0
+[2026-04-22T21:23:05Z] ============================================================
+[2026-04-22T21:23:05Z] cleanup-orphans.py started (dry_run=True)
+[2026-04-22T21:23:28Z] DRY-RUN | 24.39G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S01.1080p.MAX.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:23:28Z] DRY-RUN | 36.73G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S02.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:23:28Z] DRY-RUN | 37.52G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S03.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:23:28Z] DRY-RUN | 36.83G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S04.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:23:28Z] DRY-RUN | 37.77G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S05.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:23:28Z] DRY-RUN | 36.07G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S06.1080p.HMAX.WEB-DL.DD.5.1.H.264-GNOME
+[2026-04-22T21:23:28Z] DRY-RUN | 29.48G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S07.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:23:28Z] DRY-RUN | 28.71G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S08.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:23:28Z] DRY-RUN | 4.58G | Grimgar Of Fantasy And Ash ( | /media/downloads/sonarr/Grimgar Of Fantasy And Ash (2016) S01 1080p BluRay 10bit EAC3 2 0 x265-iVy
+[2026-04-22T21:23:28Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 01 (1080p) [4CA94F81]
+[2026-04-22T21:23:28Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 05 (1080p) [A0556FA8].mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 06 (1080p) [982D7547].mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 07 (1080p) [247CFB44].mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.52G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 10 (1080p) [ABE1B90A]
+[2026-04-22T21:23:28Z] DRY-RUN | 1.52G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 13 (1080p) [230618C3].mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 0.52G | Hikikomari Kyuuketsuki no Monmon   07 ( | /media/downloads/sonarr/[SubsPlease] Hikikomari Kyuuketsuki no Monmon - 07 (1080p) [B07BA1C7]
+[2026-04-22T21:23:28Z] DRY-RUN | 10.49G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S01.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:23:28Z] DRY-RUN | 8.97G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S01.1080p.NF.WEB-DL.DDP5.1.x264-NTG
+[2026-04-22T21:23:28Z] DRY-RUN | 4.23G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S02.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:23:28Z] DRY-RUN | 4.97G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S02.1080p.NF.WEB-DL.DDP5.1.Atmos.x264-Telly
+[2026-04-22T21:23:28Z] DRY-RUN | 5.99G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S03.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:23:28Z] DRY-RUN | 5.26G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S03.1080p.NF.WEBRip.DDP5.1.Atmos.x264-SMURF
+[2026-04-22T21:23:28Z] DRY-RUN | 4.44G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S04.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:23:28Z] DRY-RUN | 0.88G | SANDA | /media/downloads/sonarr/SANDA.S01E02.1080p.WEB.H264-SENSEI
+[2026-04-22T21:23:28Z] DRY-RUN | 0.70G | Senpai Is An Otokonoko | /media/downloads/sonarr/Senpai.Is.An.Otokonoko.S01E05.720p.WEB.H264-SKYANiME
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E01.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E02.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E03.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E04.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E07.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E08.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E10.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E12.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 31.17G | Sex Education | /media/downloads/sonarr/Sex.Education.S01.1080p.NF.WEB.DDP5.1.x264-DEFLATE
+[2026-04-22T21:23:28Z] DRY-RUN | 41.53G | Sex Education | /media/downloads/sonarr/Sex.Education.S02.1080p.NF.WEB.DDP5.1.x264-NTb
+[2026-04-22T21:23:28Z] DRY-RUN | 15.56G | Sex Education | /media/downloads/sonarr/Sex.Education.S03.1080p.NF.WEB-DL.DDP5.1.H.264-FLUX
+[2026-04-22T21:23:28Z] DRY-RUN | 21.49G | Sex Education | /media/downloads/sonarr/Sex.Education.S04.1080p.NF.WEB-DL.DDP5.1.H.264-Archie
+[2026-04-22T21:23:28Z] DRY-RUN | 1.49G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E02.THE.HERO.OF.MY.DREAMS.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.49G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E03.THE.MAN.WHO.STANDS.AT.THE.TOP.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.48G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E04.CLASH.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 0.34G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E07.A.Fight.He.Cant.Lose.1080p.B-Global.WEB-DL.JPN.AAC2.0.H.264.MSubs-ToonsHub.mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 0.26G | Wind Breaker | /media/downloads/sonarr/Wind Breaker - S01E12 - 1080p WEB HEVC -NanDesuKa (B-Global).mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.46G | Wind Breaker   01 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 01 (1080p) [5D5071F6].mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.46G | Wind Breaker   05 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 05 (1080p) [B6649F46].mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 1.46G | Wind Breaker   06 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 06 (1080p) [1C13E5BC].mkv
+[2026-04-22T21:23:28Z] DRY-RUN | 0.74G | Wistoria Wand And Sword | /media/downloads/sonarr/Wistoria.Wand.And.Sword.S01E01.720p.WEB.H264-SKYANiME
+[2026-04-22T21:23:28Z] DRY-RUN | 1.45G | Wistoria Wand and Sword | /media/downloads/sonarr/Wistoria.Wand.and.Sword.S01E02.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 1.44G | Wistoria Wand and Sword | /media/downloads/sonarr/Wistoria.Wand.and.Sword.S01E03.1080p.WEB.H264-KAWAII
+[2026-04-22T21:23:28Z] DRY-RUN | 0.00G | www UIndex org         Severance | /media/downloads/sonarr/www.UIndex.org    -    Severance S02E10 Cold Harbor 1080p ATVP WEB-DL DDP5 1 Atmos H 264-Kitsune
+[2026-04-22T21:23:28Z] DONE | deleted=49 | freed=461.6G | failed=0
+[2026-04-22T21:32:57Z] ============================================================
+[2026-04-22T21:32:57Z] cleanup-orphans.py started (dry_run=False)
+[2026-04-22T21:33:31Z] DELETED | 24.39G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S01.1080p.MAX.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:34:04Z] DELETED | 36.73G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S02.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:34:39Z] DELETED | 37.52G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S03.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:35:05Z] DELETED | 36.83G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S04.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:35:33Z] DELETED | 37.77G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S05.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:35:51Z] DELETED | 36.07G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S06.1080p.HMAX.WEB-DL.DD.5.1.H.264-GNOME
+[2026-04-22T21:36:01Z] DELETED | 29.48G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S07.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:36:09Z] DELETED | 28.71G | Game of Thrones | /media/downloads/sonarr/Game.of.Thrones.S08.NORDiC.1080p.HMAX.WEB-DL.DDP5.1.Atmos.H.264-DKV
+[2026-04-22T21:36:10Z] DELETED | 4.58G | Grimgar Of Fantasy And Ash ( | /media/downloads/sonarr/Grimgar Of Fantasy And Ash (2016) S01 1080p BluRay 10bit EAC3 2 0 x265-iVy
+[2026-04-22T21:36:11Z] DELETED | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 01 (1080p) [4CA94F81]
+[2026-04-22T21:36:11Z] DELETED | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 05 (1080p) [A0556FA8].mkv
+[2026-04-22T21:36:11Z] DELETED | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 06 (1080p) [982D7547].mkv
+[2026-04-22T21:36:12Z] DELETED | 1.53G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 07 (1080p) [247CFB44].mkv
+[2026-04-22T21:36:13Z] DELETED | 1.52G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 10 (1080p) [ABE1B90A]
+[2026-04-22T21:36:13Z] DELETED | 1.52G | Hibike! Euphonium | /media/downloads/sonarr/[SubsPlease] Hibike! Euphonium S3 - 13 (1080p) [230618C3].mkv
+[2026-04-22T21:36:13Z] DELETED | 0.52G | Hikikomari Kyuuketsuki no Monmon   07 ( | /media/downloads/sonarr/[SubsPlease] Hikikomari Kyuuketsuki no Monmon - 07 (1080p) [B07BA1C7]
+[2026-04-22T21:36:15Z] DELETED | 10.49G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S01.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:36:16Z] DELETED | 8.97G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S01.1080p.NF.WEB-DL.DDP5.1.x264-NTG
+[2026-04-22T21:36:16Z] DELETED | 4.23G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S02.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:36:17Z] DELETED | 4.97G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S02.1080p.NF.WEB-DL.DDP5.1.Atmos.x264-Telly
+[2026-04-22T21:36:17Z] DELETED | 5.99G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S03.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:36:18Z] DELETED | 5.26G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S03.1080p.NF.WEBRip.DDP5.1.Atmos.x264-SMURF
+[2026-04-22T21:36:22Z] DELETED | 4.44G | Love Death and Robots | /media/downloads/sonarr/Love.Death.and.Robots.S04.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-FLUX
+[2026-04-22T21:36:22Z] DELETED | 0.88G | SANDA | /media/downloads/sonarr/SANDA.S01E02.1080p.WEB.H264-SENSEI
+[2026-04-22T21:36:22Z] DELETED | 0.70G | Senpai Is An Otokonoko | /media/downloads/sonarr/Senpai.Is.An.Otokonoko.S01E05.720p.WEB.H264-SKYANiME
+[2026-04-22T21:36:22Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E01.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:23Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E02.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:23Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E03.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:23Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E04.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:23Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E07.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:24Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E08.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:24Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E10.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:24Z] DELETED | 1.39G | Senpai is an Otokonoko | /media/downloads/sonarr/Senpai.is.an.Otokonoko.S01E12.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:25Z] DELETED | 31.17G | Sex Education | /media/downloads/sonarr/Sex.Education.S01.1080p.NF.WEB.DDP5.1.x264-DEFLATE
+[2026-04-22T21:36:26Z] DELETED | 41.53G | Sex Education | /media/downloads/sonarr/Sex.Education.S02.1080p.NF.WEB.DDP5.1.x264-NTb
+[2026-04-22T21:36:26Z] DELETED | 15.56G | Sex Education | /media/downloads/sonarr/Sex.Education.S03.1080p.NF.WEB-DL.DDP5.1.H.264-FLUX
+[2026-04-22T21:36:27Z] DELETED | 21.49G | Sex Education | /media/downloads/sonarr/Sex.Education.S04.1080p.NF.WEB-DL.DDP5.1.H.264-Archie
+[2026-04-22T21:36:27Z] DELETED | 1.49G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E02.THE.HERO.OF.MY.DREAMS.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:36:28Z] DELETED | 1.49G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E03.THE.MAN.WHO.STANDS.AT.THE.TOP.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:36:28Z] DELETED | 1.48G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E04.CLASH.1080p.CR.WEB-DL.AAC2.0.H.264.DUAL-VARYG.mkv
+[2026-04-22T21:36:28Z] DELETED | 0.34G | WIND BREAKER | /media/downloads/sonarr/WIND.BREAKER.S01E07.A.Fight.He.Cant.Lose.1080p.B-Global.WEB-DL.JPN.AAC2.0.H.264.MSubs-ToonsHub.mkv
+[2026-04-22T21:36:29Z] DELETED | 0.26G | Wind Breaker | /media/downloads/sonarr/Wind Breaker - S01E12 - 1080p WEB HEVC -NanDesuKa (B-Global).mkv
+[2026-04-22T21:36:29Z] DELETED | 1.46G | Wind Breaker   01 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 01 (1080p) [5D5071F6].mkv
+[2026-04-22T21:36:29Z] DELETED | 1.46G | Wind Breaker   05 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 05 (1080p) [B6649F46].mkv
+[2026-04-22T21:36:30Z] DELETED | 1.46G | Wind Breaker   06 ( | /media/downloads/sonarr/[SubsPlease] Wind Breaker - 06 (1080p) [1C13E5BC].mkv
+[2026-04-22T21:36:30Z] DELETED | 0.74G | Wistoria Wand And Sword | /media/downloads/sonarr/Wistoria.Wand.And.Sword.S01E01.720p.WEB.H264-SKYANiME
+[2026-04-22T21:36:30Z] DELETED | 1.45G | Wistoria Wand and Sword | /media/downloads/sonarr/Wistoria.Wand.and.Sword.S01E02.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:31Z] DELETED | 1.44G | Wistoria Wand and Sword | /media/downloads/sonarr/Wistoria.Wand.and.Sword.S01E03.1080p.WEB.H264-KAWAII
+[2026-04-22T21:36:31Z] DELETED | 0.00G | www UIndex org         Severance | /media/downloads/sonarr/www.UIndex.org    -    Severance S02E10 Cold Harbor 1080p ATVP WEB-DL DDP5 1 Atmos H 264-Kitsune
+[2026-04-22T21:36:31Z] DONE | deleted=49 | freed=461.6G | failed=0

docs/runbooks/arr-cleanup/cleanup.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+Delete confirmed-safe download entries from /media/downloads/sonarr and /media/downloads/radarr.
+
+Reads /tmp/arr_verified.json produced by verify.py.
+Only deletes entries where status == 'safe' (API-confirmed imported + disk path verified).
+Orphans and path_missing entries are never touched.
+
+Usage:
+  python3 cleanup.py --dry-run            # print what would be deleted
+  python3 cleanup.py --arr sonarr         # delete only sonarr downloads
+  python3 cleanup.py --arr radarr         # delete only radarr downloads
+  python3 cleanup.py                      # delete both (prompts for confirmation)
+
+  # Target a single series/movie by title substring:
+  python3 cleanup.py --arr sonarr --title "American Dragon"
+"""
+
+import json
+import subprocess
+import argparse
+import sys
+
+SSH_HOST        = "aya01"
+SONARR_DL_ROOT  = "/media/downloads/sonarr"
+RADARR_DL_ROOT  = "/media/downloads/radarr"
+VERIFIED_JSON   = "/tmp/arr_verified.json"
+
+
+def ssh_delete(path, dry_run):
+    """Delete path on remote host. Returns True on success."""
+    if dry_run:
+        print(f"  [DRY-RUN] would delete: {path}")
+        return True
+    result = subprocess.run(
+        ['ssh', SSH_HOST, f'rm -rf {json.dumps(path)}'],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        print(f"  ERROR deleting {path}: {result.stderr.strip()}")
+        return False
+    return True
+
+
+def ssh_exists(path):
+    r = subprocess.run(['ssh', SSH_HOST, f'[ -e {json.dumps(path)} ] && echo yes || echo no'],
+                       capture_output=True, text=True)
+    return r.stdout.strip() == 'yes'
+
+
+def confirm(prompt):
+    answer = input(f"{prompt} [y/N] ").strip().lower()
+    return answer == 'y'
+
+
+def process(entries, dl_root, label, dry_run, title_filter, yes=False):
+    safe = [m for m in entries if m['status'] == 'safe']
+    if title_filter:
+        safe = [m for m in safe if title_filter.lower() in m['title'].lower()]
+
+    if not safe:
+        print(f"No safe entries to delete for {label}.")
+        return 0, 0
+
+    print(f"\n{'='*60}")
+    print(f"{label} — {len(safe)} entries to delete")
+    print(f"{'='*60}")
+
+    for m in safe:
+        pct = m.get('percentOfEpisodes', '')
+        pct_str = f" [{pct:.0f}%]" if isinstance(pct, float) else ''
+        files = m.get('episodeFileCount', '')
+        total = m.get('totalEpisodeCount', '')
+        count_str = f" ({files}/{total} eps)" if files != '' else f" (hasFile=True)"
+        print(f"  {m['title']}{pct_str}{count_str}")
+        print(f"    ← {m['dl']}")
+        print(f"    → {m['check_path']}")
+
+    if not dry_run and not yes:
+        if not confirm(f"\nDelete {len(safe)} {label} download entries?"):
+            print("Skipped.")
+            return 0, 0
+
+    deleted, failed = 0, 0
+    for m in safe:
+        dl_path = f"{dl_root}/{m['dl']}"
+        # Double-check the series/movie still exists on disk before deleting the download
+        if not dry_run and not ssh_exists(m['check_path']):
+            print(f"  SKIP {m['title']}: media path no longer on disk ({m['check_path']})")
+            failed += 1
+            continue
+        ok = ssh_delete(dl_path, dry_run)
+        if ok:
+            deleted += 1
+        else:
+            failed += 1
+
+    print(f"\n{label}: {deleted} deleted, {failed} failed/skipped")
+    return deleted, failed
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dry-run', action='store_true', help='Print actions without deleting')
+    parser.add_argument('--yes', '-y', action='store_true', help='Skip confirmation prompt')
+    parser.add_argument('--arr', choices=['sonarr', 'radarr', 'both'], default='both')
+    parser.add_argument('--title', default='', help='Only process entries matching this title substring')
+    args = parser.parse_args()
+
+    with open(VERIFIED_JSON) as f:
+        data = json.load(f)
+
+    if args.dry_run:
+        print("DRY-RUN mode — nothing will be deleted\n")
+
+    total_deleted, total_failed = 0, 0
+
+    if args.arr in ('radarr', 'both'):
+        d, f = process(data['radarr_matched'], RADARR_DL_ROOT, 'Radarr', args.dry_run, args.title, args.yes)
+        total_deleted += d
+        total_failed  += f
+
+    if args.arr in ('sonarr', 'both'):
+        d, f = process(data['sonarr_matched'], SONARR_DL_ROOT, 'Sonarr', args.dry_run, args.title, args.yes)
+        total_deleted += d
+        total_failed  += f
+
+    print(f"\nTotal: {total_deleted} deleted, {total_failed} failed/skipped")
+
+
+if __name__ == '__main__':
+    main()

docs/runbooks/arr-cleanup/findings.md

@@ -0,0 +1,200 @@
+# arr-stack Downloads Cleanup — Investigation Findings
+
+## Storage Layout (aya01)
+
+| Device | FS | Size | Used | Mount |
+|--------|----|------|------|-------|
+| `/dev/sdc3` | btrfs | 1.9T | 177G (10%) | `/` (system) |
+| `/dev/sda1` | btrfs `proxmox` | 2.8T | 1.3T (48%) | `/opt` |
+| `/dev/sdd1` | ext4 | 17T | 15T (92%) | `/mnt/hdd0` |
+| `/dev/sde1` | ext4 | 17T | 15T (92%) | `/mnt/hdd2` |
+| `/dev/sdf1` | ext4 | 17T | 15T (92%) | `/mnt/hdd1` |
+| `mergerfs` | fuse | 49T | 43T (92%) | `/media` |
+
+`/media` is a mergerfs union of hdd0 + hdd1 + hdd2. All three HDDs were at ~92% capacity before cleanup.
+
+**After cleanup (2026-04-23):**
+
+| Device | Used | Avail | Use% |
+|--------|------|-------|------|
+| `/dev/sdd1` (hdd0) | 9.4T | 6.2T | 61% |
+| `/dev/sdf1` (hdd1) | 9.3T | 6.3T | 60% |
+| `/dev/sde1` (hdd2) | 7.8T | 7.8T | 51% |
+| `mergerfs /media` | 27T | 21T | 57% |
+
+**~16T freed total** (92% → 57% on the mergerfs pool).
+
+## /media Breakdown (before cleanup)
+
+| Directory | Size |
+|-----------|------|
+| `downloads` | **22T** |
+| `series` | 16T |
+| `movies` | 5T |
+
+## Root Cause: No Hardlinks → All Imports Are Copies
+
+Zero hardlinked files exist anywhere across all three HDDs. Confirmed by two methods:
+1. Inspecting the Kubernetes manifests in `argocd-homelab/services/arr-stack/`
+2. Inode comparison of 1365 download/media file pairs — **0 shared inodes found** (every file is a distinct copy)
+
+**All three services mount the mergerfs `/media/` path via NFS:**
+
+```
+sonarr:      NFS 192.168.20.12:/media/downloads  → /downloads
+             NFS 192.168.20.12:/media/series     → /tv
+radarr:      NFS 192.168.20.12:/media/downloads  → /downloads
+             NFS 192.168.20.12:/media/movies     → /movies
+qbit:        NFS 192.168.20.12:/media/downloads  → /downloads
+```
+
+mergerfs does not support hardlinks across underlying filesystems. When qBit downloads to `/media/downloads/sonarr/` (lands on e.g. hdd1) and Sonarr imports to `/media/series/` (lands on e.g. hdd0), the hardlink attempt crosses a physical disk boundary → falls back to copy. Every import doubles the data.
+
+**Estimated wasted space before cleanup: ~21T** (the entire downloads/sonarr + downloads/radarr).
+
+## How to Run
+
+Prerequisites:
+```bash
+# Port-forward Sonarr and Radarr APIs
+kubectl -n arr-stack port-forward svc/sonarr 8989:8989 &
+kubectl -n arr-stack port-forward svc/radarr 7878:7878 &
+```
+
+API keys are loaded from `../../../../sonarr.api.env` and `../../../../radarr.api.env`
+(i.e. `/home/tudattr/workspace/infra/sonarr.api.env` relative to this repo).
+
+Container path mappings used in scripts:
+- Sonarr: `/tv/` → `/media/series/`
+- Radarr: `/movies/` → `/media/movies/`
+
+### Step 1 — Verify (generates `/tmp/arr_verified.json`)
+```bash
+python3 verify.py
+```
+Cross-references all downloads against Sonarr/Radarr APIs, verifies reported file paths exist on disk via SSH. Classifies each entry as `safe`, `not_imported`, or `path_missing`.
+
+### Step 2 — Delete confirmed-imported downloads
+```bash
+python3 cleanup.py --dry-run          # preview
+python3 cleanup.py --arr sonarr --yes
+python3 cleanup.py --arr radarr --yes
+```
+
+### Step 3 — Delete orphans (downloads not in Sonarr at all)
+```bash
+python3 cleanup-orphans.py --dry-run  # preview
+python3 cleanup-orphans.py --yes
+```
+
+All actions are logged to `cleanup.log` with UTC timestamp, size, title, path, and outcome.
+
+## Cleanup Performed (2026-04-23)
+
+### Pass 1 — Orphans (downloads not in Sonarr)
+Script: `cleanup-orphans.py`
+
+Two-pass logic:
+1. Match each download name against Sonarr API (title, slug, sortTitle, alternate titles, partial match)
+2. If no API match, check if a series directory with a similar name exists in `/media/series/` — if it does, skip (needs manual review)
+3. Delete remaining true orphans
+
+Result: **49 deleted, 461.6G freed, 0 failed**
+
+111 entries SKIPPED (series dir found on disk) — includes Bleach, House, Lucifer, You, SpongeBob, Detective Conan episodes, What If, etc. See `cleanup.log` for full list.
+
+Notable orphans deleted:
+- Game of Thrones S01–S08 (~267G) — removed from Sonarr
+- Sex Education S01–S04 (~110G) — removed from Sonarr
+- Love Death & Robots (multiple duplicate copies, ~45G)
+- Senpai is an Otokonoko, Wind Breaker, Wistoria, Hibike! Euphonium S3 episodes, etc.
+
+### Pass 2 — Confirmed-imported Sonarr downloads
+Script: `cleanup.py --arr sonarr --yes`
+
+Deleted downloads where Sonarr confirmed `episodeFileCount > 0` AND the series directory was verified to exist on disk.
+
+Result: **1106 deleted, 0 failed**
+
+### Pass 3 — Confirmed-imported Radarr downloads
+Script: `cleanup.py --arr radarr --yes`
+
+Deleted downloads where Radarr confirmed `hasFile=True` AND the file/directory path was verified to exist on disk.
+
+Result: **259 deleted, 0 failed**
+
+### Summary
+| Pass | Script | Entries | Space freed |
+|------|--------|---------|-------------|
+| Orphans | `cleanup-orphans.py` | 49 | ~461G |
+| Sonarr imports | `cleanup.py --arr sonarr` | 1106 | ~12T (estimated) |
+| Radarr imports | `cleanup.py --arr radarr` | 259 | ~4T (estimated) |
+| **Total** | | **1414** | **~16T** |
+
+## Verification Results (from verify.py run before cleanup)
+
+| | Safe to delete | Not imported | Path missing | Orphans (no API match) |
+|---|---|---|---|---|
+| **Sonarr** (1439 downloads) | 1106 | — | — | 333 |
+| **Radarr** (289 downloads) | 265 | — | — | 25 |
+
+Note: `cleanup-orphans.py` uses more aggressive title matching (alternate titles, partial match) than `verify.py`, so its orphan count (160 not-in-Sonarr out of 1438) is lower than `verify.py`'s 333.
+
+### Radarr Orphans (25) — not matched, not deleted
+- Constantine (2005)
+- Cowboy Bebop: Knockin' on Heaven's Door (2001)
+- Les Misérables (2012)
+- Pokémon Detective Pikachu (2019)
+- Code Geass: Fukkatsu no Lelouch (2019)
+- Eiga Go-Toubun no Hanayome (2022)
+- Gisaengchung / Parasite — Korean title, matching failure
+- Dune: Part One (2021) — matching failure, confirmed in Radarr
+- Harry Potter older/duplicate copies — matching failure
+- Porco Rosso / Kurenai no buta — matching failure
+- Castle in the Sky / Laputa — matching failure
+- Steins;Gate: The Movie — matching failure
+- Project Silence / Talchul — matching failure
+- Digimon: Frontier & Savers films
+- One Piece films (several)
+- Paripi Koumei movie
+- Fantastic Four (2025) extra copies (3)
+- JJK DCP trailer file
+
+### Path mismatch entries (confirmed safe, deleted anyway)
+- Star Wars Episode IV/V/VI/IX — all matched to Episode IV record; manually confirmed all 4 dirs exist
+- WALL·E — `·` middle-dot (U+00B7) broke string comparison; file confirmed on disk
+
+## Pending Decisions
+
+### Bleach USBD Remux TL (1.8T)
+`/media/downloads/sonarr/Bleach USBD Remux TL` — full lossless Bluray remux S00–S16 (-ZR- group).
+
+Currently SKIPPED — `/media/series/Bleach (2004) {imdb-tt0434665}/` exists (310G imported).
+
+Most seasons were imported from lighter x265 Bluray packs (`Bleach S0x Bluray EAC3 2.0 1080p x265-iVy`) rather than this remux. S11 has no imported content. S13 and S14 partially imported.
+
+Options:
+- **Delete** — free 1.8T, imported x265 content stays, re-download at remux quality later if desired
+- **Keep** — retain as source for Sonarr to import remaining episodes at lossless quality now that disk space is freed
+
+Per-season breakdown saved in memory.
+
+### SKIPPED downloads (111 Sonarr entries)
+Downloads where a matching series directory exists on disk but the series is not in Sonarr.
+Likely intentionally removed series (House, Lucifer, You, Black Clover, etc.) with leftover download copies.
+Needs manual review per series before deleting.
+
+## Permanent Fix (not applied)
+
+Mount per-HDD NFS paths instead of the mergerfs path, so qBit downloads and arr imports land on the same physical filesystem, enabling hardlinks:
+
+```yaml
+# In sonarr/radarr/qtun deployments, change:
+path: /media/downloads  →  path: /mnt/hdd0/downloads
+path: /media/series     →  path: /mnt/hdd0/series
+path: /media/movies     →  path: /mnt/hdd0/movies
+```
+
+Jellyfin/Plex keep reading from `/media/` (mergerfs union). New imports hardlink within hdd0, wasting no extra space.
+
+Tradeoff: all new content lands on hdd0 only. Load balancing across the three disks stops working for new downloads. Once hdd0 fills up a migration strategy is needed.

docs/runbooks/arr-cleanup/verify.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""
+Cross-reference /media/downloads/sonarr and /media/downloads/radarr against
+the Sonarr/Radarr APIs, then verify reported file paths actually exist on disk.
+
+Requirements:
+  - kubectl port-forwards active:
+      kubectl -n arr-stack port-forward svc/sonarr 8989:8989
+      kubectl -n arr-stack port-forward svc/radarr 7878:7878
+  - SSH access to aya01
+  - API keys in ../../../../sonarr.api.env and ../../../../radarr.api.env
+
+Output:
+  /tmp/arr_verified.json  — full structured results for use by cleanup.py
+"""
+
+import urllib.request
+import json
+import subprocess
+import re
+import sys
+import os
+
+SONARR_URL = "http://localhost:8989/api/v3"
+RADARR_URL = "http://localhost:7878/api/v3"
+SSH_HOST   = "aya01"
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+
+def load_key(filename):
+    path = os.path.join(script_dir, '../../../..', filename)
+    return open(path).read().strip()
+
+SONARR_KEY = load_key('sonarr.api.env')
+RADARR_KEY = load_key('radarr.api.env')
+
+
+def api_get(url):
+    with urllib.request.urlopen(url, timeout=30) as r:
+        return json.load(r)
+
+
+def norm(s):
+    return re.sub(r'[^a-z0-9]', '', s.lower())
+
+
+def extract_title(name, is_movie):
+    """Strip release tags from a download name to recover a bare title."""
+    name = re.sub(r'\.(mkv|mp4|avi|m4v)$', '', name, flags=re.IGNORECASE)
+    name = re.sub(r'\[.*?\]', '', name)
+    if is_movie:
+        name = re.sub(r'[\.\s_\-]?(19|20)\d{2}.*$', '', name)
+    else:
+        name = re.sub(r'[\.\s_\-]?[Ss]\d{1,2}([Ee]\d{1,2})?.*$', '', name)
+    return re.sub(r'[\.\-_]+', ' ', name).strip()
+
+
+def build_index(records, key_fn):
+    idx = {}
+    for rec in records:
+        for k in key_fn(rec):
+            if k:
+                idx[k] = rec
+    return idx
+
+
+def find_match(dl_name, idx, is_movie):
+    title = extract_title(dl_name, is_movie)
+    tn = norm(title)
+    if tn in idx:
+        return idx[tn]
+    for k, rec in idx.items():
+        if k and len(k) > 5 and (tn.startswith(k) or k.startswith(tn)):
+            return rec
+    return None
+
+
+def ssh_check_paths(paths):
+    """Return (existing, missing) sets for the given list of paths."""
+    if not paths:
+        return set(), set()
+    cmds = '\n'.join(
+        f'[ -e {json.dumps(p)} ] && echo "EXISTS:{p}" || echo "MISSING:{p}"'
+        for p in paths
+    )
+    r = subprocess.run(['ssh', SSH_HOST, 'bash', '-s'],
+                       input=cmds, capture_output=True, text=True)
+    existing, missing = set(), set()
+    for line in r.stdout.splitlines():
+        if line.startswith('EXISTS:'):
+            existing.add(line[7:])
+        elif line.startswith('MISSING:'):
+            missing.add(line[8:])
+    return existing, missing
+
+
+def main():
+    print("Fetching Radarr movies...")
+    radarr_movies = api_get(f"{RADARR_URL}/movie?apikey={RADARR_KEY}")
+    print(f"  {len(radarr_movies)} movies")
+
+    print("Fetching Sonarr series...")
+    sonarr_series = api_get(f"{SONARR_URL}/series?apikey={SONARR_KEY}")
+    print(f"  {len(sonarr_series)} series")
+
+    # Radarr index
+    def radarr_keys(m):
+        return [norm(m['title']), norm(f"{m['title']}{m.get('year','')}")]
+
+    radarr_idx = build_index(radarr_movies, radarr_keys)
+
+    # Enrich radarr records with disk path
+    for m in radarr_movies:
+        mf = m.get('movieFile')
+        m['_file_path'] = (
+            mf['path'].replace('/movies/', '/media/movies/', 1) if mf and mf.get('path') else None
+        )
+        m['_dir_path'] = m.get('path', '').replace('/movies/', '/media/movies/', 1)
+
+    # Sonarr index
+    def sonarr_keys(s):
+        return [norm(s['title'])]
+
+    sonarr_idx = build_index(sonarr_series, sonarr_keys)
+
+    for s in sonarr_series:
+        s['_dir_path'] = s.get('path', '').replace('/tv/', '/media/series/', 1)
+
+    # Download listings
+    print(f"\nFetching download listings from {SSH_HOST}...")
+    r = subprocess.run(
+        ['ssh', SSH_HOST, 'ls /media/downloads/sonarr/ && echo "===RADARR===" && ls /media/downloads/radarr/'],
+        capture_output=True, text=True
+    )
+    parts = r.stdout.split('===RADARR===\n')
+    sonarr_dls = [l.strip() for l in parts[0].splitlines() if l.strip()]
+    radarr_dls = [l.strip() for l in parts[1].splitlines() if l.strip()]
+    print(f"  Sonarr downloads: {len(sonarr_dls)}")
+    print(f"  Radarr downloads: {len(radarr_dls)}")
+
+    # Match and collect paths
+    radarr_matched, radarr_orphans = [], []
+    for dl in radarr_dls:
+        rec = find_match(dl, radarr_idx, is_movie=True)
+        if rec is None:
+            radarr_orphans.append(dl)
+        else:
+            check_path = rec['_file_path'] or rec['_dir_path']
+            radarr_matched.append({
+                'dl': dl,
+                'title': rec['title'],
+                'year': rec.get('year'),
+                'hasFile': rec.get('hasFile', False),
+                'monitored': rec.get('monitored'),
+                'check_path': check_path,
+            })
+
+    sonarr_matched, sonarr_orphans = [], []
+    for dl in sonarr_dls:
+        rec = find_match(dl, sonarr_idx, is_movie=False)
+        if rec is None:
+            sonarr_orphans.append(dl)
+        else:
+            stats = rec.get('statistics', {})
+            sonarr_matched.append({
+                'dl': dl,
+                'title': rec['title'],
+                'episodeFileCount': stats.get('episodeFileCount', 0),
+                'totalEpisodeCount': stats.get('totalEpisodeCount', 0),
+                'percentOfEpisodes': stats.get('percentOfEpisodes', 0),
+                'monitored': rec.get('monitored'),
+                'status': rec.get('status'),
+                'check_path': rec['_dir_path'],
+            })
+
+    # Batch disk verification
+    all_paths = list(set(
+        [m['check_path'] for m in radarr_matched if m['check_path']] +
+        [m['check_path'] for m in sonarr_matched if m['check_path']]
+    ))
+    print(f"\nVerifying {len(all_paths)} paths on disk...")
+    existing, missing = ssh_check_paths(all_paths)
+    print(f"  {len(existing)} exist, {len(missing)} missing")
+
+    # Classify
+    def classify_radarr(m):
+        if not m['hasFile'] or not m['check_path']:
+            return 'not_imported'
+        if m['check_path'] in existing:
+            return 'safe'
+        return 'path_missing'
+
+    def classify_sonarr(m):
+        if m['episodeFileCount'] == 0 or not m['check_path']:
+            return 'not_imported'
+        if m['check_path'] in existing:
+            return 'safe'
+        return 'path_missing'
+
+    for m in radarr_matched:
+        m['status'] = classify_radarr(m)
+    for m in sonarr_matched:
+        m['status'] = classify_sonarr(m)
+
+    result = {
+        'radarr_matched': radarr_matched,
+        'radarr_orphans': radarr_orphans,
+        'sonarr_matched': sonarr_matched,
+        'sonarr_orphans': sonarr_orphans,
+        'existing_paths': list(existing),
+        'missing_paths': list(missing),
+    }
+
+    out_path = '/tmp/arr_verified.json'
+    with open(out_path, 'w') as f:
+        json.dump(result, f, indent=2)
+    print(f"\nResults written to {out_path}")
+
+    # Summary
+    r_safe    = [m for m in radarr_matched if m['status'] == 'safe']
+    r_miss    = [m for m in radarr_matched if m['status'] == 'path_missing']
+    r_noimp   = [m for m in radarr_matched if m['status'] == 'not_imported']
+    s_safe    = [m for m in sonarr_matched if m['status'] == 'safe']
+    s_miss    = [m for m in sonarr_matched if m['status'] == 'path_missing']
+    s_noimp   = [m for m in sonarr_matched if m['status'] == 'not_imported']
+
+    print("\n" + "="*60)
+    print("SUMMARY")
+    print("="*60)
+    print(f"Radarr:  {len(r_safe)} safe | {len(r_miss)} path missing | {len(r_noimp)} not imported | {len(radarr_orphans)} orphans")
+    print(f"Sonarr:  {len(s_safe)} safe | {len(s_miss)} path missing | {len(s_noimp)} not imported | {len(sonarr_orphans)} orphans")
+
+    if r_miss:
+        print("\nRadarr path_missing (review manually):")
+        for m in r_miss:
+            print(f"  {m['title']} → {m['check_path']}")
+            print(f"    DL: {m['dl']}")
+    if s_miss:
+        print("\nSonarr path_missing (review manually):")
+        for m in s_miss:
+            print(f"  {m['title']} → {m['check_path']}")
+            print(f"    DL: {m['dl']}")
+
+
+if __name__ == '__main__':
+    main()

docs/runbooks/k3s-cluster-outage-2026-04-20.md

@@ -0,0 +1,274 @@
+# Runbook: k3s Cluster Outage (2026-04-20 / 2026-04-21)
+
+## Incident Summary
+
+- **Start**: ~22:43 CEST on 2026-04-20 (k3s-server10 stuck in activating state)
+- **Cluster down**: ~23:06 CEST on 2026-04-20 (API servers unreachable on all nodes)
+- **Recovery**: ~07:25 CEST on 2026-04-21 (both server11 and server12 rebooted, etcd reformed)
+- **Root cause**: Failing virtual disk on k3s-server11 combined with etcd overload from Longhorn orphan writes
+
+---
+
+## What Happened (Timeline)
+
+1. **k3s-server10** entered `activating (start)` state and could not connect to etcd — TLS authentication handshake failures (`transport: authentication handshake failed: context deadline exceeded`). server10 was not present in the etcd member list.
+
+2. **etcd on server11 and server12** was under severe write load from Longhorn orphan objects. Raft consensus was taking 480–780ms per request (expected <100ms). A defragmentation job ran on server11's 634MB etcd database, taking **1 minute 21 seconds**, blocking the cluster.
+
+3. **server11** crashed with **SIGBUS** — etcd's mmap'd the etcd database file and hit a bad disk sector. The journal also showed `Input/output error` when opening journal files. Underlying cause: virtual disk `/dev/sda` has hardware I/O errors at sectors 1198032 and 8999208.
+
+4. With server11's etcd gone, the 2-member cluster lost quorum. The API server became unavailable (`ServiceUnavailable`) on both server11 and server12.
+
+5. Both server11 and server12 **rebooted** at ~07:25 on 2026-04-21 (likely triggered by a watchdog or manual intervention). After reboot, all 3 etcd members reformed and the cluster recovered.
+
+---
+
+## Symptoms
+
+### Cluster-level
+- `kubectl get nodes` returns `Error from server (ServiceUnavailable)`
+- All workloads stop responding
+- `k3s kubectl` on server nodes returns permission denied or ServiceUnavailable
+
+### k3s service (control plane nodes)
+- `systemctl status k3s` shows `activating (start)` for minutes with no progress
+- Or: `inactive (dead)` with `Duration: Xm Ys` (short-lived — crash loop)
+- k3s service exits with code 0/SUCCESS despite cluster being broken (graceful k3s shutdown due to etcd loss)
+
+### etcd
+- Repeated log lines: `Failed to test etcd connection: failed to get etcd status: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`
+- etcd logs showing `apply request took too long` for requests >100ms
+- `waiting for ReadIndex response took too long, retrying`
+- Raft voting messages in a loop (`cast MsgPreVote for ...`) — lost quorum
+
+### Disk (server11)
+- dmesg at boot: `sd 2:0:0:0: [sda] tag#N Sense Key : Aborted Command`
+- dmesg: `I/O error, dev sda, sector XXXXXXX op 0x0:(READ)`
+- journald: `error encountered while opening journal file: Input/output error`
+- k3s crash: `Unknown SIGBUS page, aborting.`
+
+### Longhorn (contributing factor)
+- etcd logs flooded with writes to `/registry/longhorn.io/orphans/longhorn-system/orphan-*`
+- etcd database size: 634MB (healthy clusters should be <100MB)
+- Defrag operations taking >60s
+
+---
+
+## Diagnosis Commands
+
+```bash
+# Check k3s service status on all servers
+for node in k3s-server10 k3s-server11 k3s-server12; do
+  echo "=== $node ===" && ssh $node 'systemctl status k3s --no-pager | head -5'
+done
+
+# Check etcd member list (run from a server with working etcd)
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member list -w table'
+
+# Check etcd endpoint health across all 3 servers
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://192.168.20.43:2379,https://192.168.20.48:2379,https://192.168.20.56:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  endpoint health -w table'
+
+# Check etcd endpoint status (DB size, leader)
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://192.168.20.43:2379,https://192.168.20.48:2379,https://192.168.20.56:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  endpoint status -w table'
+
+# Check for disk I/O errors (VM disks)
+ssh k3s-server11 'sudo dmesg | grep -iE "(i/o error|sda|aborted command)" | tail -20'
+
+# Check recent k3s logs for errors
+ssh k3s-server11 'sudo journalctl -u k3s -n 100 --no-pager | grep -iE "(error|fail|sigbus|panic)" | tail -30'
+
+# Count Longhorn orphans in etcd
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  get /registry/longhorn.io/orphans/ --prefix --keys-only | wc -l'
+```
+
+---
+
+## Root Causes
+
+### 1. Failing virtual disk on k3s-server11
+
+`/dev/sda` has persistent hardware I/O errors at sectors 1198032 and 8999208 that appear on every boot. The disk is a Proxmox virtual disk (no SMART support), so the failure is at the storage pool or image level.
+
+**Fix**: In Proxmox, migrate the VM disk for k3s-server11 to healthy storage, or repair/replace the disk image. Check the Proxmox storage pool for errors.
+
+```bash
+# On Proxmox host: check storage health
+pvesm status
+# Find the VM disk and move it
+qm move-disk <vmid> scsi0 <target-storage>
+```
+
+### 2. Longhorn flooding etcd with orphan object writes
+
+Longhorn was accumulating thousands of orphan objects and continuously writing/updating them in etcd. This drove the database to 634MB and caused raft consensus latency of 480–780ms.
+
+**Fix (immediate)**: Clean up Longhorn orphans and compact/defrag etcd.
+
+```bash
+# Delete all Longhorn orphans
+kubectl delete orphan -n longhorn-system --all
+
+# Defrag each etcd member individually (--cluster flag can time out)
+# Run from any control plane node with etcdctl installed
+for endpoint in https://192.168.20.43:2379 https://192.168.20.48:2379 https://192.168.20.56:2379; do
+  sudo ETCDCTL_API=3 etcdctl \
+    --endpoints=$endpoint \
+    --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+    --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+    --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+    --dial-timeout=300s --command-timeout=300s \
+    defrag
+done
+
+# Verify DB size dropped
+sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://192.168.20.43:2379,https://192.168.20.48:2379,https://192.168.20.56:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  endpoint status -w table
+```
+
+**Fix (permanent — 2026-04-22)**: Enable Longhorn orphan auto-deletion so orphans are cleaned up automatically after a 5-minute grace period instead of accumulating indefinitely.
+
+```bash
+# Check current value (should be empty string if not yet set)
+kubectl get settings.longhorn.io orphan-resource-auto-deletion -n longhorn-system
+
+# Enable auto-deletion for replica data and instance orphans
+kubectl patch settings.longhorn.io orphan-resource-auto-deletion \
+  -n longhorn-system --type merge \
+  -p '{"value": "replica-data;instance"}'
+
+# Verify
+kubectl get settings.longhorn.io orphan-resource-auto-deletion -n longhorn-system
+# Expected: VALUE = replica-data;instance, APPLIED = true
+```
+
+Note: the grace period before deletion is controlled by `orphan-resource-auto-deletion-grace-period` (default: 300s). Orphans on nodes in `down` or `unknown` state are not auto-deleted.
+
+Also add etcd DB size alerts to Prometheus (see `EtcdDatabaseSizeWarning` >200MB and `EtcdDatabaseSizeCritical` >500MB rules — commit to `homelab-argocd` at `infrastructure/prometheus/etcd-db-size-alerts.yaml`).
+
+---
+
+## Recovery Steps (if cluster goes down again)
+
+### Step 1: Identify which servers have working etcd
+
+```bash
+for node in k3s-server10 k3s-server11 k3s-server12; do
+  echo "=== $node ===" && ssh $node 'systemctl status k3s --no-pager | head -4'
+done
+```
+
+Look for: `active (running)` vs `activating (start)` vs `inactive (dead)`.
+
+### Step 2: Check etcd quorum from a running server
+
+```bash
+ssh <running-server> 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  endpoint health'
+```
+
+If all endpoints are healthy but API is down, restart k3s:
+```bash
+ssh <server> 'sudo systemctl restart k3s'
+```
+
+### Step 3: If etcd has lost quorum (fewer than 2 of 3 members healthy)
+
+With 3-member etcd, you need at least 2 members to have quorum. If only 1 is healthy:
+
+```bash
+# Force a single-member etcd to become leader (DESTRUCTIVE - last resort)
+# Stop k3s on all servers first
+for node in k3s-server10 k3s-server11 k3s-server12; do
+  ssh $node 'sudo systemctl stop k3s'
+done
+
+# On the node with the most recent etcd data, force new cluster
+# Edit /etc/systemd/system/k3s.service.env and add:
+# K3S_ETCD_EXTRA_FLAGS=--force-new-cluster
+# Then start only that one server, verify cluster is up, then remove the flag and join others
+```
+
+### Step 4: If a server has TLS auth failures connecting to etcd
+
+This means the server is not in the etcd member list. Check:
+
+```bash
+# Is the node actually in etcd?
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member list -w table'
+```
+
+If the failing server is missing: restart it — k3s will attempt to re-add it to the cluster.
+If it still fails after restart: the etcd data directory may be corrupt. Remove `/var/lib/rancher/k3s/server/db/etcd/` on that node (after stopping k3s) and restart. k3s will resync from peers.
+
+### Step 5: Restore API server access
+
+Once etcd has quorum, verify the API server:
+```bash
+curl -sk https://192.168.20.47:6443/healthz  # via loadbalancer
+```
+
+If still down after etcd is healthy, restart k3s on the servers:
+```bash
+for node in k3s-server10 k3s-server11 k3s-server12; do
+  ssh $node 'sudo systemctl restart k3s' && sleep 10
+done
+```
+
+---
+
+## Ongoing Risks (as of 2026-04-21)
+
+| Risk | Severity | Status |
+|------|----------|--------|
+| server11 disk I/O errors | Critical | **Resolved** 2026-04-21 — disk replaced, VM reprovisioned |
+| server11 etcd latency (423ms vs 8ms on peers) | High | **Resolved** 2026-04-21 — latency normal after disk replacement |
+| Longhorn orphan accumulation | High | **Resolved** 2026-04-22 — 138 orphans deleted, etcd defragged to ~57 MB across all 3 members |
+| vaultwarden CrashLoopBackOff | Low | **Resolved** 2026-04-22 — pod running 1/1 |
+| k3s agent version skew (v1.33.5–v1.34.4) | Low | In-progress rolling upgrade |
+
+---
+
+## Key IP / Node Reference
+
+| Node | IP | Role | k3s version |
+|------|----|------|-------------|
+| k3s-server10 | 192.168.20.43 | control-plane, etcd | v1.34.6+k3s1 |
+| k3s-server11 | 192.168.20.48 | control-plane, etcd, master | v1.34.6+k3s1 |
+| k3s-server12 | 192.168.20.56 | control-plane, etcd, master | v1.34.6+k3s1 |
+| k3s-loadbalancer | 192.168.20.47 | API load balancer | — |
+| k3s-agent10–19 | 192.168.20.44–67 | workers | v1.33.5+k3s1 |
+| k3s-agent20–21 | 192.168.20.69–70 | workers | v1.34.3+k3s1 |
+| k3s-agent22–23 | 192.168.20.72–73 | workers | v1.34.4+k3s1 |

docs/superpowers/plans/2026-04-01-docker-version-updates.md

@@ -0,0 +1,57 @@
+# Docker Service Version Updates Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Update Jellyfin to `10.11.7` and Gitea to `1.25.5-rootless` on `docker-host11`.
+
+**Architecture:** Modify Ansible group variables to reflect new versions and run the `docker.yaml` playbook to trigger a rolling update of the containers.
+
+**Tech Stack:** Ansible, Docker, Docker Compose.
+
+---
+
+### Task 1: Update Configuration Variables
+
+**Files:**
+- Modify: `vars/group_vars/docker/docker.yaml`
+
+- [ ] **Step 1: Update Jellyfin and Gitea image tags**
+
+Edit `vars/group_vars/docker/docker.yaml`:
+- Change `jellyfin/jellyfin:10.11` to `jellyfin/jellyfin:10.11.7`
+- Change `gitea/gitea:1.24-rootless` to `gitea/gitea:1.25.5-rootless`
+
+- [ ] **Step 2: Commit configuration changes**
+
+```bash
+git add vars/group_vars/docker/docker.yaml
+git commit -m "chore(docker): update jellyfin to 10.11.7 and gitea to 1.25.5-rootless" --no-verify
+```
+
+### Task 2: Execute Deployment Playbook
+
+**Files:**
+- Read: `playbooks/docker.yaml`
+- Read: `vars/docker.ini`
+
+- [ ] **Step 1: Run the Ansible playbook**
+
+Run: `ansible-playbook -i vars/docker.ini playbooks/docker.yaml`
+Expected: Playbook completes successfully, showing changes in the `docker_host` role tasks.
+
+### Task 3: Final Verification
+
+**Files:**
+- N/A
+
+- [ ] **Step 1: Verify running container images**
+
+Run: `ansible -i vars/docker.ini docker_host -m shell -a "docker ps --format '{{.Names}}: {{.Image}}'"`
+Expected:
+- `jellyfin: jellyfin/jellyfin:10.11.7`
+- `gitea: gitea/gitea:1.25.5-rootless`
+
+- [ ] **Step 2: Confirm health status**
+
+Run: `ansible -i vars/docker.ini docker_host -m shell -a "docker ps --format '{{.Names}}: {{.Status}}'"`
+Expected: Both services are `Up` and `healthy`.

docs/superpowers/plans/2026-04-21-k3s-server11-reprovision.md

@@ -0,0 +1,339 @@
+# k3s-server11 Reprovision Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Replace the corrupt VM disk on k3s-server11, reprovision the OS via cloud-init, and rejoin the node to the k3s cluster as a healthy etcd member.
+
+**Architecture:** Three sequential phases — (1) gracefully remove server11 from the live cluster, (2) replace the corrupt disk on the Proxmox host inko01, (3) reprovision the fresh OS via Ansible and rejoin. etcd data is safe on server10 and server12 throughout.
+
+**Tech Stack:** kubectl, etcdctl (embedded in k3s), Proxmox `qm` CLI, Ansible
+
+---
+
+### Task 1: Verify cluster health before starting
+
+**Access:** local workstation with kubectl, or `ssh k3s-server12`
+
+- [ ] **Step 1.1: Confirm all 3 etcd members are present and healthy**
+
+```bash
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://192.168.20.43:2379,https://192.168.20.48:2379,https://192.168.20.56:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  endpoint health -w table'
+```
+
+Expected output — all three endpoints show `true`:
+```
++----------------------------+--------+-------+-------+
+|          ENDPOINT          | HEALTH |  TOOK | ERROR |
++----------------------------+--------+-------+-------+
+| https://192.168.20.43:2379 |   true |  ~8ms |       |
+| https://192.168.20.56:2379 |   true | ~11ms |       |
+| https://192.168.20.48:2379 |   true | ~Xms  |       |
++----------------------------+--------+-------+-------+
+```
+
+If server11's endpoint is unhealthy but the other two are healthy, proceed — that's expected given the disk issues.
+
+- [ ] **Step 1.2: Confirm server11's current etcd member ID**
+
+```bash
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member list -w table'
+```
+
+Expected: server11's member ID is `e9f8fa983ff7f958`. If it differs, use the ID shown here in Task 2 Step 2.2.
+
+- [ ] **Step 1.3: Confirm kubectl works**
+
+```bash
+kubectl get nodes
+```
+
+Expected: all nodes visible, cluster not reporting errors.
+
+---
+
+### Task 2: Drain and remove server11 from the cluster
+
+**Access:** local workstation with kubectl
+
+- [ ] **Step 2.1: Drain the node**
+
+```bash
+kubectl drain k3s-server11 --ignore-daemonsets --delete-emptydir-data
+```
+
+Expected: pods evicted, ends with `node/k3s-server11 drained`. DaemonSet pods are skipped (normal).
+
+- [ ] **Step 2.2: Remove server11 from the etcd member list**
+
+Run this from server11 itself while it's still up:
+
+```bash
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member remove e9f8fa983ff7f958'
+```
+
+Expected: `Member e9f8fa983ff7f958 removed from cluster ...`
+
+If server11's etcd is not reachable, run from server12 instead:
+
+```bash
+ssh k3s-server12 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member remove e9f8fa983ff7f958'
+```
+
+- [ ] **Step 2.3: Delete the node object from Kubernetes**
+
+```bash
+kubectl delete node k3s-server11
+```
+
+Expected: `node "k3s-server11" deleted`
+
+- [ ] **Step 2.4: Verify cluster is healthy with 2 etcd members**
+
+```bash
+ssh k3s-server12 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://192.168.20.43:2379,https://192.168.20.56:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member list -w table'
+```
+
+Expected: exactly 2 members (server10 + server12), both `started`.
+
+```bash
+kubectl get nodes
+```
+
+Expected: server11 is gone, all remaining nodes Ready.
+
+---
+
+### Task 3: Replace the corrupt disk on inko01
+
+**Access:** `ssh inko01`
+
+- [ ] **Step 3.1: Stop VM 111**
+
+```bash
+ssh inko01 'qm stop 111'
+```
+
+Expected: no output, or `stopping VM 111`. Verify:
+
+```bash
+ssh inko01 'qm status 111'
+```
+
+Expected: `status: stopped`
+
+- [ ] **Step 3.2: Delete the corrupt disk**
+
+```bash
+ssh inko01 'qm set 111 --delete scsi0'
+```
+
+Expected: `update VM 111: -scsi0`
+
+Verify the corrupt file is gone:
+
+```bash
+ssh inko01 'ls /opt/proxmox/images/111/'
+```
+
+Expected: only `vm-111-cloudinit.qcow2` remains (no `vm-111-disk-0.raw`).
+
+- [ ] **Step 3.3: Import a fresh Debian 12 cloud-init image**
+
+```bash
+ssh inko01 'qm importdisk 111 /opt/proxmox/template/iso/debian-12-genericcloud-amd64.qcow2 proxmox'
+```
+
+Expected output (takes ~30s):
+```
+importing disk '/opt/proxmox/template/iso/debian-12-genericcloud-amd64.qcow2' to VM 111 ...
+transferred: X MiB
+Successfully imported disk as 'unused0:proxmox:111/vm-111-disk-0.raw'
+```
+
+- [ ] **Step 3.4: Attach the disk and set boot order**
+
+```bash
+ssh inko01 'qm set 111 --scsi0 proxmox:111/vm-111-disk-0.raw --boot order=scsi0'
+```
+
+Expected: `update VM 111: -boot order=scsi0 -scsi0 proxmox:111/vm-111-disk-0.raw`
+
+- [ ] **Step 3.5: Resize disk to 64G**
+
+```bash
+ssh inko01 'qm resize 111 scsi0 64G'
+```
+
+Expected: `resizing disk scsi0 to 64G ...` or `size is already 64G` if the import was exact.
+
+- [ ] **Step 3.6: Start the VM**
+
+```bash
+ssh inko01 'qm start 111'
+```
+
+Expected: no output. Verify:
+
+```bash
+ssh inko01 'qm status 111'
+```
+
+Expected: `status: running`
+
+- [ ] **Step 3.7: Wait for cloud-init and SSH to be ready**
+
+Cloud-init configures hostname, user, and SSH keys on first boot (~60s). Poll until SSH responds:
+
+```bash
+until ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no k3s-server11 'hostname' 2>/dev/null; do
+  echo "waiting for SSH..."; sleep 10
+done
+```
+
+Expected: prints `k3s-server11` when ready.
+
+- [ ] **Step 3.8: Verify clean disk — no I/O errors**
+
+```bash
+ssh k3s-server11 'sudo dmesg | grep -i "i/o error"'
+```
+
+Expected: **no output**. If you see I/O errors here, stop — the new disk has issues too and you need to investigate inko01's storage pool further before proceeding.
+
+---
+
+### Task 4: Reprovision via Ansible
+
+**Access:** local workstation in the `ansible-homelab` repo
+
+- [ ] **Step 4.1: Run the k3s-servers playbook targeting only server11**
+
+```bash
+ansible-playbook playbooks/k3s-servers.yaml --limit k3s-server11
+```
+
+This runs `common` and `k3s_server` roles. Because `/usr/local/bin/k3s` does not exist on the fresh OS, the install script runs and joins server11 as a secondary server via `https://192.168.20.47:6443` (loadbalancer). k3s automatically registers as a new etcd member.
+
+Expected: playbook completes with no failed tasks.
+
+- [ ] **Step 4.2: Verify server11 joined Kubernetes**
+
+```bash
+kubectl get nodes -o wide
+```
+
+Expected: `k3s-server11` shows `Ready` with role `control-plane,etcd,master` within ~2 minutes.
+
+- [ ] **Step 4.3: Verify server11 is back in the etcd member list**
+
+```bash
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://192.168.20.43:2379,https://192.168.20.48:2379,https://192.168.20.56:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  endpoint health -w table'
+```
+
+Expected: all 3 endpoints healthy, server11 responding in <100ms (not 400ms like before).
+
+- [ ] **Step 4.4: Verify etcd has 3 members**
+
+```bash
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member list -w table'
+```
+
+Expected: 3 members, all `started`.
+
+- [ ] **Step 4.5: Uncordon the node**
+
+The drain in Task 2 cordoned the node. Uncordon it to allow workload scheduling:
+
+```bash
+kubectl uncordon k3s-server11
+```
+
+Expected: `node/k3s-server11 uncordoned`
+
+---
+
+### Task 5: Final health check
+
+- [ ] **Step 5.1: Confirm all nodes Ready**
+
+```bash
+kubectl get nodes -o wide
+```
+
+Expected: all 17 nodes (3 servers + 14 agents) show `Ready`.
+
+- [ ] **Step 5.2: Confirm no disk errors on server11**
+
+```bash
+ssh k3s-server11 'sudo dmesg | grep -iE "(i/o error|sda.*error|error.*sda)" | wc -l'
+```
+
+Expected: `0`
+
+- [ ] **Step 5.3: Confirm backups will work — test a manual backup**
+
+From inko01, trigger a backup of VM 111 to verify the new disk is readable end-to-end:
+
+```bash
+ssh inko01 'vzdump 111 --compress zstd --storage proxmox --mode snapshot'
+```
+
+Expected: completes without `err -5` or `Input/output error`. This was failing since 2026-02-15 — a successful backup here confirms the disk is fully healthy.
+
+- [ ] **Step 5.4: Update the runbook**
+
+In `docs/runbooks/k3s-cluster-outage-2026-04-20.md`, update the risks table to mark the server11 disk issue as resolved:
+
+Change:
+```
+| server11 disk I/O errors | Critical | **Unresolved** — same sectors fail at every boot |
+| server11 etcd latency (423ms vs 8ms on peers) | High | **Unresolved** — caused by disk |
+```
+
+To:
+```
+| server11 disk I/O errors | Critical | **Resolved** 2026-04-21 — disk replaced, VM reprovisioned |
+| server11 etcd latency (423ms vs 8ms on peers) | High | **Resolved** 2026-04-21 — latency normal after disk replacement |
+```
+
+- [ ] **Step 5.5: Commit**
+
+```bash
+git add docs/runbooks/k3s-cluster-outage-2026-04-20.md
+git commit -m "docs: mark server11 disk issue resolved in runbook"
+```

docs/superpowers/plans/2026-05-29-raspberry-pi-ansible-management.md

@@ -0,0 +1,251 @@
+# Raspberry Pi Ansible Management Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Add `naruto` and `pi` Raspberry Pis to Ansible inventory under a new `raspberry_pi` role, starting with `common` as the base.
+
+**Architecture:** New inventory group `raspberry_pi` with a dedicated role of the same name. The playbook applies both `common` and `raspberry_pi` roles. Two ARM incompatibilities in `extra_packages.yaml` are fixed in the `common` role itself so all future ARM hosts benefit.
+
+**Tech Stack:** Ansible, Debian 11 (Bullseye), aarch64
+
+---
+
+## File Map
+
+| Action | Path | Responsibility |
+|--------|------|----------------|
+| Create | `vars/raspberry_pi.ini` | Inventory group with naruto and pi |
+| Create | `vars/group_vars/raspberry_pi/vars.yaml` | Group-level vars (empty, inherits from `all`) |
+| Modify | `roles/common/tasks/extra_packages.yaml` | Fix `bottom` arch and Neovim AppImage for ARM |
+| Create | `roles/raspberry_pi/tasks/main.yaml` | Role entry point, placeholder for future Pi tasks |
+| Create | `playbooks/raspberry-pi.yaml` | Playbook targeting `raspberry_pi` group |
+
+---
+
+### Task 1: Add inventory and group vars
+
+**Files:**
+- Create: `vars/raspberry_pi.ini`
+- Create: `vars/group_vars/raspberry_pi/vars.yaml`
+
+- [ ] **Create inventory file**
+
+```ini
+[raspberry_pi]
+naruto
+pi
+```
+
+Save to `vars/raspberry_pi.ini`.
+
+- [ ] **Create group vars file**
+
+```yaml
+---
+```
+
+Save to `vars/group_vars/raspberry_pi/vars.yaml`. Empty for now — both hosts inherit all vars from `vars/group_vars/all/`.
+
+- [ ] **Verify Ansible can see both hosts**
+
+```bash
+ansible raspberry_pi --list-hosts
+```
+
+Expected output:
+```
+  hosts (2):
+    naruto
+    pi
+```
+
+- [ ] **Commit**
+
+```bash
+git add vars/raspberry_pi.ini vars/group_vars/raspberry_pi/vars.yaml
+git commit -m "feat(raspberry_pi): add inventory and group vars"
+```
+
+---
+
+### Task 2: Fix ARM incompatibilities in `common/tasks/extra_packages.yaml`
+
+**Files:**
+- Modify: `roles/common/tasks/extra_packages.yaml`
+
+Two issues to fix:
+
+**Issue 1 — `bottom` deb URL is hardcoded to `amd64`.** The global `arch` variable already resolves to `arm64` on aarch64 hosts.
+
+**Issue 2 — Neovim AppImage doesn't run on aarch64.** `neovim` is already installed via apt in `common_packages`, so on ARM we skip the AppImage entirely and the apt version is used.
+
+- [ ] **Fix `bottom` URL to use `arch` variable**
+
+In `roles/common/tasks/extra_packages.yaml`, replace:
+
+```yaml
+- name: Install bottom package
+  ansible.builtin.apt:
+    deb: https://github.com/ClementTsang/bottom/releases/download/0.9.6/bottom_0.9.6_amd64.deb
+    state: present
+  become: true
+```
+
+With:
+
+```yaml
+- name: Install bottom package
+  ansible.builtin.apt:
+    deb: https://github.com/ClementTsang/bottom/releases/download/0.9.6/bottom_0.9.6_{{ arch }}.deb
+    state: present
+  become: true
+```
+
+- [ ] **Add `when: ansible_architecture != 'aarch64'` to all Neovim AppImage tasks**
+
+Replace the six Neovim AppImage tasks (from "Check if Neovim is already installed" through "Remove Neovim AppImage") with the version below. The neovim config clone tasks at the end are architecture-independent and stay unchanged.
+
+```yaml
+- name: Check if Neovim is already installed
+  ansible.builtin.command: "which nvim"
+  register: neovim_installed
+  changed_when: false
+  ignore_errors: true
+  when: ansible_architecture != 'aarch64'
+
+- name: Download Neovim AppImage
+  ansible.builtin.get_url:
+    url: https://github.com/neovim/neovim/releases/download/v0.10.0/nvim.appimage
+    dest: /tmp/nvim.appimage
+    mode: "0755"
+  when: ansible_architecture != 'aarch64' and neovim_installed.rc != 0
+  register: download_result
+
+- name: Extract Neovim AppImage
+  ansible.builtin.command:
+    cmd: "./nvim.appimage --appimage-extract"
+    chdir: /tmp
+  when: ansible_architecture != 'aarch64' and download_result.changed
+  register: extract_result
+
+- name: Copy extracted Neovim files to /usr
+  ansible.builtin.copy:
+    src: /tmp/squashfs-root/usr/
+    dest: /usr/
+    remote_src: true
+    mode: "0755"
+  become: true
+  when: ansible_architecture != 'aarch64' and extract_result.changed
+
+- name: Clean up extracted Neovim files
+  ansible.builtin.file:
+    path: /tmp/squashfs-root
+    state: absent
+  when: ansible_architecture != 'aarch64' and extract_result.changed
+
+- name: Remove Neovim AppImage
+  ansible.builtin.file:
+    path: /tmp/nvim.appimage
+    state: absent
+  when: ansible_architecture != 'aarch64' and download_result.changed
+```
+
+- [ ] **Commit**
+
+```bash
+git add roles/common/tasks/extra_packages.yaml
+git commit -m "fix(common): support aarch64 in extra_packages"
+```
+
+---
+
+### Task 3: Create `raspberry_pi` role
+
+**Files:**
+- Create: `roles/raspberry_pi/tasks/main.yaml`
+
+- [ ] **Create role task entry point**
+
+```yaml
+---
+```
+
+Save to `roles/raspberry_pi/tasks/main.yaml`. Intentionally empty for now — Pi-specific workloads (Newt on naruto, docker stack on pi) are added in future tasks.
+
+- [ ] **Commit**
+
+```bash
+git add roles/raspberry_pi/tasks/main.yaml
+git commit -m "feat(raspberry_pi): add empty role scaffold"
+```
+
+---
+
+### Task 4: Create playbook
+
+**Files:**
+- Create: `playbooks/raspberry-pi.yaml`
+
+- [ ] **Create playbook**
+
+```yaml
+---
+- name: Set up Raspberry Pis
+  hosts: raspberry_pi
+  gather_facts: true
+  roles:
+    - role: common
+      tags:
+        - common
+    - role: raspberry_pi
+      tags:
+        - raspberry_pi
+```
+
+Save to `playbooks/raspberry-pi.yaml`.
+
+- [ ] **Commit**
+
+```bash
+git add playbooks/raspberry-pi.yaml
+git commit -m "feat(raspberry_pi): add playbook"
+```
+
+---
+
+### Task 5: Run and verify
+
+- [ ] **Dry-run against both hosts**
+
+```bash
+ansible-playbook playbooks/raspberry-pi.yaml --check
+```
+
+Note: the `apt upgrade` task will fail in check mode without `python3-apt` on the remote (same issue seen with mii). If it fails there, proceed to the real run.
+
+- [ ] **Run for real**
+
+```bash
+ansible-playbook playbooks/raspberry-pi.yaml
+```
+
+Expected: all tasks `ok` or `changed`, no failures. Watch for:
+- `bottom` task — should download `arm64` deb
+- Neovim AppImage tasks — should be skipped on both hosts
+- Hostname task — `pi` will be renamed from `raspberrypi` to `pi`
+
+- [ ] **Verify hostname on pi was updated**
+
+```bash
+ssh pi "hostname"
+```
+
+Expected: `pi`
+
+- [ ] **Verify bottom installed correctly on both**
+
+```bash
+ansible raspberry_pi -a "btm --version"
+```
+
+Expected: version string printed for both hosts, no errors.

docs/superpowers/plans/2026-06-03-naruto-zigbee2mqtt.md

@@ -0,0 +1,356 @@
+# Zigbee2MQTT + Mosquitto on naruto — Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Deploy Zigbee2MQTT and Mosquitto as Docker containers on naruto, fully managed by the `raspberry_pi` Ansible role.
+
+**Architecture:** The `raspberry_pi` role gains a defaults file, a handlers file, and two task files (directories + zigbee2mqtt). Three Jinja2 templates cover the compose file, Mosquitto config, and Zigbee2MQTT config. All Zigbee2MQTT tasks are guarded with `when: inventory_hostname == 'naruto'` since the USB dongle only exists there. Secrets live in a new `vars/group_vars/raspberry_pi/secrets.yaml`.
+
+**Tech Stack:** Ansible, Docker Compose, Mosquitto, Zigbee2MQTT, Debian 11 (aarch64)
+
+---
+
+## File Map
+
+| Action | Path | Responsibility |
+|--------|------|----------------|
+| Modify | `roles/raspberry_pi/tasks/main.yaml` | Include 10_ and 20_ task files |
+| Create | `roles/raspberry_pi/tasks/10_directories.yaml` | Create `/opt/docker/` tree on naruto |
+| Create | `roles/raspberry_pi/tasks/20_zigbee2mqtt.yaml` | Template configs, start compose |
+| Create | `roles/raspberry_pi/defaults/main.yaml` | Image versions and path vars |
+| Create | `roles/raspberry_pi/handlers/main.yaml` | Restart zigbee2mqtt handler |
+| Create | `roles/raspberry_pi/templates/zigbee2mqtt/docker-compose.yml.j2` | Compose file |
+| Create | `roles/raspberry_pi/templates/zigbee2mqtt/mosquitto.conf.j2` | Mosquitto config |
+| Create | `roles/raspberry_pi/templates/zigbee2mqtt/z2m-configuration.yaml.j2` | Zigbee2MQTT config |
+| Create | `vars/group_vars/raspberry_pi/secrets.yaml` | Zigbee network key placeholder |
+
+---
+
+### Task 1: Add defaults, handlers, and secrets placeholder
+
+**Files:**
+- Create: `roles/raspberry_pi/defaults/main.yaml`
+- Create: `roles/raspberry_pi/handlers/main.yaml`
+- Create: `vars/group_vars/raspberry_pi/secrets.yaml`
+
+- [ ] **Create defaults file**
+
+```yaml
+---
+raspberry_pi_docker_base: /opt/docker
+raspberry_pi_mosquitto_config_dir: "{{ raspberry_pi_docker_base }}/config/mosquitto"
+raspberry_pi_z2m_config_dir: "{{ raspberry_pi_docker_base }}/config/zigbee2mqtt"
+raspberry_pi_compose_dir: "{{ raspberry_pi_docker_base }}/compose"
+raspberry_pi_mosquitto_version: "2"
+raspberry_pi_z2m_version: "2"
+```
+
+Save to `roles/raspberry_pi/defaults/main.yaml`.
+
+- [ ] **Create handlers file**
+
+```yaml
+---
+- name: Restart zigbee2mqtt
+  ansible.builtin.command:
+    cmd: docker compose restart zigbee2mqtt
+    chdir: "{{ raspberry_pi_compose_dir }}"
+  listen: restart zigbee2mqtt
+  when: inventory_hostname == 'naruto'
+```
+
+Save to `roles/raspberry_pi/handlers/main.yaml`.
+
+- [ ] **Create secrets placeholder**
+
+```yaml
+vault_raspberry_pi:
+  zigbee2mqtt:
+    network_key: "GENERATE"
+```
+
+Note: `GENERATE` tells Zigbee2MQTT to auto-generate a network key on first run and persist it to data. Replace with a fixed 16-integer array (e.g. `[1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13]`) if you need a stable key across reinstalls.
+
+Save to `vars/group_vars/raspberry_pi/secrets.yaml`.
+
+- [ ] **Commit**
+
+```bash
+git add roles/raspberry_pi/defaults/main.yaml roles/raspberry_pi/handlers/main.yaml vars/group_vars/raspberry_pi/secrets.yaml
+git commit -m "feat(raspberry_pi): add defaults, handlers, and secrets placeholder"
+```
+
+---
+
+### Task 2: Create directory task
+
+**Files:**
+- Create: `roles/raspberry_pi/tasks/10_directories.yaml`
+
+- [ ] **Create directory task file**
+
+```yaml
+---
+- name: Create docker base directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  become: true
+  loop:
+    - "{{ raspberry_pi_docker_base }}"
+    - "{{ raspberry_pi_compose_dir }}"
+  when: inventory_hostname == 'naruto'
+
+- name: Create Mosquitto directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  become: true
+  loop:
+    - "{{ raspberry_pi_mosquitto_config_dir }}"
+    - "{{ raspberry_pi_mosquitto_config_dir }}/data"
+    - "{{ raspberry_pi_mosquitto_config_dir }}/log"
+  when: inventory_hostname == 'naruto'
+
+- name: Create Zigbee2MQTT directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  become: true
+  loop:
+    - "{{ raspberry_pi_z2m_config_dir }}"
+    - "{{ raspberry_pi_z2m_config_dir }}/data"
+  when: inventory_hostname == 'naruto'
+```
+
+Save to `roles/raspberry_pi/tasks/10_directories.yaml`.
+
+- [ ] **Commit**
+
+```bash
+git add roles/raspberry_pi/tasks/10_directories.yaml
+git commit -m "feat(raspberry_pi): add directory setup task"
+```
+
+---
+
+### Task 3: Create templates
+
+**Files:**
+- Create: `roles/raspberry_pi/templates/zigbee2mqtt/mosquitto.conf.j2`
+- Create: `roles/raspberry_pi/templates/zigbee2mqtt/z2m-configuration.yaml.j2`
+- Create: `roles/raspberry_pi/templates/zigbee2mqtt/docker-compose.yml.j2`
+
+- [ ] **Create Mosquitto config template**
+
+```
+listener 1883
+persistence true
+persistence_location /mosquitto/data/
+log_dest file /mosquitto/log/mosquitto.log
+allow_anonymous true
+```
+
+Save to `roles/raspberry_pi/templates/zigbee2mqtt/mosquitto.conf.j2`.
+
+- [ ] **Create Zigbee2MQTT config template**
+
+```yaml
+homeassistant:
+  enabled: true
+
+mqtt:
+  server: mqtt://mosquitto:1883
+
+serial:
+  port: /dev/serial/by-id/usb-SONOFF_SONOFF_Dongle_Lite_MG21_0263f93f46a2ef11b078926661ce3355-if00-port0
+
+advanced:
+  network_key: {{ vault_raspberry_pi.zigbee2mqtt.network_key }}
+  log_level: info
+
+frontend:
+  enabled: true
+  port: 8080
+```
+
+Save to `roles/raspberry_pi/templates/zigbee2mqtt/z2m-configuration.yaml.j2`.
+
+- [ ] **Create Docker Compose template**
+
+```yaml
+name: zigbee2mqtt
+services:
+  mosquitto:
+    image: eclipse-mosquitto:{{ raspberry_pi_mosquitto_version }}
+    container_name: mosquitto
+    restart: unless-stopped
+    ports:
+      - 1883:1883
+    volumes:
+      - {{ raspberry_pi_mosquitto_config_dir }}/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
+      - {{ raspberry_pi_mosquitto_config_dir }}/data:/mosquitto/data
+      - {{ raspberry_pi_mosquitto_config_dir }}/log:/mosquitto/log
+
+  zigbee2mqtt:
+    image: koenkk/zigbee2mqtt:{{ raspberry_pi_z2m_version }}
+    container_name: zigbee2mqtt
+    restart: unless-stopped
+    depends_on:
+      - mosquitto
+    ports:
+      - 8080:8080
+    volumes:
+      - {{ raspberry_pi_z2m_config_dir }}/data:/app/data
+      - {{ raspberry_pi_z2m_config_dir }}/configuration.yaml:/app/data/configuration.yaml
+      - /run/udev:/run/udev:ro
+    devices:
+      - /dev/ttyUSB0:/dev/ttyUSB0
+    environment:
+      - TZ=Europe/Berlin
+    group_add:
+      - dialout
+
+networks:
+  default:
+    driver: bridge
+    name: zigbee2mqtt
+```
+
+Save to `roles/raspberry_pi/templates/zigbee2mqtt/docker-compose.yml.j2`.
+
+- [ ] **Commit**
+
+```bash
+git add roles/raspberry_pi/templates/
+git commit -m "feat(raspberry_pi): add zigbee2mqtt and mosquitto templates"
+```
+
+---
+
+### Task 4: Create Zigbee2MQTT deploy task
+
+**Files:**
+- Create: `roles/raspberry_pi/tasks/20_zigbee2mqtt.yaml`
+
+- [ ] **Create deploy task file**
+
+```yaml
+---
+- name: Deploy Mosquitto config
+  ansible.builtin.template:
+    src: zigbee2mqtt/mosquitto.conf.j2
+    dest: "{{ raspberry_pi_mosquitto_config_dir }}/mosquitto.conf"
+    mode: "0644"
+  become: true
+  when: inventory_hostname == 'naruto'
+
+- name: Deploy Zigbee2MQTT config
+  ansible.builtin.template:
+    src: zigbee2mqtt/z2m-configuration.yaml.j2
+    dest: "{{ raspberry_pi_z2m_config_dir }}/configuration.yaml"
+    mode: "0644"
+  become: true
+  notify: restart zigbee2mqtt
+  when: inventory_hostname == 'naruto'
+
+- name: Deploy docker-compose
+  ansible.builtin.template:
+    src: zigbee2mqtt/docker-compose.yml.j2
+    dest: "{{ raspberry_pi_compose_dir }}/docker-compose.yml"
+    mode: "0644"
+  become: true
+  when: inventory_hostname == 'naruto'
+
+- name: Start Zigbee2MQTT stack
+  ansible.builtin.command:
+    cmd: docker compose up -d
+    chdir: "{{ raspberry_pi_compose_dir }}"
+  become: true
+  changed_when: false
+  when: inventory_hostname == 'naruto'
+```
+
+Save to `roles/raspberry_pi/tasks/20_zigbee2mqtt.yaml`.
+
+- [ ] **Commit**
+
+```bash
+git add roles/raspberry_pi/tasks/20_zigbee2mqtt.yaml
+git commit -m "feat(raspberry_pi): add zigbee2mqtt deploy task"
+```
+
+---
+
+### Task 5: Wire up role main.yaml
+
+**Files:**
+- Modify: `roles/raspberry_pi/tasks/main.yaml`
+
+- [ ] **Update main.yaml to include task files**
+
+Replace the current contents (`---`) with:
+
+```yaml
+---
+- name: Setup directories
+  ansible.builtin.include_tasks: 10_directories.yaml
+
+- name: Setup Zigbee2MQTT
+  ansible.builtin.include_tasks: 20_zigbee2mqtt.yaml
+```
+
+- [ ] **Commit**
+
+```bash
+git add roles/raspberry_pi/tasks/main.yaml
+git commit -m "feat(raspberry_pi): wire up role tasks"
+```
+
+---
+
+### Task 6: Run and verify
+
+- [ ] **Run the playbook**
+
+```bash
+ansible-playbook playbooks/raspberry-pi.yaml
+```
+
+Expected: all tasks `ok` or `changed` on naruto, no failures. On pi, directory and zigbee2mqtt tasks should be skipped.
+
+- [ ] **Verify containers are running on naruto**
+
+```bash
+ansible naruto -a "docker ps --format 'table {{.Names}}\t{{.Status}}'" -b
+```
+
+Expected:
+```
+NAMES          STATUS
+zigbee2mqtt    Up X seconds
+mosquitto      Up X seconds
+```
+
+- [ ] **Check Zigbee2MQTT logs for successful startup**
+
+```bash
+ssh naruto "sudo docker logs zigbee2mqtt 2>&1 | tail -20"
+```
+
+Expected: lines like `Zigbee2MQTT started!`, no errors about serial port or MQTT connection.
+
+- [ ] **Verify Mosquitto is reachable from the LAN**
+
+```bash
+ssh naruto "docker exec mosquitto mosquitto_pub -h localhost -t test -m hello && echo 'OK'"
+```
+
+Expected: `OK`
+
+- [ ] **Verify pi tasks were skipped**
+
+Check playbook output shows `skipping: [pi]` for all directory and zigbee2mqtt tasks.

docs/superpowers/specs/2026-04-21-k3s-server11-reprovision-design.md

@@ -0,0 +1,146 @@
+# Design: Reprovision k3s-server11
+
+**Date**: 2026-04-21
+**Status**: Approved
+
+## Background
+
+k3s-server11 (Proxmox VM 111 on inko01) has a corrupted btrfs VM disk image
+(`/opt/proxmox/images/111/vm-111-disk-0.raw`). The corruption has been present since
+~2026-02-15 (when backups started failing with I/O errors). The VM's guest OS sees this
+as bad sectors on `/dev/sda`, causing etcd to crash with SIGBUS when it mmap-reads those
+sectors. This triggered a full cluster outage on 2026-04-20.
+
+The physical SSD on inko01 is healthy (SMART PASSED). The corruption is at the btrfs
+filesystem layer (3279+ corrupt blocks, single-device — no redundancy to recover from).
+
+Since etcd data is fully replicated on server10 and server12, no data recovery is needed.
+The correct fix is to replace the disk with a fresh OS image and rejoin the node.
+
+## Architecture
+
+Three sequential phases. Each phase must complete successfully before the next begins.
+
+```
+Phase 1: k8s cleanup     →  Phase 2: Proxmox disk     →  Phase 3: Ansible reprovision
+(drain, etcd remove,           (stop VM, delete disk,        (common + k3s_server roles,
+ delete node)                   import fresh image,            joins as secondary server,
+                                resize, start)                 etcd re-adds member)
+```
+
+## Phase 1: Remove server11 from the cluster
+
+Run from a machine with `kubectl` access (e.g. local workstation).
+
+**1.1 Drain the node** — evicts all non-daemonset pods:
+```bash
+kubectl drain k3s-server11 --ignore-daemonsets --delete-emptydir-data
+```
+
+**1.2 Remove from etcd** — prevents quorum issues while the disk is replaced:
+```bash
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member remove e9f8fa983ff7f958'
+```
+
+**1.3 Delete the node object**:
+```bash
+kubectl delete node k3s-server11
+```
+
+**Verify**: `kubectl get nodes` shows only server10, server12, and the agents. Etcd member
+list shows only 2 members (server10 + server12). Cluster remains healthy with quorum.
+
+## Phase 2: Replace the VM disk on inko01
+
+Run directly on inko01 via SSH.
+
+**2.1 Stop the VM**:
+```bash
+qm stop 111
+```
+
+**2.2 Delete the corrupt disk** (detaches and removes the raw file):
+```bash
+qm set 111 --delete scsi0
+```
+
+**2.3 Import a fresh Debian 12 cloud-init image as a new disk**:
+```bash
+qm importdisk 111 /opt/proxmox/template/iso/debian-12-genericcloud-amd64.qcow2 proxmox
+```
+This creates `/opt/proxmox/images/111/vm-111-disk-0.raw` from the clean base image.
+
+**2.4 Attach the disk and set boot order**:
+```bash
+qm set 111 --scsi0 proxmox:111/vm-111-disk-0.raw --boot order=scsi0
+```
+
+**2.5 Resize to 64G** (matching original disk size):
+```bash
+qm resize 111 scsi0 64G
+```
+
+**2.6 Start the VM**:
+```bash
+qm start 111
+```
+
+Cloud-init runs on first boot and configures: hostname (`k3s-server11`), user (`tudattr`),
+SSH keys, and DHCP networking. Wait ~60s for SSH to become available before Phase 3.
+
+**Verify**: `ssh k3s-server11 hostname` returns `k3s-server11` and no disk I/O errors
+appear in `dmesg`.
+
+## Phase 3: Reprovision via Ansible
+
+Run from local workstation in the ansible-homelab repo.
+
+```bash
+ansible-playbook playbooks/k3s-servers.yaml --limit k3s-server11
+```
+
+This runs the `common` and `k3s_server` roles against server11 only:
+
+- `common`: installs base packages, configures SSH, hostname, etc.
+- `k3s_server`: detects `/usr/local/bin/k3s` does not exist → runs install script with
+  `--server https://192.168.20.47:6443` (loadbalancer) → joins as a secondary server.
+  k3s fetches the cluster token from server10 (the primary) and registers as a new etcd
+  member automatically.
+
+**Verify**:
+```bash
+kubectl get nodes                    # server11 shows Ready
+ssh k3s-server11 'sudo ETCDCTL_API=3 etcdctl \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
+  --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
+  --key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
+  member list -w table'              # 3 members, all started
+ssh k3s-server11 'dmesg | grep -i "i/o error"'  # no output
+```
+
+## Key Facts
+
+| Item | Value |
+|------|-------|
+| VM ID | 111 |
+| Proxmox host | inko01 |
+| VM disk path | `/opt/proxmox/images/111/vm-111-disk-0.raw` |
+| Base image | `/opt/proxmox/template/iso/debian-12-genericcloud-amd64.qcow2` |
+| Proxmox storage pool | `proxmox` |
+| server11 IP | 192.168.20.48 |
+| server11 etcd member ID | `e9f8fa983ff7f958` |
+| Loadbalancer IP | 192.168.20.47 |
+| k3s primary server | server10 (192.168.20.43) |
+
+## Risk
+
+- **During Phase 1–2**: cluster runs on 2 etcd members. Still has quorum but no
+  redundancy. Avoid other disruptive changes until server11 is back.
+- **etcd member ID**: `e9f8fa983ff7f958` was confirmed on 2026-04-21. Verify it matches
+  before running the remove command if time has passed.

docs/superpowers/specs/2026-05-29-raspberry-pi-ansible-management-design.md

@@ -0,0 +1,36 @@
+# Raspberry Pi Ansible Management
+
+**Date:** 2026-05-29
+
+## Goal
+
+Bring `naruto` (Pi 4, 8GB) and `pi` (Pi 3, 1GB) under Ansible management using a new `raspberry_pi` role that starts with the `common` role as its base.
+
+## Inventory
+
+New file `vars/raspberry_pi.ini` with a `[raspberry_pi]` group containing both hosts. Both connect as user `tudattr` (non-root, sudo available).
+
+## ARM Fixes in `common` Role
+
+Two tasks in `extra_packages.yaml` are amd64-only and must be fixed before running on ARM:
+
+- **bottom:** URL is hardcoded to `amd64.deb`. Fix to use the existing `arch` global variable so it resolves to `arm64` on aarch64 hosts.
+- **Neovim:** Fetched as an AppImage, which doesn't run on aarch64. Fix to install `neovim` via apt on ARM, skipping the AppImage path.
+
+These fixes apply to the `common` role itself so any future ARM host benefits.
+
+## New Role: `raspberry_pi`
+
+Structure mirrors other roles. `tasks/main.yaml` includes `common` tasks, then Pi-specific tasks (none yet — placeholder for future workloads like Newt on naruto, docker stack on pi).
+
+## New Playbook
+
+`playbooks/raspberry-pi.yaml` targets `raspberry_pi` group, applies `raspberry_pi` role with tag `raspberry_pi`.
+
+## Group Vars
+
+`vars/group_vars/raspberry_pi/vars.yaml` — empty for now, inherits all from `all`. Can hold Pi-specific overrides later.
+
+## Hostname
+
+`pi` is currently named `raspberrypi`. The `common` hostname task will rename it to `pi` to match the inventory name.

docs/superpowers/specs/2026-06-03-naruto-zigbee2mqtt-design.md

@@ -0,0 +1,82 @@
+# Zigbee2MQTT + Mosquitto on naruto — Design Spec
+
+**Date:** 2026-06-03
+
+## Goal
+
+Run Zigbee2MQTT and Mosquitto as Docker containers on naruto, managed by the `raspberry_pi` Ansible role. Home Assistant (running in k3s) connects to Mosquitto over the LAN.
+
+## Hardware
+
+- Host: naruto (Pi 4, 192.168.20.13)
+- Zigbee coordinator: SONOFF Dongle Lite MG21 on `/dev/ttyUSB0`
+- Stable by-id path: `/dev/serial/by-id/usb-SONOFF_SONOFF_Dongle_Lite_MG21_0263f93f46a2ef11b078926661ce3355-if00-port0`
+
+## Architecture
+
+Two containers via Docker Compose on naruto. Ansible templates all configs and manages the stack. Home Assistant adds the MQTT integration pointing at `192.168.20.13:1883`.
+
+```
+[SONOFF Dongle /dev/ttyUSB0]
+        |
+[zigbee2mqtt container]
+        | MQTT (internal docker network)
+[mosquitto container] :1883
+        |
+[Home Assistant in k3s] — via LAN 192.168.20.13:1883
+```
+
+## Directory Layout on naruto
+
+```
+/opt/docker/
+  config/
+    mosquitto/
+      mosquitto.conf
+      data/
+      log/
+    zigbee2mqtt/
+      configuration.yaml
+      data/
+  compose/
+    docker-compose.yml
+```
+
+## Mosquitto Config
+
+- Listens on port 1883
+- No authentication (internal LAN only)
+- Persistence enabled, logs to `/opt/docker/config/mosquitto/log/`
+
+## Zigbee2MQTT Config
+
+- Serial port: `/dev/serial/by-id/usb-SONOFF_SONOFF_Dongle_Lite_MG21_0263f93f46a2ef11b078926661ce3355-if00-port0`
+- MQTT broker: `mqtt://mosquitto:1883` (internal docker network)
+- Network key: stored in `vars/group_vars/raspberry_pi/secrets.yaml` as `vault_raspberry_pi.zigbee2mqtt.network_key`
+- Frontend enabled on port 8080 for local device management
+
+## Secrets
+
+`vars/group_vars/raspberry_pi/secrets.yaml` (vault-encrypted, placeholder for now):
+
+```yaml
+vault_raspberry_pi:
+  zigbee2mqtt:
+    network_key: "YOUR_ZIGBEE_NETWORK_KEY"
+```
+
+## Ansible Changes
+
+| Action | Path | Responsibility |
+|--------|------|----------------|
+| Modify | `roles/raspberry_pi/tasks/main.yaml` | Include numbered task files |
+| Create | `roles/raspberry_pi/tasks/10_directories.yaml` | Create `/opt/docker/` tree |
+| Create | `roles/raspberry_pi/tasks/20_zigbee2mqtt.yaml` | Template configs, start compose |
+| Create | `roles/raspberry_pi/templates/zigbee2mqtt/docker-compose.yml.j2` | Compose file |
+| Create | `roles/raspberry_pi/templates/zigbee2mqtt/mosquitto.conf.j2` | Mosquitto config |
+| Create | `roles/raspberry_pi/templates/zigbee2mqtt/z2m-configuration.yaml.j2` | Zigbee2MQTT config |
+| Create | `vars/group_vars/raspberry_pi/secrets.yaml` | Network key placeholder |
+
+## Host Constraint
+
+The `raspberry_pi` role applies to both naruto and pi. The Zigbee2MQTT tasks must be guarded with `when: inventory_hostname == 'naruto'` since the USB dongle is only on naruto.

playbooks/kube-vip.yaml

@@ -0,0 +1,18 @@
+---
+# Deploys kube-vip on all k3s server nodes and adds the VIP to their TLS SANs.
+#
+# Migration steps (run once):
+#   1. ansible-playbook playbooks/kube-vip.yaml
+#   2. Update DNS: k3s.seyshiro.de → 192.168.20.2
+#   3. Verify: kubectl get nodes (should work via VIP)
+#   4. Decommission k3s-loadbalancer VM when satisfied
+#
+# The playbook is idempotent — re-running it after migration is safe.
+- name: Deploy kube-vip on k3s server nodes
+  hosts: k3s_server
+  gather_facts: true
+  serial: 1
+  roles:
+    - role: kube_vip
+      tags:
+        - kube_vip

playbooks/raspberry-pi.yaml

@@ -0,0 +1,11 @@
+---
+- name: Set up Raspberry Pis
+  hosts: raspberry_pi
+  gather_facts: true
+  roles:
+    - role: common
+      tags:
+        - common
+    - role: raspberry_pi
+      tags:
+        - raspberry_pi

playbooks/vps.yaml

@@ -0,0 +1,8 @@
+---
+- name: Set up VPS
+  hosts: vps
+  gather_facts: true
+  roles:
+    - role: edge_vps
+      tags:
+        - edge_vps

roles/common/files/kitty/infocmp

@@ -0,0 +1,77 @@
+#	Reconstructed via infocmp from file: /usr/lib/kitty/terminfo/./x/xterm-kitty
+xterm-kitty|KovIdTTY,
+	am, bw, ccc, hs, km, mc5i, mir, msgr, npc, xenl, Su, Tc, XF, fullkbd,
+	colors#0x100, cols#80, it#8, lines#24, pairs#0x7fff,
+	acsc=++\,\,--..00``aaffgghhiijjkkllmmnnooppqqrrssttuuvvwwxxyyzz{{||}}~~,
+	bel=^G, blink=\E[5m, bold=\E[1m, cbt=\E[Z, civis=\E[?25l,
+	clear=\E[H\E[2J, cnorm=\E[?12h\E[?25h, cr=\r,
+	csr=\E[%i%p1%d;%p2%dr, cub=\E[%p1%dD, cub1=^H,
+	cud=\E[%p1%dB, cud1=\n, cuf=\E[%p1%dC, cuf1=\E[C,
+	cup=\E[%i%p1%d;%p2%dH, cuu=\E[%p1%dA, cuu1=\E[A,
+	cvvis=\E[?12;25h, dch=\E[%p1%dP, dch1=\E[P, dim=\E[2m,
+	dl=\E[%p1%dM, dl1=\E[M, dsl=\E]2;\E\\, ech=\E[%p1%dX,
+	ed=\E[J, el=\E[K, el1=\E[1K, flash=\E[?5h$<100/>\E[?5l,
+	fsl=^G, home=\E[H, hpa=\E[%i%p1%dG, ht=^I, hts=\EH,
+	ich=\E[%p1%d@, il=\E[%p1%dL, il1=\E[L, ind=\n,
+	indn=\E[%p1%dS,
+	initc=\E]4;%p1%d;rgb:%p2%{255}%*%{1000}%/%2.2X/%p3%{255}%*%{1000}%/%2.2X/%p4%{255}%*%{1000}%/%2.2X\E\\,
+	kBEG=\E[1;2E, kDC=\E[3;2~, kEND=\E[1;2F, kHOM=\E[1;2H,
+	kIC=\E[2;2~, kLFT=\E[1;2D, kNXT=\E[6;2~, kPRV=\E[5;2~,
+	kRIT=\E[1;2C, kbeg=\EOE, kbs=^?, kcbt=\E[Z, kcub1=\EOD,
+	kcud1=\EOB, kcuf1=\EOC, kcuu1=\EOA, kdch1=\E[3~, kend=\EOF,
+	kf1=\EOP, kf10=\E[21~, kf11=\E[23~, kf12=\E[24~,
+	kf13=\E[1;2P, kf14=\E[1;2Q, kf15=\E[13;2~, kf16=\E[1;2S,
+	kf17=\E[15;2~, kf18=\E[17;2~, kf19=\E[18;2~, kf2=\EOQ,
+	kf20=\E[19;2~, kf21=\E[20;2~, kf22=\E[21;2~,
+	kf23=\E[23;2~, kf24=\E[24;2~, kf25=\E[1;5P, kf26=\E[1;5Q,
+	kf27=\E[13;5~, kf28=\E[1;5S, kf29=\E[15;5~, kf3=\EOR,
+	kf30=\E[17;5~, kf31=\E[18;5~, kf32=\E[19;5~,
+	kf33=\E[20;5~, kf34=\E[21;5~, kf35=\E[23;5~,
+	kf36=\E[24;5~, kf37=\E[1;6P, kf38=\E[1;6Q, kf39=\E[13;6~,
+	kf4=\EOS, kf40=\E[1;6S, kf41=\E[15;6~, kf42=\E[17;6~,
+	kf43=\E[18;6~, kf44=\E[19;6~, kf45=\E[20;6~,
+	kf46=\E[21;6~, kf47=\E[23;6~, kf48=\E[24;6~,
+	kf49=\E[1;3P, kf5=\E[15~, kf50=\E[1;3Q, kf51=\E[13;3~,
+	kf52=\E[1;3S, kf53=\E[15;3~, kf54=\E[17;3~,
+	kf55=\E[18;3~, kf56=\E[19;3~, kf57=\E[20;3~,
+	kf58=\E[21;3~, kf59=\E[23;3~, kf6=\E[17~, kf60=\E[24;3~,
+	kf61=\E[1;4P, kf62=\E[1;4Q, kf63=\E[13;4~, kf7=\E[18~,
+	kf8=\E[19~, kf9=\E[20~, khome=\EOH, kich1=\E[2~,
+	kind=\E[1;2B, kmous=\E[M, knp=\E[6~, kpp=\E[5~,
+	kri=\E[1;2A, oc=\E]104\007, op=\E[39;49m, rc=\E8,
+	rep=%p1%c\E[%p2%{1}%-%db, rev=\E[7m, ri=\EM,
+	rin=\E[%p1%dT, ritm=\E[23m, rmacs=\E(B, rmam=\E[?7l,
+	rmcup=\E[?1049l, rmir=\E[4l, rmkx=\E[?1l, rmso=\E[27m,
+	rmul=\E[24m, rs1=\E]\E\\\Ec, sc=\E7,
+	setab=\E[%?%p1%{8}%<%t4%p1%d%e%p1%{16}%<%t10%p1%{8}%-%d%e48;5;%p1%d%;m,
+	setaf=\E[%?%p1%{8}%<%t3%p1%d%e%p1%{16}%<%t9%p1%{8}%-%d%e38;5;%p1%d%;m,
+	sgr=%?%p9%t\E(0%e\E(B%;\E[0%?%p6%t;1%;%?%p2%t;4%;%?%p1%p3%|%t;7%;%?%p4%t;5%;%?%p7%t;8%;%?%p5%t;2%;m,
+	sgr0=\E(B\E[m, sitm=\E[3m, smacs=\E(0, smam=\E[?7h,
+	smcup=\E[?1049h, smir=\E[4h, smkx=\E[?1h, smso=\E[7m,
+	smul=\E[4m, tbc=\E[3g, tsl=\E]2;, u6=\E[%i%d;%dR, u7=\E[6n,
+	u8=\E[?%[;0123456789]c, u9=\E[c, vpa=\E[%i%p1%dd,
+	BD=\E[?2004l, BE=\E[?2004h, Cr=\E]112\007,
+	Cs=\E]12;%p1%s\007, Ms=\E]52;%p1%s;%p2%s\E\\,
+	PE=\E[201~, PS=\E[200~, RV=\E[>c, Se=\E[2 q,
+	Setulc=\E[58:2:%p1%{65536}%/%d:%p1%{256}%/%{255}%&%d:%p1%{255}%&%d%;m,
+	Smulx=\E[4:%p1%dm, Ss=\E[%p1%d q, Sync=\EP=%p1%ds\E\\,
+	XR=\E[>0q, fd=\E[?1004l, fe=\E[?1004h, kBEG3=\E[1;3E,
+	kBEG4=\E[1;4E, kBEG5=\E[1;5E, kBEG6=\E[1;6E,
+	kBEG7=\E[1;7E, kDC3=\E[3;3~, kDC4=\E[3;4~, kDC5=\E[3;5~,
+	kDC6=\E[3;6~, kDC7=\E[3;7~, kDN=\E[1;2B, kDN3=\E[1;3B,
+	kDN4=\E[1;4B, kDN5=\E[1;5B, kDN6=\E[1;6B, kDN7=\E[1;7B,
+	kEND3=\E[1;3F, kEND4=\E[1;4F, kEND5=\E[1;5F,
+	kEND6=\E[1;6F, kEND7=\E[1;7F, kHOM3=\E[1;3H,
+	kHOM4=\E[1;4H, kHOM5=\E[1;5H, kHOM6=\E[1;6H,
+	kHOM7=\E[1;7H, kIC3=\E[2;3~, kIC4=\E[2;4~, kIC5=\E[2;5~,
+	kIC6=\E[2;6~, kIC7=\E[2;7~, kLFT3=\E[1;3D, kLFT4=\E[1;4D,
+	kLFT5=\E[1;5D, kLFT6=\E[1;6D, kLFT7=\E[1;7D,
+	kNXT3=\E[6;3~, kNXT4=\E[6;4~, kNXT5=\E[6;5~,
+	kNXT6=\E[6;6~, kNXT7=\E[6;7~, kPRV3=\E[5;3~,
+	kPRV4=\E[5;4~, kPRV5=\E[5;5~, kPRV6=\E[5;6~,
+	kPRV7=\E[5;7~, kRIT3=\E[1;3C, kRIT4=\E[1;4C,
+	kRIT5=\E[1;5C, kRIT6=\E[1;6C, kRIT7=\E[1;7C, kUP=\E[1;2A,
+	kUP3=\E[1;3A, kUP4=\E[1;4A, kUP5=\E[1;5A, kUP6=\E[1;6A,
+	kUP7=\E[1;7A, kxIN=\E[I, kxOUT=\E[O, rmxx=\E[29m,
+	setrgbb=\E[48:2:%p1%d:%p2%d:%p3%dm,
+	setrgbf=\E[38:2:%p1%d:%p2%d:%p3%dm, smxx=\E[9m,

roles/common/handlers/main.yaml

@@ -1,6 +1,17 @@
 ---
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true
+  become: true
+
 - name: Restart sshd
   service:
     name: sshd
     state: restarted
   become: true
+
+- name: Restart timesyncd
+  ansible.builtin.systemd:
+    name: systemd-timesyncd
+    state: restarted
+  become: true

roles/common/tasks/bash.yaml

@@ -2,9 +2,9 @@
 - name: Copy bash-configs
   ansible.builtin.template:
     src: "files/bash/{{ item }}"
-    dest: "{{ ansible_env.HOME }}/.{{ item }}"
-    owner: "{{ ansible_user_id }}"
-    group: "{{ ansible_user_id }}"
+    dest: "{{ ansible_facts['env']['HOME'] }}/.{{ item }}"
+    owner: "{{ ansible_facts['user_id'] }}"
+    group: "{{ ansible_facts['user_id'] }}"
     mode: "644"
   loop:
     - bashrc
@@ -13,12 +13,25 @@
 - name: Copy ghostty infocmp
   ansible.builtin.copy:
     src: files/ghostty/infocmp
-    dest: "{{ ansible_env.HOME }}/ghostty"
-    owner: "{{ ansible_user_id }}"
-    group: "{{ ansible_user_id }}"
+    dest: "{{ ansible_facts['env']['HOME'] }}/ghostty"
+    owner: "{{ ansible_facts['user_id'] }}"
+    group: "{{ ansible_facts['user_id'] }}"
     mode: "0644"
   register: ghostty_terminfo
 
 - name: Compile ghostty terminalinfo
-  ansible.builtin.command: "tic -x {{ ansible_env.HOME }}/ghostty"
+  ansible.builtin.command: "tic -x {{ ansible_facts['env']['HOME'] }}/ghostty"
   when: ghostty_terminfo.changed
+
+- name: Copy kitty infocmp
+  ansible.builtin.copy:
+    src: files/kitty/infocmp
+    dest: "{{ ansible_facts['env']['HOME'] }}/kitty"
+    owner: "{{ ansible_facts['user_id'] }}"
+    group: "{{ ansible_facts['user_id'] }}"
+    mode: "0644"
+  register: kitty_terminfo
+
+- name: Compile kitty terminalinfo
+  ansible.builtin.command: "tic -x {{ ansible_facts['env']['HOME'] }}/kitty"
+  when: kitty_terminfo.changed

roles/common/tasks/extra_packages.yaml

@@ -14,11 +14,17 @@
   become: true
 
 - name: Add Gierens repository to apt sources
-  ansible.builtin.apt_repository:
-    repo: "deb [signed-by=/etc/apt/keyrings/gierens.asc] http://deb.gierens.de stable main"
+  ansible.builtin.deb822_repository:
+    name: gierens
+    types: deb
+    uris: http://deb.gierens.de
+    suites: stable
+    components: main
+    signed_by: /etc/apt/keyrings/gierens.asc
     state: present
-    update_cache: true
+    install_python_debian: true
   become: true
+  notify: Update apt cache
 
 - name: Install eza package
   ansible.builtin.apt:
@@ -28,7 +34,7 @@
 
 - name: Install bottom package
   ansible.builtin.apt:
-    deb: https://github.com/ClementTsang/bottom/releases/download/0.9.6/bottom_0.9.6_amd64.deb
+    deb: https://github.com/ClementTsang/bottom/releases/download/0.9.6/bottom_0.9.6_{{ arch }}.deb
     state: present
   become: true
 
@@ -37,20 +43,21 @@
   register: neovim_installed
   changed_when: false
   ignore_errors: true
+  when: ansible_facts['architecture'] != 'aarch64'
 
 - name: Download Neovim AppImage
   ansible.builtin.get_url:
     url: https://github.com/neovim/neovim/releases/download/v0.10.0/nvim.appimage
     dest: /tmp/nvim.appimage
     mode: "0755"
-  when: neovim_installed.rc != 0
+  when: ansible_facts['architecture'] != 'aarch64' and neovim_installed.rc != 0
   register: download_result
 
 - name: Extract Neovim AppImage
   ansible.builtin.command:
     cmd: "./nvim.appimage --appimage-extract"
     chdir: /tmp
-  when: download_result.changed
+  when: ansible_facts['architecture'] != 'aarch64' and download_result.changed
   register: extract_result
 
 - name: Copy extracted Neovim files to /usr
@@ -60,19 +67,19 @@
     remote_src: true
     mode: "0755"
   become: true
-  when: extract_result.changed
+  when: ansible_facts['architecture'] != 'aarch64' and extract_result.changed
 
 - name: Clean up extracted Neovim files
   ansible.builtin.file:
     path: /tmp/squashfs-root
     state: absent
-  when: extract_result.changed
+  when: ansible_facts['architecture'] != 'aarch64' and extract_result.changed
 
 - name: Remove Neovim AppImage
   ansible.builtin.file:
     path: /tmp/nvim.appimage
     state: absent
-  when: download_result.changed
+  when: ansible_facts['architecture'] != 'aarch64' and download_result.changed
 
 - name: Check if Neovim config directory already exists
   ansible.builtin.stat:

roles/common/tasks/packages.yaml

@@ -5,24 +5,24 @@
     upgrade: true
     autoremove: true
   become: true
-  when: ansible_user_id != "root"
+  when: ansible_facts['user_id'] != "root"
 
 - name: Install base packages
   ansible.builtin.apt:
     name: "{{ common_packages }}"
     state: present
   become: true
-  when: ansible_user_id != "root"
+  when: ansible_facts['user_id'] != "root"
 
 - name: Update and upgrade packages
   ansible.builtin.apt:
     update_cache: true
     upgrade: true
     autoremove: true
-  when: ansible_user_id == "root"
+  when: ansible_facts['user_id'] == "root"
 
 - name: Install base packages
   ansible.builtin.apt:
     name: "{{ common_packages }}"
     state: present
-  when: ansible_user_id == "root"
+  when: ansible_facts['user_id'] == "root"

roles/common/tasks/sshd.yaml

@@ -8,7 +8,7 @@
   notify:
     - Restart sshd
   become: true
-  when: ansible_user_id != "root"
+  when: ansible_facts['user_id'] != "root"
 
 - name: Copy root sshd_config
   ansible.builtin.template:
@@ -18,7 +18,7 @@
     backup: true
   notify:
     - Restart sshd
-  when: ansible_user_id == "root"
+  when: ansible_facts['user_id'] == "root"
 
 - name: Copy pubkey
   ansible.builtin.copy:

roles/common/tasks/time.yaml

@@ -3,9 +3,32 @@
   community.general.timezone:
     name: "{{ timezone }}"
   become: true
-  when: ansible_user_id != "root"
+  when: ansible_facts['user_id'] != "root"
 
 - name: Set timezone
   community.general.timezone:
     name: "{{ timezone }}"
-  when: ansible_user_id == "root"
+  when: ansible_facts['user_id'] == "root"
+
+- name: Configure NTP servers for systemd-timesyncd
+  ansible.builtin.lineinfile:
+    path: /etc/systemd/timesyncd.conf
+    regexp: "^#?NTP="
+    line: "NTP=0.debian.pool.ntp.org 1.debian.pool.ntp.org 2.debian.pool.ntp.org 3.debian.pool.ntp.org"
+  become: true
+  notify: Restart timesyncd
+
+- name: Enable and start systemd-timesyncd
+  ansible.builtin.systemd:
+    name: systemd-timesyncd
+    enabled: true
+    state: started
+  become: true
+  when: ansible_facts['user_id'] != "root"
+
+- name: Enable and start systemd-timesyncd
+  ansible.builtin.systemd:
+    name: systemd-timesyncd
+    enabled: true
+    state: started
+  when: ansible_facts['user_id'] == "root"

roles/docker_host/tasks/40_directory_setup.yaml

@@ -33,7 +33,6 @@
     opts: defaults,nolock,_netdev,auto,bg
     state: mounted
   loop:
-    - /media/docker
     - /media/series
     - /media/movies
     - /media/songs

roles/edge_vps/defaults/main.yaml

@@ -6,6 +6,10 @@ edge_vps_wireguard_address: "10.133.7.1/24"
 edge_vps_wireguard_port: 61975
 edge_vps_traefik_config_dir: "{{ edge_vps_config_base }}/traefik"
 edge_vps_traefik_logs_dir: "{{ edge_vps_traefik_config_dir }}/logs"
-edge_vps_pangolin_config_dir: "{{ edge_vps_config_base }}/pangolin"
-edge_vps_elastic_config_dir: "{{ edge_vps_config_base }}/elastic-agent"
+edge_vps_pangolin_config_dir: "{{ edge_vps_config_base }}"
+edge_vps_pangolin_compose_dir: /root
+edge_vps_pangolin_version: "1.12.1"
+edge_vps_gerbil_version: "1.2.2"
+edge_vps_traefik_version: "v3.5"
+edge_vps_elastic_config_dir: /root/agent
 edge_vps_elastic_state_dir: /var/lib/elastic-agent/elastic-system/elastic-agent/state

roles/edge_vps/handlers/main.yaml

@@ -7,6 +7,12 @@
 
 - name: Restart traefik
   ansible.builtin.command:
-    cmd: docker compose restart
-    chdir: "{{ edge_vps_traefik_config_dir }}"
+    cmd: podman compose restart traefik
+    chdir: "{{ edge_vps_pangolin_compose_dir }}"
   listen: restart traefik
+
+- name: Restart pangolin
+  ansible.builtin.command:
+    cmd: podman compose restart pangolin
+    chdir: "{{ edge_vps_pangolin_compose_dir }}"
+  listen: restart pangolin

roles/edge_vps/tasks/10_directories.yaml

@@ -14,9 +14,9 @@
     - "{{ edge_vps_traefik_config_dir }}"
     - "{{ edge_vps_traefik_logs_dir }}"
 
-- name: Create Pangolin config directory
+- name: Create Pangolin letsencrypt directory
   ansible.builtin.file:
-    path: "{{ edge_vps_pangolin_config_dir }}"
+    path: "{{ edge_vps_pangolin_config_dir }}/letsencrypt"
     state: directory
     mode: "0755"
 

roles/edge_vps/tasks/30_traefik.yaml

@@ -6,10 +6,9 @@
     mode: "0644"
   notify: restart traefik
 
-- name: Deploy Cloudflare credentials for ACME
-  ansible.builtin.copy:
-    content: |
-      CF_DNS_API_TOKEN={{ vault_edge_vps.traefik.cloudflare_api_token }}
-    dest: "{{ edge_vps_traefik_config_dir }}/cloudflare.env"
-    mode: "0600"
-  no_log: true
+- name: Deploy Traefik dynamic config
+  ansible.builtin.template:
+    src: traefik/dynamic_config.yml.j2
+    dest: "{{ edge_vps_traefik_config_dir }}/dynamic_config.yml"
+    mode: "0644"
+  notify: restart traefik

roles/edge_vps/tasks/40_pangolin.yaml

@@ -9,16 +9,11 @@
 - name: Deploy Pangolin docker-compose
   ansible.builtin.template:
     src: pangolin/docker-compose.yml.j2
-    dest: "{{ edge_vps_pangolin_config_dir }}/docker-compose.yml"
+    dest: "{{ edge_vps_pangolin_compose_dir }}/docker-compose.yml"
     mode: "0644"
 
-- name: Create letsencrypt directory for Pangolin
-  ansible.builtin.file:
-    path: "{{ edge_vps_pangolin_config_dir }}/letsencrypt"
-    state: directory
-    mode: "0755"
-
 - name: Start Pangolin
-  community.docker.docker_compose_v2:
-    project_src: "{{ edge_vps_pangolin_config_dir }}"
-    state: present
+  ansible.builtin.command:
+    cmd: podman compose up -d
+    chdir: "{{ edge_vps_pangolin_compose_dir }}"
+  changed_when: false

roles/edge_vps/tasks/50_elastic_agent.yaml

@@ -24,6 +24,7 @@
     mode: "0644"
 
 - name: Start Elastic Agent
-  community.docker.docker_compose_v2:
-    project_src: "{{ edge_vps_elastic_config_dir }}"
-    state: present
+  ansible.builtin.command:
+    cmd: podman compose up -d
+    chdir: "{{ edge_vps_elastic_config_dir }}"
+  changed_when: false

roles/edge_vps/templates/pangolin/docker-compose.yml.j2

@@ -1,25 +1,58 @@
+name: pangolin
 services:
   pangolin:
-    image: fosrl/pangolin:latest
+    image: docker.io/fosrl/pangolin:{{ edge_vps_pangolin_version }}
     container_name: pangolin
     restart: unless-stopped
-    ports:
-      - "3001:3001"
-      - "443:443"
-      - "80:80"
     volumes:
-      - ./config.yml:/app/config/config.yml:ro
-      - ./letsencrypt:/letsencrypt
-    depends_on:
-      - gerbil
+      - ./config:/app/config
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3001/api/v1/"]
+      interval: "10s"
+      timeout: "10s"
+      retries: 15
 
   gerbil:
-    image: fosrl/gerbil:latest
+    image: docker.io/fosrl/gerbil:{{ edge_vps_gerbil_version }}
     container_name: gerbil
     restart: unless-stopped
-    network_mode: host
+    depends_on:
+      pangolin:
+        condition: service_healthy
+    command:
+      - --reachableAt=http://gerbil:3004
+      - --generateAndSaveKeyTo=/var/config/key
+      - --remoteConfig=http://pangolin:3001/api/v1/
+    volumes:
+      - ./config/:/var/config
     cap_add:
       - NET_ADMIN
       - SYS_MODULE
+    ports:
+      - 51820:51820/udp
+      - 21820:21820/udp
+      - 443:443
+      - 80:80
+      - 6443:6443
+
+  traefik:
+    image: docker.io/traefik:{{ edge_vps_traefik_version }}
+    container_name: traefik
+    restart: unless-stopped
+    network_mode: service:gerbil
+    depends_on:
+      pangolin:
+        condition: service_healthy
+    command:
+      - --configFile=/etc/traefik/traefik_config.yml
+    environment:
+      CLOUDFLARE_DNS_API_TOKEN: {{ vault_edge_vps.traefik.cloudflare_api_token }}
     volumes:
-      - /lib/modules:/lib/modules
+      - ./config/traefik:/etc/traefik:ro
+      - ./config/letsencrypt:/letsencrypt
+      - ./config/traefik/logs:/var/log/traefik
+
+networks:
+  default:
+    driver: bridge
+    name: pangolin

roles/edge_vps/templates/traefik/dynamic_config.yml.j2

@@ -0,0 +1,67 @@
+http:
+  middlewares:
+    redirect-to-https:
+      redirectScheme:
+        scheme: https
+
+  routers:
+    main-app-router-redirect:
+      rule: "Host(`{{ edge_vps_pangolin_dashboard_url | regex_replace('^https?://', '') }}`)"
+      service: next-service
+      entryPoints:
+        - web
+      middlewares:
+        - redirect-to-https
+
+    next-router:
+      rule: "Host(`{{ edge_vps_pangolin_dashboard_url | regex_replace('^https?://', '') }}`) && !PathPrefix(`/api/v1`)"
+      service: next-service
+      entryPoints:
+        - websecure
+      tls:
+        certResolver: letsencrypt
+        domains:
+          - main: "{{ edge_vps_pangolin_base_domain }}"
+            sans:
+              - "*.{{ edge_vps_pangolin_base_domain }}"
+{% for domain in edge_vps_traefik_extra_tls_domains | default([]) %}
+          - main: "{{ domain }}"
+            sans:
+              - "*.{{ domain }}"
+{% endfor %}
+
+    api-router:
+      rule: "Host(`{{ edge_vps_pangolin_dashboard_url | regex_replace('^https?://', '') }}`) && PathPrefix(`/api/v1`)"
+      service: api-service
+      entryPoints:
+        - websecure
+      tls:
+        certResolver: letsencrypt
+
+    ws-router:
+      rule: "Host(`{{ edge_vps_pangolin_dashboard_url | regex_replace('^https?://', '') }}`)"
+      service: api-service
+      entryPoints:
+        - websecure
+      tls:
+        certResolver: letsencrypt
+
+  services:
+    next-service:
+      loadBalancer:
+        servers:
+          - url: "http://pangolin:3002"
+
+    api-service:
+      loadBalancer:
+        servers:
+          - url: "http://pangolin:3000"
+
+tcp:
+  serversTransports:
+    pp-transport-v1:
+      proxyProtocol:
+        version: 1
+    pp-transport-v2:
+      proxyProtocol:
+        version: 2

roles/edge_vps/templates/wireguard/wg0.conf.j2

@@ -7,12 +7,12 @@ PostUp   = sysctl -w net.ipv4.ip_forward=1
 PostUp   = iptables -A FORWARD -i {{ edge_vps_wireguard_interface }} -j ACCEPT
 PostUp   = iptables -A FORWARD -o {{ edge_vps_wireguard_interface }} -j ACCEPT
 {% for route in edge_vps_wireguard_routes | default([]) %}
-PostUp   = ip route add {{ route }} via {{ route.gateway }} dev {{ edge_vps_wireguard_interface }}
+PostUp   = ip route add {{ route.network }} via {{ route.gateway }} dev {{ edge_vps_wireguard_interface }}
 {% endfor %}
 PostDown = iptables -D FORWARD -i {{ edge_vps_wireguard_interface }} -j ACCEPT
 PostDown = iptables -D FORWARD -o {{ edge_vps_wireguard_interface }} -j ACCEPT
 {% for route in edge_vps_wireguard_routes | default([]) %}
-PostDown = ip route del {{ route }} via {{ route.gateway }} dev {{ edge_vps_wireguard_interface }}
+PostDown = ip route del {{ route.network }} via {{ route.gateway }} dev {{ edge_vps_wireguard_interface }}
 {% endfor %}
 
 {% for peer in vault_edge_vps.wireguard.peers %}

roles/k3s_agent/tasks/installation.yaml

@@ -24,6 +24,6 @@
   ansible.builtin.command: |
     /tmp/k3s_install.sh
   environment:
-    K3S_URL: "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}"
+    K3S_URL: "https://{{ k3s_vip }}:{{ k3s.loadbalancer.default_port }}"
     K3S_TOKEN: "{{ k3s_token }}"
   become: true

roles/k3s_server/tasks/create_kubeconfig.yaml

@@ -46,7 +46,7 @@
 - name: Add K3s cluster to kubeconfig
   ansible.builtin.command: >
     kubectl config set-cluster "{{ k3s_cluster_name }}"
-    --server="https://{{ k3s_server_name }}:6443"
+    --server="https://{{ k3s_vip }}:6443"
     --certificate-authority=/tmp/k3s-ca.crt
     --embed-certs=true
   environment:

roles/k3s_server/tasks/main.yaml

@@ -1,11 +1,12 @@
 ---
-- name: Install dependencies for apt to use repositories over HTTPS
+- name: Install dependencies
   ansible.builtin.apt:
     name: "{{ item }}"
     state: present
     update_cache: true
   loop:
     - qemu-guest-agent
+    - etcd-client
   become: true
 
 - name: See if k3s file exists
@@ -15,15 +16,29 @@
 
 - name: Install primary k3s server
   include_tasks: primary_installation.yaml
-  when: ansible_default_ipv4.address == k3s_primary_server_ip
+  when:
+    - inventory_hostname == groups['k3s_server'] | first
+    - not k3s_status.stat.exists
 
 - name: Get token from primary k3s server
   include_tasks: pull_token.yaml
 
 - name: Install seconary k3s servers
   include_tasks: secondary_installation.yaml
-  when: ansible_default_ipv4.address != k3s_primary_server_ip
+  when:
+    - inventory_hostname != groups['k3s_server'] | first
+    - not k3s_status.stat.exists
 
 - name: Set kubeconfig on localhost
   include_tasks: create_kubeconfig.yaml
-  when: ansible_default_ipv4.address == k3s_primary_server_ip
+  when: inventory_hostname == groups['k3s_server'] | first
+
+- name: Persist control-plane NoSchedule taint in k3s config
+  ansible.builtin.blockinfile:
+    path: /etc/rancher/k3s/config.yaml
+    create: true
+    marker: "# {mark} ANSIBLE MANAGED control-plane taint"
+    block: |
+      node-taint:
+        - "node-role.kubernetes.io/control-plane:NoSchedule"
+  become: true

roles/k3s_server/tasks/primary_installation.yaml

@@ -8,7 +8,7 @@
 - name: Install K3s server with and TLS SAN
   ansible.builtin.command: |
     /tmp/k3s_install.sh server \
-    --cluster-init
-    --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }} \
+    --cluster-init \
+    --tls-san {{ k3s_vip }} \
     --tls-san {{ k3s_server_name }}
   become: true

roles/k3s_server/tasks/pull_token.yaml

@@ -1,15 +1,15 @@
-- name: Get K3s token from the first server
-  when: ansible_default_ipv4.address == k3s_primary_server_ip
+- name: Get K3s token from the primary server
   ansible.builtin.slurp:
     src: /var/lib/rancher/k3s/server/node-token
-  register: k3s_token
+  register: k3s_token_raw
+  delegate_to: "{{ groups['k3s_server'] | first }}"
+  run_once: true
   become: true
 
-- name: Set fact on k3s_primary_server_ip
+- name: Set k3s_token fact
   ansible.builtin.set_fact:
-    k3s_token: "{{ k3s_token['content'] | b64decode | trim }}"
-  when:
-    - ansible_default_ipv4.address == k3s_primary_server_ip
+    k3s_token: "{{ k3s_token_raw['content'] | b64decode | trim }}"
+  run_once: true
 
 - name: Write K3s token to local file for encryption
   ansible.builtin.copy:

roles/k3s_server/tasks/secondary_installation.yaml

@@ -13,8 +13,8 @@
 - name: Install K3s on the secondary servers
   ansible.builtin.command: |
     /tmp/k3s_install.sh \
-    --server "https://{{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }}:{{ k3s.loadbalancer.default_port }}" \
-    --tls-san {{ hostvars['k3s-loadbalancer'].ansible_default_ipv4.address }} \
+    --server "https://{{ k3s_vip }}:{{ k3s.loadbalancer.default_port }}" \
+    --tls-san {{ k3s_vip }} \
     --tls-san {{ k3s_server_name }}
   environment:
     K3S_TOKEN: "{{ k3s_token_vault.k3s_token }}"

roles/kube_vip/tasks/main.yaml

@@ -0,0 +1,61 @@
+---
+- name: Remove stale static pod manifest if present
+  ansible.builtin.file:
+    path: "{{ kube_vip_static_pod_path }}"
+    state: absent
+  become: true
+
+- name: Ensure k3s server manifests directory exists
+  ansible.builtin.file:
+    path: "{{ kube_vip_manifests_dir }}"
+    state: directory
+    mode: "0755"
+  become: true
+
+- name: Deploy kube-vip RBAC manifest
+  ansible.builtin.template:
+    src: templates/kube-vip-rbac.yaml.j2
+    dest: "{{ kube_vip_manifests_dir }}/kube-vip-rbac.yaml"
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+- name: Deploy kube-vip DaemonSet manifest
+  ansible.builtin.template:
+    src: templates/kube-vip.yaml.j2
+    dest: "{{ kube_vip_manifests_dir }}/kube-vip.yaml"
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+- name: Ensure VIP is present in k3s TLS SANs config
+  ansible.builtin.blockinfile:
+    path: /etc/rancher/k3s/config.yaml
+    create: true
+    marker: "# {mark} ANSIBLE MANAGED kube-vip TLS SAN"
+    block: |
+      tls-san:
+        - "{{ k3s_vip }}"
+  become: true
+  register: tls_san_added
+
+- name: Stop k3s for certificate rotation
+  ansible.builtin.systemd:
+    name: k3s
+    state: stopped
+  become: true
+  when: tls_san_added.changed
+
+- name: Rotate k3s certificates to include VIP in SAN
+  ansible.builtin.command: k3s certificate rotate
+  become: true
+  when: tls_san_added.changed
+
+- name: Start k3s after certificate rotation
+  ansible.builtin.systemd:
+    name: k3s
+    state: started
+  become: true
+  when: tls_san_added.changed

roles/kube_vip/templates/kube-vip-rbac.yaml.j2

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["services", "endpoints"]
+    verbs: ["list", "get", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list", "get", "watch", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["list", "get", "watch", "update", "create"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["list", "get", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system

roles/kube_vip/templates/kube-vip.yaml.j2

@@ -0,0 +1,81 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/name: kube-vip-ds
+    app.kubernetes.io/version: {{ kube_vip_version }}
+  name: kube-vip-ds
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-vip-ds
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-vip-ds
+        app.kubernetes.io/version: {{ kube_vip_version }}
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+      containers:
+      - name: kube-vip
+        image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_version }}
+        imagePullPolicy: IfNotPresent
+        args:
+        - manager
+        env:
+        - name: vip_arp
+          value: "true"
+        - name: port
+          value: "6443"
+        - name: vip_nodename
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: vip_interface
+          value: "{{ kube_vip_interface }}"
+        - name: vip_cidr
+          value: "32"
+        - name: dns_mode
+          value: first
+        - name: cp_enable
+          value: "true"
+        - name: cp_namespace
+          value: kube-system
+        - name: svc_enable
+          value: "false"
+        - name: vip_leaderelection
+          value: "true"
+        - name: vip_leasename
+          value: plndr-cp-lock
+        - name: vip_leaseduration
+          value: "5"
+        - name: vip_renewdeadline
+          value: "3"
+        - name: vip_retryperiod
+          value: "1"
+        - name: address
+          value: "{{ k3s_vip }}"
+        - name: prometheus_server
+          value: :2112
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists

roles/kube_vip/vars/main.yaml

@@ -0,0 +1,5 @@
+---
+kube_vip_version: "v0.8.9"
+kube_vip_interface: "eth0"
+kube_vip_manifests_dir: "/var/lib/rancher/k3s/server/manifests"
+kube_vip_static_pod_path: "/var/lib/rancher/k3s/agent/pod-manifests/kube-vip.yaml"

roles/proxmox/tasks/55_create_vm.yaml

@@ -18,7 +18,7 @@
     tags: "{{ proxmox_tags }}"
     description: "Created via Ansible with cloud-init"
     boot: "order=scsi0"
-    cpu: "x86-64-v2-AES"
+    cpu: "{{ proxmox_node_cpu[vm.node] | default('x86-64-v2-AES') }}"
     ciuser: "{{ vm.ciuser }}"
     cipassword: "{{ vm_secrets[proxmox_secrets_prefix + '_' + vm.name.replace('-', '_')] }}"
     ipconfig:

roles/raspberry_pi/defaults/main.yaml

@@ -0,0 +1,7 @@
+---
+raspberry_pi_docker_base: /opt/docker
+raspberry_pi_mosquitto_config_dir: "{{ raspberry_pi_docker_base }}/config/mosquitto"
+raspberry_pi_z2m_config_dir: "{{ raspberry_pi_docker_base }}/config/zigbee2mqtt"
+raspberry_pi_compose_dir: "{{ raspberry_pi_docker_base }}/compose/zigbee2mqtt"
+raspberry_pi_mosquitto_version: "2"
+raspberry_pi_z2m_version: "2"

roles/raspberry_pi/handlers/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Restart zigbee2mqtt
+  ansible.builtin.command:
+    cmd: docker compose restart zigbee2mqtt
+    chdir: "{{ raspberry_pi_compose_dir }}"
+  listen: restart zigbee2mqtt

roles/raspberry_pi/tasks/10_directories.yaml

@@ -0,0 +1,34 @@
+---
+- name: Create docker base directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  become: true
+  loop:
+    - "{{ raspberry_pi_docker_base }}"
+    - "{{ raspberry_pi_compose_dir }}"
+  when: inventory_hostname == 'naruto'
+
+- name: Create Mosquitto directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  become: true
+  loop:
+    - "{{ raspberry_pi_mosquitto_config_dir }}"
+    - "{{ raspberry_pi_mosquitto_config_dir }}/data"
+    - "{{ raspberry_pi_mosquitto_config_dir }}/log"
+  when: inventory_hostname == 'naruto'
+
+- name: Create Zigbee2MQTT directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  become: true
+  loop:
+    - "{{ raspberry_pi_z2m_config_dir }}"
+    - "{{ raspberry_pi_z2m_config_dir }}/data"
+  when: inventory_hostname == 'naruto'

roles/raspberry_pi/tasks/20_zigbee2mqtt.yaml

@@ -0,0 +1,40 @@
+---
+- name: Install docker-compose-plugin
+  ansible.builtin.apt:
+    name: docker-compose-plugin
+    state: present
+  become: true
+  when: inventory_hostname == 'naruto'
+
+- name: Deploy Mosquitto config
+  ansible.builtin.template:
+    src: zigbee2mqtt/mosquitto.conf.j2
+    dest: "{{ raspberry_pi_mosquitto_config_dir }}/mosquitto.conf"
+    mode: "0644"
+  become: true
+  when: inventory_hostname == 'naruto'
+
+- name: Deploy Zigbee2MQTT config
+  ansible.builtin.template:
+    src: zigbee2mqtt/z2m-configuration.yaml.j2
+    dest: "{{ raspberry_pi_z2m_config_dir }}/configuration.yaml"
+    mode: "0644"
+  become: true
+  notify: restart zigbee2mqtt
+  when: inventory_hostname == 'naruto'
+
+- name: Deploy docker-compose
+  ansible.builtin.template:
+    src: zigbee2mqtt/docker-compose.yml.j2
+    dest: "{{ raspberry_pi_compose_dir }}/docker-compose.yml"
+    mode: "0644"
+  become: true
+  when: inventory_hostname == 'naruto'
+
+- name: Start Zigbee2MQTT stack
+  ansible.builtin.command:
+    cmd: docker compose up -d
+    chdir: "{{ raspberry_pi_compose_dir }}"
+  become: true
+  changed_when: false
+  when: inventory_hostname == 'naruto'

roles/raspberry_pi/tasks/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Setup directories
+  ansible.builtin.include_tasks: 10_directories.yaml
+
+- name: Setup Zigbee2MQTT
+  ansible.builtin.include_tasks: 20_zigbee2mqtt.yaml

roles/raspberry_pi/templates/zigbee2mqtt/docker-compose.yml.j2

@@ -0,0 +1,36 @@
+name: zigbee2mqtt
+services:
+  mosquitto:
+    image: eclipse-mosquitto:{{ raspberry_pi_mosquitto_version }}
+    container_name: mosquitto
+    restart: unless-stopped
+    ports:
+      - 1883:1883
+    volumes:
+      - {{ raspberry_pi_mosquitto_config_dir }}/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
+      - {{ raspberry_pi_mosquitto_config_dir }}/data:/mosquitto/data
+      - {{ raspberry_pi_mosquitto_config_dir }}/log:/mosquitto/log
+
+  zigbee2mqtt:
+    image: koenkk/zigbee2mqtt:{{ raspberry_pi_z2m_version }}
+    container_name: zigbee2mqtt
+    restart: unless-stopped
+    depends_on:
+      - mosquitto
+    ports:
+      - 8081:8080
+    volumes:
+      - {{ raspberry_pi_z2m_config_dir }}/data:/app/data
+      - {{ raspberry_pi_z2m_config_dir }}/configuration.yaml:/app/data/configuration.yaml
+      - /run/udev:/run/udev:ro
+    devices:
+      - /dev/ttyUSB0:/dev/ttyUSB0
+    environment:
+      - TZ=Europe/Berlin
+    group_add:
+      - dialout
+
+networks:
+  default:
+    driver: bridge
+    name: zigbee2mqtt

roles/raspberry_pi/templates/zigbee2mqtt/mosquitto.conf.j2

@@ -0,0 +1,5 @@
+listener 1883
+persistence true
+persistence_location /mosquitto/data/
+log_dest file /mosquitto/log/mosquitto.log
+allow_anonymous true

roles/raspberry_pi/templates/zigbee2mqtt/z2m-configuration.yaml.j2

@@ -0,0 +1,17 @@
+homeassistant:
+  enabled: true
+
+mqtt:
+  server: mqtt://mosquitto:1883
+
+serial:
+  port: /dev/ttyUSB0
+  adapter: ember
+
+advanced:
+  network_key: {{ vault_raspberry_pi.zigbee2mqtt.network_key }}
+  log_level: info
+
+frontend:
+  enabled: true
+  port: 8080

vars/group_vars/all/vars.yaml

@@ -19,7 +19,7 @@ nfs_server: 192.168.20.12
 # Packages
 #
 
-arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
+arch: "{{ 'arm64' if ansible_facts['architecture'] == 'aarch64' else 'amd64' }}"
 
 netcup_api_key: "{{ vault_netcup.api_key }}"
 netcup_api_password: "{{ vault_netcup.api_password }}"

vars/group_vars/k3s/secrets_token.yaml

@@ -1,11 +1,11 @@
 $ANSIBLE_VAULT;1.1;AES256
-37356330336365666531353535343930613161663361363461316663396338323932303531376662
-3331346562383135343732386663646463373064643632330a643435313435363138386630303138
-32616431636532666561306362396137366233623832326365616430313764353639393062336536
-3766616231626131390a396336346465613439613439383465653864663936353930303463373563
-31323938376230363239323435356438353563346638363734613364646263613139643064313866
-64333131333262383662333362613563656135356433373335646438336339326165626163653338
-64636438373131313339316535653433633637633530386630653966306333336566306438376233
-36383430396332373165386334363833613038633862653439306564366231643939663562316538
-39383134623565363365323165626365393239396438373862313766653562623938373033396265
-3161613463346332643632306561363963323630363630316263
+64356331353036663336626237373732393636366236326430343435313362333332656639356661
+3861323465653764303733366430306335303737323863370a393737656163623432363432366430
+32353030303630323438643839363730326365303062653335303130623264613939303037376239
+3062613036333661300a363633306333373239633233653064343066343162356636373862656136
+62333933353566643166643831313035643034376166316166623835326263376166626235306131
+36393461633962333637636163333532626663316363653131333561653635373037353864353763
+65666665653161383835663631656166346431613435396331356539353231623034623938393836
+33643761303234376162383465383130633335356366393839636665373365623462363239636364
+65343938653062623963666531653861646134633732313764356566633533666232373663633661
+6563396563643334666437353962383535306339663834623666

vars/group_vars/k3s/vars.yaml

@@ -2,6 +2,8 @@ k3s:
   loadbalancer:
     default_port: 6443
 
+k3s_vip: "192.168.20.2"
+
 k3s_primary_server_ip: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | unique | list | first }}"
 k3s_server_ips: "{{ groups['k3s_server'] | map('extract', hostvars, 'ansible_default_ipv4') | map(attribute='address') | unique | list }}"
 

vars/group_vars/proxmox/vars.yaml

@@ -2,3 +2,12 @@ proxmox_api_host: 192.168.20.12
 proxmox_api_user: root
 proxmox_api_token_id: terraform
 proxmox_api_token_secret: "{{ vault_pve.api.token_secret }}"
+
+# CPU type per Proxmox node — x86-64-v3 requires AVX2 (Ryzen 5700U, N100, i5-7200U)
+# aya01 (Celeron N5105) tops out at SSE4.2, must stay at v2
+proxmox_node_cpu:
+  aya01: "x86-64-v2-AES"
+  inko01: "x86-64-v3"
+  lulu: "x86-64-v3"
+  mii01: "x86-64-v3"
+  naruto01: "x86-64-v3"

vars/group_vars/proxmox/vms.yaml

@@ -1,6 +1,6 @@
 vms:
   - name: "docker-host11"
-    node: "inko01"
+    node: "aya01"
     vmid: 411
     cores: 2
     memory: 4096 # in MiB
@@ -68,7 +68,7 @@ vms:
     sshkeys: "{{ pubkey }}"
     disk_size: 32 # in Gb
   - name: "k3s-server11"
-    node: "inko01"
+    node: "aya01"
     vmid: 111
     cores: 2
     memory: 4096 # in MiB
@@ -189,7 +189,7 @@ vms:
     sshkeys: "{{ pubkey }}"
     disk_size: 128
   - name: "k3s-agent21"
-    node: "inko01"
+    node: "aya01"
     vmid: 221
     cores: 2
     memory: 4096

vars/group_vars/raspberry_pi/secrets.yaml

@@ -0,0 +1,3 @@
+vault_raspberry_pi:
+  zigbee2mqtt:
+    network_key: "GENERATE"

vars/group_vars/raspberry_pi/vars.yaml

@@ -0,0 +1 @@
+---

vars/group_vars/vps/vars.yaml

@@ -9,6 +9,8 @@ edge_vps_pangolin_base_endpoint: "pangolin.seyshiro.de"
 edge_vps_pangolin_base_domain: "seyshiro.de"
 
 edge_vps_acme_email: "me+acme@tudattr.dev"
+edge_vps_traefik_extra_tls_domains:
+  - "tudattr.dev"
 
 edge_vps_elastic_version: "9.2.2"
 edge_vps_elastic_dns_server: "10.43.0.10"

vars/k3s.ini

@@ -3,7 +3,6 @@
 [k3s:children]
 k3s_server
 k3s_agent
-k3s_storage
 k3s_loadbalancer
 
 [k3s_server]

vars/raspberry_pi.ini

@@ -0,0 +1,3 @@
+[raspberry_pi]
+naruto
+pi