feat(reverse_proxy): add Netcup DNS ACME challenge support and refactor Caddy setup

Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>

group_vars/docker/docker.yml

@@ -341,6 +341,7 @@ services:
       - docker-host00
     container_name: changedetection
     image: dgtlmoon/changedetection.io:0.49
+    healthcheck: curl
     volumes:
       - name: "Data"
         internal: /datastore

group_vars/docker/secrets.yml

@@ -1,56 +1,65 @@
 $ANSIBLE_VAULT;1.1;AES256
-32623863646365383136636631383936353032333935623162386465643139663835303063666138
-3336626338376466386265663737383062653236383430310a633138323038626134636362616166
-37383831323239366338333038326665643932643237656265316361323466376636373662343761
-6234366130373535330a343432663638393566613963303530653937613139366330653933376137
-65356265306139326361336632323332663135373735626539376565313466323236323862623531
-65623932633936666338653164646661373937376133333937336434613264393637363065353462
-31376333336433643432626531373731656238336431376630653832363437646665353333313764
-63656565326636383537373736303933636264633939323262656363346639376439383632386530
-64373230623135316634323565623736386263613630383038643636323965326464333533333136
-30346132616237356662626462363266376261333434663634353330613137626538376433333235
-63346434386538663335333262386536663330653835343335323636363233333135626434356131
-61346465643231646338346435396662323834373634613834393231326531666637636566316434
-66663737643037336332313338663739653939333866383835663835386165373664623433623237
-35353734616431666561656231336463336234656362623265356361626161383136653064616664
-35623638653935643465646538653931643935313638366133343233616565623433376435323739
-31376236626131623765303761396666346330633734373137366366336265663361613337366236
-35356239373361666337663661333834623039323639373131363638393435303161636336316639
-35376231366162626536396130666631323337313034363066303737613764336232383235613764
-66356530333733363030396633626438326134356535653538363561643837303462653732376462
-64663034653135386364643434653162343338343437323062396565643466643264653165393064
-32333561303035626463363461303866316465323966636166376432616532353438656633346363
-62656464303165646463336636386630333561373537386330663531616466643164623865393233
-66356337633238316235636632626234313938386338363164613231336434396566666666616538
-32396235383930306362343466656535393036303931663063626465373831636134346237346530
-64396464323538333433636461303231306538373861393932636336313061383032323662633432
-39376265353734333339313266353964383830373665373234633236613830636432326636353933
-65656238393438633862366363366665643364313534623833656634393035336634663837656661
-33643338393330376464356232633638303732626336383936626662313430303338373438653865
-62613765626332396636636433623364386135316265643163326534646138663930306363353737
-33353537396135386637313132393365616638323330313966323461383666326664303231353734
-34336663333865346538386663316638306239343832616231323730393363353933393365653830
-31393933313963396236653234383564376264616332373230663961313638343933336261646435
-35386437336130376139646563383137666466356361386366323735346130613866313330306631
-62383566363832333633653564313936363564346166663931653831616634633135353565306464
-64613863343766613764623461633335643137363065643864313337653665346230363331626434
-30306235343661393336656434666637623930333038393865653865643836613235366562386232
-39653336633034646233353633323135336639653062356233643131346666376664356262343938
-32396335356532323231646330383734666435666164643731323634326134393732316131353836
-65633631326133663633376361373631653739613633313161313935323066643530356337613835
-64316431653437653163626234386164303465353731616530623863323937343565666339323639
-31343562373433303535626465333936373433323834363965323732336535333565616231316235
-39663431356633326466393862383133313030656431333839396333326461323130366533306139
-31316338323333356334623332663166323035373864313739363335356162633937613164373637
-62643538323066363734353136323537613263306138613761643865383062343934313666316530
-65666166303263643163633666323861633765626438343739613164386333316335323963326334
-31663433653534383866666639353036616565363230626136626330303061623936363531333139
-65376333616331316637633461623836663965633462383830633165376631356631396564323330
-66346561613133353438653365333361643166393535393466373330316136376263643163666139
-64656233326333656438613235303937653363323761636666373633623938656134366262323931
-35323133373163393964323962346433366434623636383133323535363632363465663862306439
-33633564643030306638343430313831376333613363643839303330343338393964623038343165
-39346233303864393537316531396333356363373565626530633237653337393434653034633263
-32386431613462363430623761333961393834353664626238653063336536653531626266613463
-30623438313430663165303064336532613637613566623864643730633232353538336131666566
-366331336161363266613532653336343131
+62353938306631616432613936343031386266643837393733336533306532643332383761336462
+3566663762343161373266353236323532666562383031310a663661316264313737633732313166
+35336535353964646238393563333339646634346532633130633364343864363565353461616663
+6336343138623762320a366132383634383231316130643535313465356238343534656237626362
+38373439663730353739386636313865336262363864323633343839636434353261313432386135
+33343438663564323465373435613765306538633339303362656163636237643661623637376135
+65346465303530663161356666333062326536313135363536313237616564363838326339646162
+62323066626431376231386432333766366434326239303734353036396433333662333733373830
+66336433643032636166306332323063393333363734326333363936303033396336626135363832
+30636136656235376163613033616563663663633161643937666537333066343135326138643663
+64646638393364376466306438383337383231303637313366333638393939373739646338353036
+62303162383362393830316163303236336236363531333665353163373530323063313164656562
+33383561613530346561336166653536393137346630333262633738383838383338643761666463
+61303239636631646634373266303930343437636464326132316534616261376137396233653265
+39383137666533613739363764643162663361333465386332383964343534646537343065343833
+66643938623734643537313866316335396135613239393262613562356332663861646261373630
+34373939663239646534396638636265303438386239636439663635313665613634373832313237
+62306366633139333937646534393765663130396466346161376235656461346638323063353662
+64386466373433376133343266396537656435333831356531346531653262396330346238623431
+61303466366161336664333239663066643232623532643933373661663266366639646139666636
+62393532643535656566643862353337333533633861396164643766316637393638363662653863
+32643566333961663065383636383436666137356237643634326464636463303530306466616635
+36366365636337366335333630306237356366306535613464636463373063653861623464323764
+62336139653361376239303632326431643231346137333835356362333962613039643332373166
+32316234376431376136666161383039633035356636626664376137323630323966646161313664
+38623463376366623430663363663662303166636165646138363631643261376137336636636663
+61656631393963353066333930303932653730613431366131616233363662316139663038336538
+36383532316162356235373566313832323131326466363734613438323233353330613561383435
+39623435366236306431636232323838386462346464653561653638346338613833613133373133
+38626364643738373938336237323836646532356539643933333730353333626138646239633234
+66316563306230636139323335323665646462343861393366666462623966376431393438376134
+37376339356430316235633337376462666439643430303062656538386630613763623433646133
+65663530626533663266623861326431633137363466346634656634623166623331306636616666
+31643761343632336531356566636165363737646639326533386333646434393736643934643064
+39393039346639353439653766326138613164343030306436383461663636346534346365333265
+66653535623962653762633934646131653334363232636634303130306632383263373161363462
+35323133616665366238353535346561323834353634613730613439643536376337353234313337
+61623264616433336532383533376631396438313739616462323064613665396638333438306336
+34633338366235336131303462346665663464376334353431343363336662356335356562366532
+64366461623864633238666339346138663931363331613463333762336230313530613235303766
+34313064383461623230383730623731323533326663613565646436303230653264323061616536
+38636162356164656432626433373864326264623063343662323563366133363336313739326137
+31326164646364613865396534626533616366613565303032636637366435326336396464313232
+66393538393862616466313833326666316231393130666238636130613339663664393434613732
+65383363323138343335393636626138303561613532306131666334346631336333336639626466
+38343337346566346334383934306433366239666662346463666166643338613264636563653434
+36306338313363636665333763323135386165313939336432636339613432323736326635303162
+36656234656563376633373333633430313430333834623964653530626539333265363563376239
+33633430396338663063383338333732356532313435613737393465323431393035356136306166
+62633035653731636361396235613162643332393233326434353831613731373333326464326234
+36366166633437356336616166306164343636623962623136653861333866393039653939333037
+31343261663534356530373233336165326134613961616331316531313435386464396438363838
+31353935666566326630373336376438326366623537356536653564303066343837653030373962
+30393363336232646662663166326166386636356466616165376435623031666664373664623330
+31613030616162303732353738386434666566386138373238363732303138316533356435656662
+38636136353134303166636438663036363834663639613464376662666364386635333138353035
+39363236653336386332313930306663366130303836333664363335386331636431623036336535
+32366339386539306364343065323263366563643663623731643866346232653838333561336331
+36363030383263666137393035356331323038316239356637303665653164363739313664396235
+32366231613532323865623861636263383731303164366333303636356633323161653635393830
+38616139656264393932353332303264393038396663663236353838343432373965663561333531
+36363432323362643634623030356539396562633238653732313739616464643436666130633364
+37383764623938626332316630636630343236663338323661333933333730333630353061653061
+62656233653439353438

group_vars/docker/vars.yml

@@ -1,4 +1,8 @@
 caddy:
   admin_email: me+acme@tudattr.dev
 
-domain: "seyshiro.de"
+domain: "{{ internal_domain }}"
+
+netcup_api_key: "{{ vault.netcup.api_key }}"
+netcup_api_password: "{{ vault.netcup.api_password }}"
+netcup_customer_id: "{{ vault.netcup.customer_id }}"

group_vars/proxmox/secrets_vm.yml

@@ -1,8 +1,8 @@
 $ANSIBLE_VAULT;1.1;AES256
-37613534383362613234623233396435336239353166353561613666643137356466663139626439
-6233666333623133343533323036646630613463323638350a613932343238316339383633316266
-65333436623532386139386331386330363664323864313536356365373165386363336439656161
-6363623734623633340a626165353035316135356630356461363533653066643735373762363035
-61623435643337613236313035333366373131363132656235623363343832663732656437363832
-61313235323862653833313531306638373137633063323939373537353165316139633235393137
-306131653436333463666637363363646530
+34303365623966316461623032386163326137623233353933313536343633623339356430303738
+6662353066663134356637633865396531616334636263360a313266393030623761626636333832
+39623262656664653430303162633739613761316536303865326465353333376230346632333737
+6635333534343330610a393730396538333932393836323963376333393239666132616364323166
+30373933363131353339376333633530313263663830393661353966306162613666396465366437
+37326630633463313864636239303030366633366635323266383664346466356166353433653661
+313938346264623634366464363330313863

playbooks/common-k3s.yml

@@ -1,10 +0,0 @@
----
-- name: Run the common role on k3s
-  hosts: k3s
-  gather_facts: yes
-  vars_files:
-    - secrets.yml
-  roles:
-    - role: common
-      tags:
-        - common

playbooks/docker-lb.yml

@@ -1,6 +1,6 @@
 ---
 - name: Set up reverse proxy for docker
-  hosts: docker_lb
+  hosts: docker
   gather_facts: true
   vars_files:
     - secrets.yml
@@ -8,6 +8,8 @@
     - role: common
       tags:
         - common
+      when: inventory_hostname in groups["docker_lb"]
     - role: reverse_proxy
       tags:
         - reverse_proxy
+      when: inventory_hostname in groups["docker_lb"]

playbooks/test.yml

@@ -1,9 +0,0 @@
----
-- hosts: db
-  gather_facts: yes
-  vars_files:
-    - secrets.yml
-  tasks:
-    - name: Print the database connection string
-      debug:
-        msg: "{{ k3s_db_connection_string }}"

production.ini

@@ -23,6 +23,7 @@ k3s-agent[00:02]
 k3s-server[00:02]
 k3s-longhorn[00:02]
 docker-host[00:01]
+test-vm-00
 
 [k3s_nodes]
 k3s-server[00:02]
@@ -56,4 +57,3 @@ docker-host[00:01]
 
 [docker_lb]
 docker-lb
-test-vm-00

requirements.txt

@@ -1,6 +1,7 @@
 certifi==2025.1.31
 charset-normalizer==3.4.1
 idna==3.10
+nc-dnsapi==0.1.3
 proxmoxer==2.2.0
 requests==2.32.3
 urllib3==2.3.0

roles/common/tasks/extra_packages.yml

@@ -11,7 +11,6 @@
     url: https://raw.githubusercontent.com/eza-community/eza/main/deb.asc
     dest: /etc/apt/keyrings/gierens.asc
     mode: "0644"
-  register: gpg_key_result
   become: true
 
 - name: Add Gierens repository to apt sources

roles/docker_host/tasks/provision.yml

@@ -3,7 +3,7 @@
   ansible.builtin.set_fact:
     is_keycloak_host: "{{ inventory_hostname in (services | selectattr('name', 'equalto', 'keycloak') | map(attribute='vm') | first) }}"
 
-- name: Run Keycloak tasks
+- name: Create Keycloak directories
   ansible.builtin.file:
     path: "{{ docker.directories.local }}/keycloak/"
     owner: "{{ ansible_user_id }}"
@@ -13,7 +13,7 @@
   when: is_keycloak_host | bool
   become: true
 
-- name: Run Keycloak tasks
+- name: Setup Keycloak realms
   ansible.builtin.template:
     src: "templates/keycloak/realm.json.j2"
     dest: "{{ docker.directories.local }}/keycloak/{{ keycloak.realm }}-realm.json"

roles/docker_host/templates/compose.yaml.j2

@@ -34,7 +34,15 @@ services:
 {% endif %}
 {% if chosen_http_port_value is defined %}
     healthcheck:
+{% set healthcheck = 'curl' %}
+{% if service.healthcheck is defined %}
+{% set healthcheck = service.healthcheck %}
+{% endif %}
+{% if healthcheck == 'curl' %}
+      test: ["CMD", "curl", "-f", "--silent", "--show-error", "--connect-timeout", "5", "http://localhost:{{ chosen_http_port_value }}/"]
+{% elif healthcheck == 'wget' %}
       test: ["CMD-SHELL", "wget --quiet --spider --timeout=5 http://localhost:{{ chosen_http_port_value }}/ || exit 1"]
+{% endif %}
       interval: 30s
       timeout: 10s
       retries: 5

roles/reverse_proxy/handlers/main.yml

@@ -1,4 +1,11 @@
 ---
-- name: Restart Caddy
-  ansible.builtin.command: "{{ caddy_binary }} reload --config {{ caddy_config_path  }}"
+- name: Restart caddy service
+  ansible.builtin.systemd:
+    name: caddy
+    state: restarted
+  become: true
+
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true
   become: true

roles/reverse_proxy/tasks/10_caddy_install.yml

@@ -0,0 +1,23 @@
+---
+- name: Download Caddy GPG Key
+  ansible.builtin.get_url:
+    url: https://dl.cloudsmith.io/public/caddy/stable/gpg.key
+    dest: /usr/share/keyrings/caddy-stable-archive-keyring.asc
+    mode: "0644"
+  become: true
+
+- name: Add Caddy repository source list
+  ansible.builtin.apt_repository:
+    repo: "{{ item }}"
+    state: present
+  become: true
+  notify: Update apt cache
+  loop:
+    - "deb [signed-by=/usr/share/keyrings/caddy-stable-archive-keyring.asc] https://dl.cloudsmith.io/public/caddy/stable/deb/debian any-version main"
+    - "deb-src [signed-by=/usr/share/keyrings/caddy-stable-archive-keyring.asc] https://dl.cloudsmith.io/public/caddy/stable/deb/debian any-version main"
+
+- name: Install Caddy
+  ansible.builtin.apt:
+    name: caddy
+    state: present
+  become: true

roles/reverse_proxy/tasks/20_xcaddy_install.yml

@@ -24,8 +24,8 @@
     state: present
   become: true
 
-- name: Install Caddy
-  ansible.builtin.command: xcaddy build --with github.com/caddy-dns/netcup
+- name: Build Custom Caddy with netcup
+  ansible.builtin.command: xcaddy build --with github.com/caddy-dns/netcup {{ reverse_proxy_caddy_version}}
   environment:
     PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
   register: xcaddy_build

roles/reverse_proxy/tasks/30_custom_caddy.yml

@@ -0,0 +1,41 @@
+---
+- name: Check current diversion status for {{ reverse_proxy_default_caddy_path }}
+  ansible.builtin.command:
+    cmd: dpkg-divert --list {{ reverse_proxy_default_caddy_path }}
+  register: divert_check_result
+  changed_when: false # This task only checks state
+  failed_when: false # Don't fail if diversion doesn't exist (rc=1)
+  become: true
+
+- name: Divert package manager's caddy binary path
+  ansible.builtin.command:
+    cmd: dpkg-divert --divert {{ reverse_proxy_diverted_caddy_path }} --rename {{ reverse_proxy_default_caddy_path }}
+  # Only run if the diversion isn't already set correctly
+  when: "reverse_proxy_diverted_caddy_path not in divert_check_result.stdout"
+  notify: Restart caddy service # Notify restart if diversion happens
+  become: true
+
+- name: Copy custom Caddy binary to destination path
+  ansible.builtin.copy:
+    src: "{{ reverse_proxy_custom_caddy_source_path }}"
+    dest: "{{ reverse_proxy_custom_caddy_dest_path }}"
+    owner: root
+    group: root
+    mode: "0755"
+    remote_src: true
+  notify: Restart caddy service # Notify restart if binary changes
+  become: true
+
+- name: Install original (diverted) caddy binary alternative
+  ansible.builtin.command:
+    # Use --force if the link /usr/bin/caddy might exist but not be managed by alternatives yet
+    cmd: update-alternatives --install {{ reverse_proxy_alternatives_link }} {{ reverse_proxy_alternatives_name }} {{ reverse_proxy_diverted_caddy_path }} 10
+  changed_when: false # update-alternatives is idempotent but often reports no change via rc
+  become: true
+
+- name: Install custom caddy binary alternative with higher priority
+  ansible.builtin.command:
+    cmd: update-alternatives --install {{ reverse_proxy_alternatives_link }} {{ reverse_proxy_alternatives_name }} {{ reverse_proxy_custom_caddy_dest_path }} 50
+  changed_when: false # update-alternatives is idempotent but often reports no change via rc
+  notify: Restart caddy service
+  become: true

roles/reverse_proxy/tasks/50_netcup_dns.yml

@@ -0,0 +1,14 @@
+---
+- name: Setup DNS on Netcup
+  community.general.netcup_dns:
+    api_key: "{{ reverse_proxy_netcup_api_key }}"
+    api_password: "{{ reverse_proxy_netcup_api_password }}"
+    customer_id: "{{ reverse_proxy_netcup_customer_id }}"
+    domain: "{{ domain }}"
+    name: "{{ service.name }}"
+    type: "A"
+    value: "{{ hostvars['docker-lb'].ansible_default_ipv4.address }}"
+  loop: "{{ services }}"
+  loop_control:
+    loop_var: service
+  delegate_to: localhost

roles/reverse_proxy/tasks/80_configure.yml

@@ -13,4 +13,10 @@
     mode: "0644"
     backup: true
   become: true
-  notify: Restart Caddy
+  notify: Restart caddy service
+
+- name: Format Caddy configuration file
+  ansible.builtin.command:
+    cmd: "caddy fmt --overwrite {{ caddy_config_path }}"
+  become: true
+  notify: Restart caddy service

roles/reverse_proxy/tasks/main.yml

@@ -1,9 +1,18 @@
 ---
-- name: Install Prerequisites
-  ansible.builtin.include_tasks: prereq.yml
+- name: Install Go for Caddy
+  ansible.builtin.include_tasks: 00_go_install.yml
+
 - name: Install Caddy
-  ansible.builtin.include_tasks: install.yml
+  ansible.builtin.include_tasks: 10_caddy_install.yml
+
+- name: Install xCaddy
+  ansible.builtin.include_tasks: 20_xcaddy_install.yml
+
+- name: Setup Custom Caddy
+  ansible.builtin.include_tasks: 30_custom_caddy.yml
+
+- name: Setup Netcup DNS
+  ansible.builtin.include_tasks: 50_netcup_dns.yml
+
 - name: Configure Caddy
-  ansible.builtin.include_tasks: configure.yml
-- name: Start Caddy
-  ansible.builtin.include_tasks: start.yml
+  ansible.builtin.include_tasks: 80_configure.yml

roles/reverse_proxy/tasks/start.yml

@@ -1,4 +0,0 @@
----
-- name: Ensure Caddy service is running
-  ansible.builtin.command: "{{ caddy_binary }} start --config {{ caddy_config_path }}"
-  become: true

roles/reverse_proxy/templates/Caddyfile.j2

@@ -9,13 +9,14 @@
 {% set http_port = service.ports | selectattr('name', 'equalto', 'http') | map(attribute='external') | list %}
 {% if http_port %}
 {{ service.name }}.{{ domain }} {
-    {% for vm in service.vm %}
-    reverse_proxy {{ hostvars[vm].ansible_host }}:{{ http_port[0] }}
-    {% endfor %}
-    log {
-        output file /var/log/caddy/{{ service.name }}.log
-        format json
-    }
+  {% for vm in service.vm -%}
+  reverse_proxy {{ hostvars[vm].ansible_default_ipv4.address }}:{{ http_port[0] }}
+  {% endfor %}{{''}}
+  log {
+      output file /var/log/caddy/{{ service.name }}.log
+      format json
+  }
+
 	tls {
 		dns netcup {
 			customer_number {{ vault_netcup.customer_number }}

roles/reverse_proxy/vars/main.yml

@@ -0,0 +1,12 @@
+reverse_proxy_caddy_version: v2.9.1
+
+reverse_proxy_custom_caddy_source_path: "{{ ansible_env.HOME }}/caddy"
+reverse_proxy_default_caddy_path: "/usr/bin/caddy"
+reverse_proxy_custom_caddy_dest_path: "/usr/bin/caddy.custom"
+reverse_proxy_diverted_caddy_path: "/usr/bin/caddy.default"
+reverse_proxy_alternatives_link: "/usr/bin/caddy"
+reverse_proxy_alternatives_name: "caddy"
+
+reverse_proxy_netcup_api_key: "{{ netcup_api_key }}"
+reverse_proxy_netcup_api_password: "{{ netcup_api_password }}"
+reverse_proxy_netcup_customer_id: "{{ netcup_customer_id }}"