Refactor tasks for compose.yml and add tasks for pihole, homeassistant, and prometheus

Signed-off-by: TuDatTr <tuan-dat.tran@tudattr.dev>

README.md

@@ -14,6 +14,61 @@ but first of all we need to create the buckets and provide ansible with the need
   - `vault_mysql_user_password: <YOURPASSWORD>` (arbitrary password, used internally)
   - `vault_ddns_tudattrdev_password: <YOURPASSWORD>` (password needed for ddns, refer to [here](https://www.namecheap.com/support/knowledgebase/article.aspx/595/11/how-do-i-enable-dynamic-dns-for-a-domain/))
   - `vault_ddns_borgland_password: <YOURPASSWORD>` (password needed for ddns, refer to [here](https://www.namecheap.com/support/knowledgebase/article.aspx/595/11/how-do-i-enable-dynamic-dns-for-a-domain/))
+  
+## Docker
+To add new docker containers to the docker role you need to add the following and replace `service` with the name of your service:
+
+- Add relevent vars to `group_vars/all/vars.yaml`:
+```yaml
+service_port: "19999" # Exposed port
+service_config: "{{ docker_dir }}/service/" # config folder or your dir
+service_data: "{{ docker_data_dir }}/service/" # data folder or your dir (only works on aya01)
+```
+- Create necessary directories for service in the docker role `roles/docker/tasks/service.yaml`
+```yaml
+- name: Create service dirs
+  file:
+    path: "{{ item }}"
+    owner: 1000
+    group: 1000
+    mode: '777'
+    state: directory
+  loop:
+    - "{{ service_config }}"
+    - "{{ service_data }}"
+
+# optional:
+# - name: Place service config
+#   template:
+#     owner: 1000
+#     mode: '660'
+#     src: "templates/hostname/service/service.yml"
+#     dest: "{{ prm_config }}/service.yml"
+```
+
+- Includ new tasks to `roles/docker/tasks/hostname_compose.yaml`:
+```yaml
+- include_tasks: service.yaml
+  tags:
+    - service
+```
+
+- Add new service to compose `roles/docker/templates/hostname/compose.yaml`
+```yaml
+  service:
+    image: service/service
+    container_name: service
+    hostname: service
+    networks:
+      - net
+    ports:
+      - "{{service_port}}:19999"
+    restart: unless-stopped
+    volumes:
+      - "{{service_config}}:/etc/service"
+      - "{{service_lib}}:/var/lib/service"
+      - "{{service_cache}}:/var/cache/service"
+```
 
 ## Server
 - Install Debian (debian-11.5.0-amd64-netinst.iso) on remote system

group_vars/all/vars.yml

@@ -58,20 +58,17 @@ mysql_user: user
 # aya01
 # 
 
+aya01_host: "aya01"
+aya01_ip: "192.168.20.12"
+
 zoneminder_config: "{{ docker_dir }}/zm/"
 zoneminder_data: "{{ docker_data_dir }}/zm/data/"
 
 syncthing_data: "{{docker_data_dir}}/syncthing/"
 
-grafana_data: "{{docker_data_dir}}/grafana/"
-grafana_log: "{{docker_dir}}/grafana/logs/"
-grafana_config: "{{docker_dir}}/grafana/config/"
-
-prometheus_data: "{{docker_data_dir}}/prometheus/"
-prometheus_config: "{{docker_dir}}/prometheus/config"
-
 softserve_data: "{{docker_dir}}/softserve/data"
 
+
 #
 # pi
 #
@@ -84,10 +81,11 @@ ha_config: "{{ docker_dir }}/home-assistant/config/"
 pihole_pihole: "{{ docker_dir }}/pihole/etc-pihole/"
 pihole_dnsmasq: "{{ docker_dir }}/pihole/etc-dnsmasq.d/"
 
-
 #
 # backblaze
 #
+
+# Directories that will be backupped to backblaze
 backblaze_paths:
   aya01:
     - "{{ docker_compose_dir }}"
@@ -109,3 +107,35 @@ smb_config: "templates/smb.conf"
 smb_media_dir: "/media"
 smb_group: "smbshare"
 smb_user: "smbuser"
+
+
+#
+# prometheus/grafana
+#
+prm_user: "prometheus"
+exporter_dir: "{{ docker_dir }}/exporter/"
+
+prm_data: "{{docker_data_dir}}/prometheus/"
+prm_config: "{{docker_dir}}/prometheus/"
+prm_port: "9091"
+
+e_node_port: "9100"
+
+e_mikrotik_ip: "192.168.20.1"
+e_mikrotik_version: "1.0.11"
+e_mikrotik_config: "{{ exporter_dir }}/mikrotik/config/"
+e_mikrotik_port: "9436"
+
+grafana_data: "{{docker_data_dir}}/grafana/"
+grafana_log: "{{docker_dir}}/grafana/logs/"
+grafana_config: "{{docker_dir}}/grafana/config/"
+
+
+#
+# netdata
+#
+
+netdata_port: "19999"
+netdata_config: "{{ docker_dir }}/netdata/"
+netdata_lib: "{{ docker_data_dir }}/netdata/lib/"
+netdata_cache: "{{ docker_data_dir }}/netdata/cache"

roles/common/tasks/aya01_fstab.yml

@@ -15,7 +15,9 @@
     backup: true
   loop: "{{ fstab_entries }}"
   become: true
+  register: fstab
 
 - name: Mount all disks
   command: mount -a
   become: true
+  when: fstab.changed

roles/docker/tasks/aya01_compose.yml

@@ -1,99 +1,46 @@
 ---
-- name: Create zonminder user
-  user:
-    name: zm
-    uid: 911
-    shell: /bin/false
-  become: true
-    
-- name: Create Zoneminder config directory
-  file:
-    path: "{{ item }}"
-    owner: 911
-    group: 911
-    mode: '700'
-    state: directory
-  loop:
-    - "{{ zoneminder_config }}"
-  become: true
 
-- name: Create Zoneminder data directory
-  file:
-    path: "{{ item }}"
-    owner: 911
-    group: 911
-    mode: '755'
-    state: directory
-  loop:
-    - "{{ zoneminder_data }}"
-  become: true
+- include_tasks: zoneminder.yml
+  tags:
+    - zoneminder
 
-- name: Create syncthing directory
-  file:
-    path: "{{ item }}"
-    owner: "{{ puid }}"
-    group: "{{ pgid }}"
-    mode: '755'
-    state: directory
-  loop:
-    - "{{ syncthing_data }}"
-  become: true
+- include_tasks: pihole.yml
+  tags:
+    - pihole
 
-- name: Resolve inotify error for syncthing
-  template:
-    src: "templates/aya01/syncthing/syncthing.conf"
-    dest: "/etc/sysctl.d/syncthing.conf"
-    mode: "660"
-  become: true
+- include_tasks: syncthing.yml
+  tags:
+    - syncthing
 
-- name: Create grafana data directory
-  file:
-    path: "{{ item }}"
-    owner: "{{ puid }}"
-    group: "{{ pgid }}"
-    mode: '755'
-    state: directory
-  loop:
-    - "{{ grafana_data }}"
-    - "{{ grafana_log }}"
-    - "{{ grafana_config }}"
-  become: true
+- include_tasks: grafana.yml
+  tags:
+    - grafana
 
-- name: Copy grafana config
-  template:
-    owner: "{{ puid }}"
-    src: "templates/aya01/grafana/etc-grafana/grafana.ini"
-    dest: "{{ grafana_config }}/grafana.ini"
-    mode: '660'
-  become: true
+- include_tasks: softserve.yml
+  tags:
+    - softserve
 
-- name: Create soft-serve directory
-  file:
-    path: "{{ item }}"
-    owner: "{{ puid }}"
-    group: "{{ pgid }}"
-    mode: '755'
-    state: directory
-  loop:
-    - "{{ softserve_data }}"
-  become: true
+- include_tasks: prometheus.yml
+  tags:
+    - prometheus
 
-# Todo, check if docker compose is running
-# - name: Shut down docker
-#   shell:
-#     cmd: "docker compose down --remove-orphans"
-#     chdir: "{{ docker_compose_dir }}"
+- include_tasks: netdata.yaml
+  tags:
+    - netdata
 
 - name: Copy the compose file
   template: 
     src: templates/aya01/compose.yaml
     dest: "{{ docker_compose_dir }}/compose.yaml"
-  tags:
-    - reload_compose
+  register: compose
+
+- name: Shut down docker
+  shell:
+    cmd: "docker compose down --remove-orphans"
+    chdir: "{{ docker_compose_dir }}"
+  when: compose.changed
 
 - name: Run docker compose
   shell:
     cmd: "docker compose up -d"
     chdir: "{{ docker_compose_dir }}"
-  tags:
-    - reload_compose

roles/docker/tasks/ddns.yml

@@ -0,0 +1,16 @@
+---
+- name: Create ddns-config directory
+  file:
+    path: "{{ docker_dir }}/ddns-updater/data/"
+    owner: 1000
+    group: 1000
+    mode: '700'
+    state: directory
+    
+- name: Copy ddns-config
+  template:
+    owner: 1000
+    src: "templates/pi/ddns-updater/data/config.json"
+    dest: "{{ docker_dir }}/ddns-updater/data/config.json"
+    mode: '400'
+

roles/docker/tasks/grafana.yml

@@ -0,0 +1,22 @@
+---
+- name: Create grafana data directory
+  file:
+    path: "{{ item }}"
+    owner: "{{ puid }}"
+    group: "{{ pgid }}"
+    mode: '755'
+    state: directory
+  loop:
+    - "{{ grafana_data }}"
+    - "{{ grafana_log }}"
+    - "{{ grafana_config }}"
+  become: true
+
+- name: Copy grafana config
+  template:
+    owner: "{{ puid }}"
+    src: "templates/aya01/grafana/etc-grafana/grafana.ini"
+    dest: "{{ grafana_config }}/grafana.ini"
+    mode: '660'
+  become: true
+

roles/docker/tasks/homeassistant.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Create homeassistant-config directory
+  file:
+    path: "{{ ha_config }}"
+    mode: '755'
+    state: directory   
+  become: true

roles/docker/tasks/netdata.yaml

@@ -0,0 +1,13 @@
+---
+
+- name: Create netdata dirs
+  file:
+    path: "{{ item }}"
+    owner: 1000
+    group: 1000
+    mode: '755'
+    state: directory   
+  loop:
+    - "{{ netdata_config }}"
+    - "{{ netdata_cache }}"
+    - "{{ netdata_lib }}"

roles/docker/tasks/pi_compose.yml

@@ -1,49 +1,20 @@
 ---
-- name: Create ddns-config directory
-  file:
-    path: "{{ docker_dir }}/ddns-updater/data/"
-    owner: 1000
-    mode: '700'
-    state: directory
-    
-- name: Copy ddns-config
-  template:
-    owner: 1000
-    src: "templates/pi/ddns-updater/data/config.json"
-    dest: "{{ docker_dir }}/ddns-updater/data/config.json"
-    mode: '400'
 
-- name: Create traefik-config directory
-  file:
-    path: "{{ item }}"
-    owner: 1000
-    mode: '700'
-    state: directory   
-  loop:
-    - "{{ docker_dir }}/traefik/etc-traefik/"
-    - "{{ docker_dir }}/traefik/var-log/"
+- include_tasks: traefik.yml
+  tags:
+    - traefik
 
-- name: Create pihole-config directory
-  file:
-    path: "{{ item }}"
-    owner: 1000
-    mode: '777'
-    state: directory   
-  loop:
-    - "{{ docker_dir }}/pihole/etc-pihole/"
-    - "{{ docker_dir }}/pihole/etc-dnsmasq.d/"
-  become: yes
+- include_tasks: ddns.yml
+  tags:
+    - ddns
 
-- name: Copy traefik-config
-  template:
-    owner: 1000
-    src: "templates/pi/{{ item }}"
-    dest: "{{ docker_dir }}/{{ item }}"
-    mode: '400'
-  loop:
-    - "traefik/etc-traefik/traefik.yml"
-    - "traefik/var-log/access.log"
-    - "traefik/var-log/traefik.log"
+- include_tasks: homeassistant.yml
+  tags:
+    - homeassistant
+
+- include_tasks: pihole.yml
+  tags:
+    - pihole
 
 # Todo, check if docker compose is running
 # - name: Shut down docker

roles/docker/tasks/pihole.yml

@@ -0,0 +1,12 @@
+---
+- name: Create pihole-config directory
+  file:
+    path: "{{ item }}"
+    owner: "{{ puid }}"
+    group: "{{ pgid }}"
+    mode: '755'
+    state: directory   
+  loop:
+    - "{{ docker_dir }}/pihole/etc-pihole/"
+    - "{{ docker_dir }}/pihole/etc-dnsmasq.d/"
+  become: true

roles/docker/tasks/prometheus.yml

@@ -0,0 +1,42 @@
+---
+
+- name: Create prometheus dirs
+  file:
+    path: "{{ item }}"
+    owner: 1000
+    group: 1000
+    mode: '777'
+    state: directory   
+  loop:
+    - "{{ prm_config }}"
+    - "{{ prm_data}}"
+
+- name: Place prometheus config
+  template:
+    owner: 1000
+    mode: '777'
+    src: "templates/aya01/prometheus/prometheus.yml"
+    dest: "{{ prm_config }}/prometheus.yml"
+
+- name: Create prometheus exporter dir
+  file:
+    path: "{{ exporter_dir }}"
+    owner: 1000
+    group: 1000
+    mode: '755'
+    state: directory   
+
+- name: Create mikrotik exporters config dir
+  file:
+    path: "{{ e_mikrotik_config }}"
+    owner: 1000
+    group: 1000
+    mode: '755'
+    state: directory   
+
+- name: Place mikrotik exporter config
+  template:
+    owner: 1000
+    mode: '400'
+    src: "templates/aya01/prometheus/exporter/mikrotik/config/config.yml"
+    dest: "{{ e_mikrotik_config }}/config.yml"

roles/docker/tasks/softserve.yml

@@ -0,0 +1,12 @@
+---
+
+- name: Create soft-serve directory
+  file:
+    path: "{{ item }}"
+    owner: "{{ puid }}"
+    group: "{{ pgid }}"
+    mode: '755'
+    state: directory
+  loop:
+    - "{{ softserve_data }}"
+  become: true

roles/docker/tasks/syncthing.yml

@@ -0,0 +1,18 @@
+---
+- name: Create syncthing directory
+  file:
+    path: "{{ item }}"
+    owner: "{{ puid }}"
+    group: "{{ pgid }}"
+    mode: '755'
+    state: directory
+  loop:
+    - "{{ syncthing_data }}"
+  become: true
+
+- name: Resolve inotify error for syncthing
+  template:
+    src: "templates/aya01/syncthing/syncthing.conf"
+    dest: "/etc/sysctl.d/syncthing.conf"
+    mode: "660"
+  become: true

roles/docker/tasks/traefik.yml

@@ -0,0 +1,23 @@
+---
+
+- name: Create traefik-config directory
+  file:
+    path: "{{ item }}"
+    owner: 1000
+    mode: '700'
+    state: directory   
+  loop:
+    - "{{ docker_dir }}/traefik/etc-traefik/"
+    - "{{ docker_dir }}/traefik/var-log/"
+
+- name: Copy traefik-config
+  template:
+    owner: 1000
+    src: "templates/pi/{{ item }}"
+    dest: "{{ docker_dir }}/{{ item }}"
+    mode: '400'
+  loop:
+    - "traefik/etc-traefik/traefik.yml"
+    - "traefik/var-log/access.log"
+    - "traefik/var-log/traefik.log"
+

roles/docker/tasks/zoneminder.yml

@@ -0,0 +1,30 @@
+---
+- name: Create zoneminder user
+  user:
+    name: zm
+    uid: 911
+    shell: /bin/false
+  become: true
+    
+- name: Create Zoneminder config directory
+  file:
+    path: "{{ item }}"
+    owner: 911
+    group: 911
+    mode: '700'
+    state: directory
+  loop:
+    - "{{ zoneminder_config }}"
+  become: true
+
+- name: Create Zoneminder data directory
+  file:
+    path: "{{ item }}"
+    owner: 911
+    group: 911
+    mode: '755'
+    state: directory
+  loop:
+    - "{{ zoneminder_data }}"
+  become: true
+

roles/docker/templates/aya01/compose.yaml

@@ -1,6 +1,7 @@
 version: '3'
 services:
   db:
+    container_name: zoneminder_db
     image: mariadb
     restart: always
     networks:
@@ -17,6 +18,7 @@ services:
       - "MAX_LOG_NUMBER=20"
       - "TZ=Europe/Berlin"
   zoneminder:
+    container_name: zoneminder
     image: ghcr.io/zoneminder-containers/zoneminder-base:latest
     restart: always
     stop_grace_period: 45s
@@ -50,7 +52,7 @@ services:
     image: pihole/pihole:latest
     restart: unless-stopped
     networks:
-      net: {}
+      - net
     ports:
       - "53:53/tcp"
       - "53:53/udp"
@@ -58,7 +60,7 @@ services:
       - "8089:80/tcp"
     environment:
       - "WEBPASSWORD={{ vault_aya01_pihole_password }}"
-      - "ServerIP=192.168.20.12"
+      - "ServerIP={{aya01_ip}}"
       - "INTERFACE=eth0"
       - "DNS1=1.1.1.1"
       - "DNS1=1.0.0.1"
@@ -109,15 +111,66 @@ services:
       - "{{ grafana_log }}:/var/log/grafana/"
     ports:
       - 3000:3000
+
   soft-serve:
     image: charmcli/soft-serve:latest
     container_name: soft-serve
+    networks:
+      - net
     volumes:
       - "{{ softserve_data }}:/soft-serve"
     ports:
       - 23231:23231
     restart: unless-stopped
 
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    networks:
+      - net
+    volumes:
+      - "{{ prm_config }}:/etc/prometheus"
+    ports:
+      - "{{ prm_port }}:9090"
+
+  exporter_mikrotik:
+    container_name: exporter_mikrotik
+    user: "{{ puid }}:{{ pgid }}"
+    image: "nshttpd/mikrotik-exporter:{{ e_mikrotik_version }}"
+    networks:
+      - net
+    volumes:
+      - "{{ e_mikrotik_config }}:/config"
+    environment:
+      - "CONFIG_FILE=/config/config.yml"
+    ports:
+      - "{{ e_mikrotik_port }}:9436"
+    restart: unless-stopped
+
+  netdata:
+    image: netdata/netdata
+    container_name: netdata
+    hostname: "{{ aya01_host }}"
+    networks:
+      - net
+    ports:
+      - "{{netdata_port}}:19999"
+    restart: unless-stopped
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - "{{netdata_config}}:/etc/netdata"
+      - "{{netdata_lib}}:/var/lib/netdata"
+      - "{{netdata_cache}}:/var/cache/netdata"
+      - /etc/passwd:/host/etc/passwd:ro
+      - /etc/group:/host/etc/group:ro
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /etc/os-release:/host/etc/os-release:ro
+
+
 networks:
   zoneminder:
   net:

roles/docker/templates/aya01/prometheus/exporter/mikrotik/config/config.yml

@@ -0,0 +1,18 @@
+devices:
+  - name: mikrotik
+    address: "{{ e_mikrotik_ip }}"
+    user: "{{ prm_user }}"
+    password: "{{ vault_prm_user_password }}"
+
+features:
+  bgp: false
+  dhcp: true
+  dhcpv6: true
+  dhcpl: true
+  routes: true
+  pools: true
+  optics: true
+
+
+
+

roles/docker/templates/aya01/prometheus/prometheus.yml

@@ -8,7 +8,7 @@ global:
   # Attach these labels to any time series or alerts when communicating with
   # external systems (federation, remote storage, Alertmanager).
   external_labels:
-      monitor: 'Mikrotik'
+      monitor: 'tudattr'
 
 # Alertmanager configuration
 alerting:
@@ -37,18 +37,8 @@ scrape_configs:
 
     #static_configs:
       #- targets: ['localhost:9090']
-
-  - job_name: Mikrotik
+  - job_name: 'mikrotik'
     static_configs:
       - targets:
-        - {{ mikrotik_ip }} # mikrotik_ip
-    metrics_path: /snmp
-    params:
-      module: [mikrotik]
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: mk_snmp_exporter:9116  # The SNMP exporter's real hostname:port.
+          - "{{aya01_ip}}:{{ e_mikrotik_port }}"
+

roles/docker/templates/pi/compose.yaml

@@ -20,6 +20,7 @@ services:
       - "traefik.http.routers.traefik.rule=Host(`traefik.{{local_domain}}`)"
 #      - "traefik.http.routers.traefik.entrypoints=web"
 #      - "traefik.http.services.traefik.loadbalancer.server.port=80"
+
   ddns-updater:
     container_name: ddns-updater
     image: "ghcr.io/qdm12/ddns-updater"
@@ -30,11 +31,11 @@ services:
       - "{{ ddns_updater_data }}:/updater/data/"
     ports:
       - 8000:8000/tcp
+
   homeassistant:
     container_name: homeassistant
     image: "ghcr.io/home-assistant/home-assistant:stable"
     restart: unless-stopped
-    # network_mode: host
     volumes:
       - "/etc/localtime:/etc/localtime:ro"
       - "{{ ha_config }}:/config/"
@@ -49,6 +50,7 @@ services:
       - "traefik.http.routers.homeassistant.rule=Host(`hass.{{local_domain}}`)"
 #      - "traefik.http.routers.homeassistant.entrypoints=web"
 #      - "traefik.http.services.homeassistant.loadbalancer.server.port=8123"
+
   pihole:
     container_name: pihole
     image: pihole/pihole:latest

roles/samba/tasks/config.yaml

@@ -4,9 +4,11 @@
     src: "{{ smb_config }}"
     dest: /etc/samba/smb.conf
   become: true
+  register: smbconf
 
 - name: Restart nmbd.service
   systemd:
     name: nmbd
     state: restarted
   become: true
+  when: smbconf.changed

roles/samba/tasks/install.yaml

@@ -32,10 +32,12 @@
     groups: "{{ smb_group }}"
     append: true
   become: true
+  register: new_user
 
 - name: Add password to "{{ smb_user }}"
   shell:
     cmd: smbpasswd -a "{{ smb_user }}"
     stdin: "{{ vault_smb_user_password }}\n{{ vault_smb_user_password }}"
   become: true
+  when: new_user.changed