From 8fb4eaf6101dae33cb4f594df6cbb36c7ba4f13b Mon Sep 17 00:00:00 2001 From: Tuan-Dat Tran Date: Fri, 20 Sep 2024 16:57:59 +0200 Subject: [PATCH] Added k3s agents Signed-off-by: Tuan-Dat Tran --- group_vars/k3s/vars.yml | 6 ++ host_vars/k3s-agent00.yml | 10 +++ host_vars/k3s-agent01.yml | 10 +++ host_vars/k3s-agent02.yml | 10 +++ k3s-agents.yml | 31 ++++++++++ production | 22 ++++++- roles/helm/tasks/main.yml | 3 + roles/k3s_agent/handlers/main.yml | 6 ++ roles/k3s_agent/tasks/installation.yml | 21 +++++++ roles/k3s_agent/tasks/main.yml | 2 + roles/k3s_server/handlers/main.yml | 2 +- roles/k3s_server/tasks/installation.yml | 72 +++++++++++----------- roles/loadbalancer/templates/nginx.conf.j2 | 4 +- 13 files changed, 157 insertions(+), 42 deletions(-) create mode 100644 host_vars/k3s-agent00.yml create mode 100644 host_vars/k3s-agent01.yml create mode 100644 host_vars/k3s-agent02.yml create mode 100644 k3s-agents.yml create mode 100644 roles/helm/tasks/main.yml create mode 100644 roles/k3s_agent/handlers/main.yml create mode 100644 roles/k3s_agent/tasks/installation.yml create mode 100644 roles/k3s_agent/tasks/main.yml diff --git a/group_vars/k3s/vars.yml b/group_vars/k3s/vars.yml index 6cf8e99..7b8527a 100644 --- a/group_vars/k3s/vars.yml +++ b/group_vars/k3s/vars.yml @@ -14,8 +14,14 @@ k3s: - 192.168.20.24 loadbalancer: ip: 192.168.20.22 + default_port: 6443 db: ip: 192.168.20.23 default_port: "5432" + agent: + ips: + - 192.168.20.25 + - 192.168.20.26 + - 192.168.20.27 k3s_db_connection_string: "postgres://{{db.user}}:{{db.password}}@{{k3s.db.ip}}:{{k3s.db.default_port}}/{{db.name}}" diff --git a/host_vars/k3s-agent00.yml b/host_vars/k3s-agent00.yml new file mode 100644 index 0000000..ecb0252 --- /dev/null +++ b/host_vars/k3s-agent00.yml @@ -0,0 +1,10 @@ +--- +ansible_user: "{{ user }}" +ansible_host: 192.168.20.25 +ansible_port: 22 +ansible_ssh_private_key_file: "{{ pk_path }}" +ansible_become_pass: "{{ vault.k3s.server01.sudo }}" + +host: + hostname: "k3s-agent00" + ip: "{{ ansible_host }}" diff --git a/host_vars/k3s-agent01.yml b/host_vars/k3s-agent01.yml new file mode 100644 index 0000000..818ce7f --- /dev/null +++ b/host_vars/k3s-agent01.yml @@ -0,0 +1,10 @@ +--- +ansible_user: "{{ user }}" +ansible_host: 192.168.20.26 +ansible_port: 22 +ansible_ssh_private_key_file: "{{ pk_path }}" +ansible_become_pass: "{{ vault.k3s.server01.sudo }}" + +host: + hostname: "k3s-agent01" + ip: "{{ ansible_host }}" diff --git a/host_vars/k3s-agent02.yml b/host_vars/k3s-agent02.yml new file mode 100644 index 0000000..04d5110 --- /dev/null +++ b/host_vars/k3s-agent02.yml @@ -0,0 +1,10 @@ +--- +ansible_user: "{{ user }}" +ansible_host: 192.168.20.27 +ansible_port: 22 +ansible_ssh_private_key_file: "{{ pk_path }}" +ansible_become_pass: "{{ vault.k3s.server01.sudo }}" + +host: + hostname: "k3s-agent02" + ip: "{{ ansible_host }}" diff --git a/k3s-agents.yml b/k3s-agents.yml new file mode 100644 index 0000000..de2d130 --- /dev/null +++ b/k3s-agents.yml @@ -0,0 +1,31 @@ +- name: Set up Agents + hosts: k3s_nodes + gather_facts: yes + vars_files: + - secrets.yml + pre_tasks: + - name: Get K3s token from the first server + when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"] + slurp: + src: /var/lib/rancher/k3s/server/node-token + register: k3s_token + become: true + + - name: Set fact on k3s.server.ips[0] + when: host.ip == k3s.server.ips[0] and inventory_hostname in groups["k3s_server"] + set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}" + + roles: + - role: common + when: inventory_hostname in groups["k3s_agent"] + tags: + - common + - role: k3s_agent + when: inventory_hostname in groups["k3s_agent"] + k3s_token: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}" + tags: + - k3s_agent + - role: node_exporter + when: inventory_hostname in groups["k3s_agent"] + tags: + - node_exporter diff --git a/production b/production index 3df332f..91b9cfa 100644 --- a/production +++ b/production @@ -2,21 +2,39 @@ mii [k3s] -k3s-server00 -k3s-server01 k3s-postgres k3s-loadbalancer +k3s-server00 +k3s-server01 +k3s-agent00 +k3s-agent01 +k3s-agent02 [k3s_server] k3s-server00 k3s-server01 +[k3s_agent] +k3s-agent00 +k3s-agent01 +k3s-agent02 + [vm] +k3s-agent00 +k3s-agent01 +k3s-agent02 k3s-server00 k3s-server01 k3s-postgres k3s-loadbalancer +[k3s_nodes] +k3s-server00 +k3s-server01 +k3s-agent00 +k3s-agent01 +k3s-agent02 + [db] k3s-postgres diff --git a/roles/helm/tasks/main.yml b/roles/helm/tasks/main.yml new file mode 100644 index 0000000..4a19930 --- /dev/null +++ b/roles/helm/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- include_tasks: installation.yml +- include_tasks: apps.yml diff --git a/roles/k3s_agent/handlers/main.yml b/roles/k3s_agent/handlers/main.yml new file mode 100644 index 0000000..1300aee --- /dev/null +++ b/roles/k3s_agent/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart k3s + service: + name: k3s + state: restarted + become: yes diff --git a/roles/k3s_agent/tasks/installation.yml b/roles/k3s_agent/tasks/installation.yml new file mode 100644 index 0000000..8726f67 --- /dev/null +++ b/roles/k3s_agent/tasks/installation.yml @@ -0,0 +1,21 @@ +--- +- name: See if k3s file exists + stat: + path: /usr/local/bin/k3s + register: k3s_status + +- name: Download K3s install script to /tmp/ + when: not k3s_status.stat.exists + ansible.builtin.get_url: + url: https://get.k3s.io + dest: /tmp/k3s_install.sh + mode: "0755" + +- name: Install K3s on the secondary servers + when: not k3s_status.stat.exists + command: | + /tmp/k3s_install.sh + environment: + K3S_URL: "https://{{ k3s.loadbalancer.ip }}:{{k3s.loadbalancer.default_port}}" + K3S_TOKEN: "{{ k3s_token }}" + become: true diff --git a/roles/k3s_agent/tasks/main.yml b/roles/k3s_agent/tasks/main.yml new file mode 100644 index 0000000..fc446ac --- /dev/null +++ b/roles/k3s_agent/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include_tasks: installation.yml diff --git a/roles/k3s_server/handlers/main.yml b/roles/k3s_server/handlers/main.yml index a92a08b..1300aee 100644 --- a/roles/k3s_server/handlers/main.yml +++ b/roles/k3s_server/handlers/main.yml @@ -1,5 +1,5 @@ --- -- name: Restart sshd +- name: Restart k3s service: name: k3s state: restarted diff --git a/roles/k3s_server/tasks/installation.yml b/roles/k3s_server/tasks/installation.yml index 5f47980..3f20ba4 100644 --- a/roles/k3s_server/tasks/installation.yml +++ b/roles/k3s_server/tasks/installation.yml @@ -1,32 +1,38 @@ --- -# - name: Download K3s install script to /tmp/ -# ansible.builtin.get_url: -# url: https://get.k3s.io -# dest: /tmp/k3s_install.sh -# mode: "0755" -# -# - name: Install K3s server with node taint and TLS SAN -# when: host.ip == k3s.server.ips[0] -# command: | -# /tmp/k3s_install.sh server \ -# --node-taint CriticalAddonsOnly=true:NoExecute \ -# --tls-san {{ k3s.loadbalancer.ip }} -# environment: -# K3S_DATASTORE_ENDPOINT: "{{ k3s_db_connection_string }}" -# become: true -# async: 300 -# poll: 0 -# register: k3s_primary_install -# -# - name: Wait for K3s to be installed -# when: host.ip == k3s.server.ips[0] -# async_status: -# jid: "{{ k3s_primary_install.ansible_job_id }}" -# register: k3s_primary_install_status -# until: k3s_primary_install_status.finished -# retries: 60 -# delay: 5 -# become: true +- name: See if k3s file exists + stat: + path: /usr/local/bin/k3s + register: k3s_status + +- name: Download K3s install script to /tmp/ + when: not k3s_status.stat.exists + ansible.builtin.get_url: + url: https://get.k3s.io + dest: /tmp/k3s_install.sh + mode: "0755" + +- name: Install K3s server with node taint and TLS SAN + when: (host.ip == k3s.server.ips[0] and (not k3s_status.stat.exists)) + command: | + /tmp/k3s_install.sh server \ + --node-taint CriticalAddonsOnly=true:NoExecute \ + --tls-san {{ k3s.loadbalancer.ip }} + environment: + K3S_DATASTORE_ENDPOINT: "{{ k3s_db_connection_string }}" + become: true + async: 300 + poll: 0 + register: k3s_primary_install + +- name: Wait for K3s to be installed + when: (host.ip == k3s.server.ips[0] and (not k3s_status.stat.exists)) + async_status: + jid: "{{ k3s_primary_install.ansible_job_id }}" + register: k3s_primary_install_status + until: k3s_primary_install_status.finished + retries: 60 + delay: 5 + become: true - name: Get K3s token from the first server when: host.ip == k3s.server.ips[0] @@ -39,16 +45,8 @@ when: host.ip == k3s.server.ips[0] set_fact: k3s_token="{{ k3s_token['content'] | b64decode | trim }}" -- name: showdata - when: host.ip != k3s.server.ips[0] - debug: - msg: "{{a}} {{k3s_datastore_endpoint}}" - vars: - k3s_datastore_endpoint: "{{ k3s_db_connection_string }}" - a: "{{ hostvars[(hostvars | dict2items | map(attribute='value') | map('dict2items') | map('selectattr', 'key', 'match', 'host') | map('selectattr', 'value.ip', 'match', k3s.server.ips[0] ) | select() | first | items2dict).host.hostname].k3s_token }}" - - name: Install K3s on the secondary servers - when: host.ip != k3s.server.ips[0] + when: (host.ip != k3s.server.ips[0] and (not k3s_status.stat.exists)) command: | /tmp/k3s_install.sh server \ --node-taint CriticalAddonsOnly=true:NoExecute \ diff --git a/roles/loadbalancer/templates/nginx.conf.j2 b/roles/loadbalancer/templates/nginx.conf.j2 index 4862934..0e384ac 100644 --- a/roles/loadbalancer/templates/nginx.conf.j2 +++ b/roles/loadbalancer/templates/nginx.conf.j2 @@ -5,12 +5,12 @@ events {} stream { upstream k3s_servers { {% for ip in k3s_server_ips %} - server {{ ip }}:6443; + server {{ ip }}:{{k3s.loadbalancer.default_port}}; {% endfor %} } server { - listen 6443; + listen {{k3s.loadbalancer.default_port}}; proxy_pass k3s_servers; } }