From 5f69245b17c7d962a4077051d8335200dc191f5e Mon Sep 17 00:00:00 2001 From: Jean-Marie Mineau Date: Sat, 9 Oct 2021 18:24:01 +0200 Subject: [PATCH] register correctly the endpoints to moniroring (when possible) --- roles/grafana/tasks/main.yml | 6 ++ .../tasks/register-cert-to-monitoring.yml | 23 ++++++++ .../tasks/kassandra.yml | 15 +++-- roles/prometheus-alert-manager/tasks/main.yml | 5 ++ .../tasks/register-cert-to-monitoring.yml | 23 ++++++++ .../tasks/main.yml | 19 ++++--- .../tasks/register-cert-to-monitoring.yml | 23 ++++++++ roles/prometheus-node-exporter/tasks/main.yml | 57 ++++++++++--------- .../tasks/register-cert-to-monitoring.yml | 23 ++++++++ roles/prometheus/tasks/main.yml | 19 ++++--- .../tasks/register-cert-to-monitoring.yml | 23 ++++++++ 11 files changed, 191 insertions(+), 45 deletions(-) create mode 100644 roles/grafana/tasks/register-cert-to-monitoring.yml create mode 100644 roles/prometheus-alert-manager/tasks/register-cert-to-monitoring.yml create mode 100644 roles/prometheus-blackbox-exporter/tasks/register-cert-to-monitoring.yml create mode 100644 roles/prometheus-node-exporter/tasks/register-cert-to-monitoring.yml create mode 100644 roles/prometheus/tasks/register-cert-to-monitoring.yml diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index d268302..98c4d41 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -57,6 +57,12 @@ subject_alt_name: "IP:{{ lan_address }}" # Need an equivalent to notify here +## THIS CERT CANNOT BE MONITORED BECAUSE IT IS A CLIENT CERT :'( +#- name: Ensured the certificate is monitored +# import_tasks: register-cert-to-monitoring.yml +# vars: +# target: "{{ lan_address }}:|grafana-{{ lan_address }}|{{ ansible_facts['nodename'] }}" + - name: Add Prometheus data source template: src: prometheus_datasource.yaml diff --git a/roles/grafana/tasks/register-cert-to-monitoring.yml b/roles/grafana/tasks/register-cert-to-monitoring.yml new file mode 100644 index 0000000..82d550a --- /dev/null +++ b/roles/grafana/tasks/register-cert-to-monitoring.yml @@ -0,0 +1,23 @@ +--- +- name: Get the list of targets of the server + slurp: + src: /etc/prometheus/targets/blackbox-tls-internal-targets.json + register: server_tls_targets_file + delegate_to: "{{ appointed_prometheus_server }}" + +- name: Set target variable from file + set_fact: + server_tls_targets: "{{ server_tls_targets_file['content'] | b64decode | from_json }}" + +- name: Register the endpoint to the prometheus server + block: + - name: Add the target + set_fact: + new_server_tls_targets: "[{{ server_tls_targets[0] | combine({'targets': [target]}, list_merge='append_rp') }}]" + + - name: Put the new target list + copy: + content: "{{ new_server_tls_targets | to_nice_json }}" + dest: /etc/prometheus/targets/blackbox-tls-internal-targets.json + delegate_to: "{{ appointed_prometheus_server }}" + when: target not in server_tls_targets.0.targets diff --git a/roles/prometheus-alert-manager/tasks/kassandra.yml b/roles/prometheus-alert-manager/tasks/kassandra.yml index 5c3419f..6136298 100644 --- a/roles/prometheus-alert-manager/tasks/kassandra.yml +++ b/roles/prometheus-alert-manager/tasks/kassandra.yml @@ -37,6 +37,12 @@ notify: Restart kassandra no_log: true +- name: Copy the CA cert + copy: + content: "{{ ca_cert }}" + dest: /opt/kassandra/ca.crt + notify: Restart kassandra + - name: Generate certificate include_role: name: generate-cert @@ -49,11 +55,10 @@ subject_alt_name: "IP:{{ lan_address }}" # Need an equivalent to notify here -- name: Copy the CA cert - copy: - content: "{{ ca_cert }}" - dest: /opt/kassandra/ca.crt - notify: Restart kassandra +- name: Ensured the certificate is monitored + import_tasks: register-cert-to-monitoring.yml + vars: + target: "{{ lan_address }}:8000|kassandra-{{ lan_address }}|{{ ansible_facts['nodename'] }}" - name: Copy the daemon configuration template: diff --git a/roles/prometheus-alert-manager/tasks/main.yml b/roles/prometheus-alert-manager/tasks/main.yml index cb2e3a6..be60549 100644 --- a/roles/prometheus-alert-manager/tasks/main.yml +++ b/roles/prometheus-alert-manager/tasks/main.yml @@ -42,6 +42,11 @@ subject_alt_name: "IP:{{ lan_address }}" # Need an equivalent to notify here +- name: Ensured the certificate is monitored + import_tasks: register-cert-to-monitoring.yml + vars: + target: "{{ lan_address }}:9093|alertmanager-{{ lan_address }}|{{ ansible_facts['nodename'] }}" + - name: Setup the alertmanager config template: src: alertmanager.yml diff --git a/roles/prometheus-alert-manager/tasks/register-cert-to-monitoring.yml b/roles/prometheus-alert-manager/tasks/register-cert-to-monitoring.yml new file mode 100644 index 0000000..82d550a --- /dev/null +++ b/roles/prometheus-alert-manager/tasks/register-cert-to-monitoring.yml @@ -0,0 +1,23 @@ +--- +- name: Get the list of targets of the server + slurp: + src: /etc/prometheus/targets/blackbox-tls-internal-targets.json + register: server_tls_targets_file + delegate_to: "{{ appointed_prometheus_server }}" + +- name: Set target variable from file + set_fact: + server_tls_targets: "{{ server_tls_targets_file['content'] | b64decode | from_json }}" + +- name: Register the endpoint to the prometheus server + block: + - name: Add the target + set_fact: + new_server_tls_targets: "[{{ server_tls_targets[0] | combine({'targets': [target]}, list_merge='append_rp') }}]" + + - name: Put the new target list + copy: + content: "{{ new_server_tls_targets | to_nice_json }}" + dest: /etc/prometheus/targets/blackbox-tls-internal-targets.json + delegate_to: "{{ appointed_prometheus_server }}" + when: target not in server_tls_targets.0.targets diff --git a/roles/prometheus-blackbox-exporter/tasks/main.yml b/roles/prometheus-blackbox-exporter/tasks/main.yml index f8b31ab..83506e3 100644 --- a/roles/prometheus-blackbox-exporter/tasks/main.yml +++ b/roles/prometheus-blackbox-exporter/tasks/main.yml @@ -9,6 +9,14 @@ retries: 3 until: apt_result is succeeded +- name: Copy the CA cert + copy: + content: "{{ ca_cert }}" + dest: /etc/prometheus/ca.crt + notify: + - Restart blackbox-exporter + - Reload nginx + - name: Generate certificate include_role: name: generate-cert @@ -21,13 +29,10 @@ subject_alt_name: "IP:{{ lan_address }}" # Need an equivalent to notify here -- name: Copy the CA cert - copy: - content: "{{ ca_cert }}" - dest: /etc/prometheus/ca.crt - notify: - - Restart blackbox-exporter - - Reload nginx +- name: Ensured the certificate is monitored + import_tasks: register-cert-to-monitoring.yml + vars: + target: "{{ lan_address }}:9115|blackbox-{{ lan_address }}|{{ ansible_facts['nodename'] }}" - name: Setup the blackbox config template: diff --git a/roles/prometheus-blackbox-exporter/tasks/register-cert-to-monitoring.yml b/roles/prometheus-blackbox-exporter/tasks/register-cert-to-monitoring.yml new file mode 100644 index 0000000..82d550a --- /dev/null +++ b/roles/prometheus-blackbox-exporter/tasks/register-cert-to-monitoring.yml @@ -0,0 +1,23 @@ +--- +- name: Get the list of targets of the server + slurp: + src: /etc/prometheus/targets/blackbox-tls-internal-targets.json + register: server_tls_targets_file + delegate_to: "{{ appointed_prometheus_server }}" + +- name: Set target variable from file + set_fact: + server_tls_targets: "{{ server_tls_targets_file['content'] | b64decode | from_json }}" + +- name: Register the endpoint to the prometheus server + block: + - name: Add the target + set_fact: + new_server_tls_targets: "[{{ server_tls_targets[0] | combine({'targets': [target]}, list_merge='append_rp') }}]" + + - name: Put the new target list + copy: + content: "{{ new_server_tls_targets | to_nice_json }}" + dest: /etc/prometheus/targets/blackbox-tls-internal-targets.json + delegate_to: "{{ appointed_prometheus_server }}" + when: target not in server_tls_targets.0.targets diff --git a/roles/prometheus-node-exporter/tasks/main.yml b/roles/prometheus-node-exporter/tasks/main.yml index 2bcc23d..0e23e8f 100644 --- a/roles/prometheus-node-exporter/tasks/main.yml +++ b/roles/prometheus-node-exporter/tasks/main.yml @@ -52,6 +52,12 @@ mode: u=rw,g=r,o=r notify: Restart prometheus-node-exporter +- name: Copy the CA cert + copy: + content: "{{ ca_cert }}" + dest: /etc/node_exporter/ca.crt + notify: Restart prometheus-node-exporter + - name: Generate certificate include_role: name: generate-cert @@ -64,11 +70,10 @@ subject_alt_name: "IP:{{ lan_address }}" # Need an equivalent to notify here -- name: Copy the CA cert - copy: - content: "{{ ca_cert }}" - dest: /etc/node_exporter/ca.crt - notify: Restart prometheus-node-exporter +- name: Ensured the certificate is monitored + import_tasks: register-cert-to-monitoring.yml + vars: + target: "{{ lan_address }}:9100|node-exp-{{ lan_address }}|{{ ansible_facts['nodename'] }}" - name: Setup the arguments for node-exporter template: @@ -85,30 +90,30 @@ - name: web.config value: /etc/node_exporter/config.yaml -# Add the node to the server targets - -- name: Get the list of targets of the server - slurp: - src: /etc/prometheus/targets/node-targets.json - register: server_target_file - delegate_to: "{{ appointed_prometheus_server }}" - -- name: Set target variable - set_fact: - server_target: "{{ server_target_file['content'] | b64decode | from_json }}" - -- name: Register the node to the prometheus server +- name: Add the node to the server targets block: - - name: Add the node to the targets + - name: Get the list of targets of the server + slurp: + src: /etc/prometheus/targets/node-targets.json + register: server_node_target_file + delegate_to: "{{ appointed_prometheus_server }}" + + - name: Set target variable set_fact: - new_server_target: "[{{ server_target[0] | combine({'targets': [lan_address + '|' + ansible_facts['nodename']]}, list_merge='append_rp') }}]" + server_node_target: "{{ server_node_target_file['content'] | b64decode | from_json }}" - - name: Put the new target list - copy: - content: "{{ new_server_target | to_nice_json }}" - dest: /etc/prometheus/node-targets.json - delegate_to: "{{ appointed_prometheus_server }}" - when: (lan_address + '|' + ansible_facts['nodename']) not in server_target.0.targets + - name: Register the node to the prometheus server + block: + - name: Add the node to the targets + set_fact: + new_server_node_target: "[{{ server_node_target[0] | combine({'targets': [lan_address + '|' + ansible_facts['nodename']]}, list_merge='append_rp') }}]" + + - name: Put the new target list + copy: + content: "{{ new_server_node_target | to_nice_json }}" + dest: /etc/prometheus/node-targets.json + delegate_to: "{{ appointed_prometheus_server }}" + when: (lan_address + '|' + ansible_facts['nodename']) not in server_node_target.0.targets - name: Add alert rules for node on the prometheus server copy: diff --git a/roles/prometheus-node-exporter/tasks/register-cert-to-monitoring.yml b/roles/prometheus-node-exporter/tasks/register-cert-to-monitoring.yml new file mode 100644 index 0000000..82d550a --- /dev/null +++ b/roles/prometheus-node-exporter/tasks/register-cert-to-monitoring.yml @@ -0,0 +1,23 @@ +--- +- name: Get the list of targets of the server + slurp: + src: /etc/prometheus/targets/blackbox-tls-internal-targets.json + register: server_tls_targets_file + delegate_to: "{{ appointed_prometheus_server }}" + +- name: Set target variable from file + set_fact: + server_tls_targets: "{{ server_tls_targets_file['content'] | b64decode | from_json }}" + +- name: Register the endpoint to the prometheus server + block: + - name: Add the target + set_fact: + new_server_tls_targets: "[{{ server_tls_targets[0] | combine({'targets': [target]}, list_merge='append_rp') }}]" + + - name: Put the new target list + copy: + content: "{{ new_server_tls_targets | to_nice_json }}" + dest: /etc/prometheus/targets/blackbox-tls-internal-targets.json + delegate_to: "{{ appointed_prometheus_server }}" + when: target not in server_tls_targets.0.targets diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index db9a26a..78c4c94 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -26,6 +26,14 @@ owner: prometheus mode: u=rwx,g=rx,o=rx +- name: Copy the CA cert + copy: + content: "{{ ca_cert }}" + dest: /etc/prometheus/ca.crt + notify: + - Restart prometheus + - Reload nginx + - name: Generate certificate include_role: name: generate-cert @@ -38,13 +46,10 @@ subject_alt_name: "IP:{{ lan_address }}" # Need an equivalent to notify here -- name: Copy the CA cert - copy: - content: "{{ ca_cert }}" - dest: /etc/prometheus/ca.crt - notify: - - Restart prometheus - - Reload nginx +- name: Ensured the certificate is monitored + import_tasks: register-cert-to-monitoring.yml + vars: + target: "{{ lan_address }}:9090|prometheus-{{ lan_address }}|{{ ansible_facts['nodename'] }}" - name: Setup the prometheus config template: diff --git a/roles/prometheus/tasks/register-cert-to-monitoring.yml b/roles/prometheus/tasks/register-cert-to-monitoring.yml new file mode 100644 index 0000000..82d550a --- /dev/null +++ b/roles/prometheus/tasks/register-cert-to-monitoring.yml @@ -0,0 +1,23 @@ +--- +- name: Get the list of targets of the server + slurp: + src: /etc/prometheus/targets/blackbox-tls-internal-targets.json + register: server_tls_targets_file + delegate_to: "{{ appointed_prometheus_server }}" + +- name: Set target variable from file + set_fact: + server_tls_targets: "{{ server_tls_targets_file['content'] | b64decode | from_json }}" + +- name: Register the endpoint to the prometheus server + block: + - name: Add the target + set_fact: + new_server_tls_targets: "[{{ server_tls_targets[0] | combine({'targets': [target]}, list_merge='append_rp') }}]" + + - name: Put the new target list + copy: + content: "{{ new_server_tls_targets | to_nice_json }}" + dest: /etc/prometheus/targets/blackbox-tls-internal-targets.json + delegate_to: "{{ appointed_prometheus_server }}" + when: target not in server_tls_targets.0.targets