From 265bd5fbb7cc31dcb265c691b182c6676d5f8355 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Fri, 31 Dec 2021 15:08:44 +0100 Subject: [PATCH] prometheus: use static targets --- playbooks/prometheus.yml | 142 ++++++++----------- roles/prometheus/tasks/main.yml | 69 ++------- roles/prometheus/templates/prometheus.yml.j2 | 63 ++++---- 3 files changed, 107 insertions(+), 167 deletions(-) diff --git a/playbooks/prometheus.yml b/playbooks/prometheus.yml index fe4be29..553f960 100755 --- a/playbooks/prometheus.yml +++ b/playbooks/prometheus.yml @@ -8,15 +8,13 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json - prometheus_targets: - - targets: | - {{ groups['fleming_pve'] + groups['fleming_vm'] | list | sort }} - prometheus_unifi_snmp_targets: - - targets: "{{ groups['fleming_unifi'] | list | sort }}" - prometheus_ipmi_targets: - - targets: | - {{ groups['fleming_pve'] + groups['fleming_server'] | list | sort }} + prometheus_servers_targets: | + {{ groups['fleming_pve'] + groups['fleming_vm'] | list | sort }} + prometheus_unifi_snmp_targets: | + {{ groups['fleming_unifi'] | list | sort }} + prometheus_ipmi_targets: | + {{ groups['fleming_pve'] + groups['fleming_server'] | list | sort }} + update_motd: prometheus: >- Prometheus (en configuration fleming) est déployé (/etc/prometheus). @@ -32,18 +30,16 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json - prometheus_targets: - - targets: | - {{ groups['pacaterie_pve'] + groups['pacaterie_vm'] | list | sort }} - prometheus_unifi_snmp_targets: - - targets: "{{ groups['pacaterie_unifi'] | list | sort }}" + prometheus_servers_targets: | + {{ groups['pacaterie_pve'] + groups['pacaterie_vm'] | list | sort }} + prometheus_unifi_snmp_targets: | + {{ groups['pacaterie_unifi'] | list | sort }} prometheus_ups_snmp_targets: - ups-pn-1.ups.auro.re - ups-ps-1.ups.auro.re - prometheus_ipmi_targets: - - targets: | - {{ groups['pacaterie_pve'] | list | sort }} + prometheus_ipmi_targets: | + {{ groups['pacaterie_pve'] | list | sort }} + update_motd: prometheus: >- Prometheus (en configuration pacaterie) est déployé (/etc/prometheus). @@ -59,20 +55,17 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json prometheus_ups_snmp_targets: - ups-ec-1.ups.auro.re - ups-ec-2.ups.auro.re - ups-ec-3.ups.auro.re + prometheus_servers_targets: | + {{ groups['edc_pve'] + groups['edc_vm'] + groups['edc_server'] | list | sort }} + prometheus_unifi_snmp_targets: | + {{ groups['edc_unifi'] | list | sort }} + prometheus_ipmi_targets: | + {{ groups['edc_pve'] + groups['edc_server'] | list | sort }} - prometheus_targets: - - targets: | - {{ groups['edc_pve'] + groups['edc_vm'] + groups['edc_server'] | list | sort }} - prometheus_unifi_snmp_targets: - - targets: "{{ groups['edc_unifi'] | list | sort }}" - prometheus_ipmi_targets: - - targets: | - {{ groups['edc_pve'] + groups['edc_server'] | list | sort }} update_motd: prometheus: >- Prometheus (en configuration edc) est déployé (/etc/prometheus). @@ -88,19 +81,17 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json - prometheus_targets: - - targets: | - {{ groups['gs_pve'] + groups['gs_vm'] | list | sort }} - prometheus_unifi_snmp_targets: - - targets: "{{ groups['gs_unifi'] | list | sort }}" + prometheus_servers_targets: | + {{ groups['gs_pve'] + groups['gs_vm'] | list | sort }} + prometheus_unifi_snmp_targets: | + {{ groups['gs_unifi'] | list | sort }} prometheus_ups_snmp_targets: - ups-gk-1.ups.auro.re - prometheus_pdu_snmp_targets: + prometheus_apc_pdu_snmp_targets: - pdu-ga-1.ups.auro.re - prometheus_ipmi_targets: - - targets: | - {{ groups['gs_pve'] | list | sort }} + prometheus_ipmi_targets: | + {{ groups['gs_pve'] | list | sort }} + update_motd: prometheus: >- Prometheus (en configuration gs) est déployé (/etc/prometheus). @@ -116,19 +107,16 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json prometheus_ups_snmp_targets: - ups-r3-1.ups.auro.re - ups-r1-1.ups.auro.re + prometheus_servers_targets: | + {{ groups['rives_pve'] + groups['rives_vm'] | list | sort }} + prometheus_unifi_snmp_targets: | + {{ groups['rives_unifi'] | list | sort }} + prometheus_ipmi_targets: | + {{ groups['rives_pve'] | list | sort }} - prometheus_targets: - - targets: | - {{ groups['rives_pve'] + groups['rives_vm'] | list | sort }} - prometheus_unifi_snmp_targets: - - targets: "{{ groups['rives_unifi'] | list | sort }}" - prometheus_ipmi_targets: - - targets: | - {{ groups['rives_pve'] | list | sort }} update_motd: prometheus: >- Prometheus (en configuration rives) est déployé (/etc/prometheus). @@ -144,32 +132,28 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json - prometheus_targets: - - targets: | - {{ groups['aurore_pve'] + groups['aurore_vm'] | list | sort }} - prometheus_postgres_targets: - - targets: | - {{ groups['bdd'] + groups['radius'] | list | sort }} + prometheus_servers_targets: | + {{ groups['aurore_pve'] + groups['aurore_vm'] | list | sort }} + prometheus_postgresql_targets: | + {{ groups['bdd'] + groups['radius'] | list | sort }} prometheus_switch_snmp_targets: - - targets: - - yggdrasil.switch.auro.re - - sw-pn-serveurs.switch.auro.re - - sw-ec-serveurs.switch.auro.re - - sw-gk-serveurs.switch.auro.re - - sw-fl-serveurs.switch.auro.re - - sw-ff-uplink.switch.auro.re - - sw-fl-core.switch.auro.re - - sw-fd-vcore.switch.auro.re - - sw-fl-vcore.switch.auro.re - - sw-ff-vcore.switch.auro.re - - sw-pn-core.switch.auro.re - - sw-ec-core.switch.auro.re - - sw-gk-core.switch.auro.re - - sw-r3-core.switch.auro.re - prometheus_ipmi_targets: - - targets: | - {{ groups['aurore_pve'] | list | sort }} + - yggdrasil.switch.auro.re + - sw-pn-serveurs.switch.auro.re + - sw-ec-serveurs.switch.auro.re + - sw-gk-serveurs.switch.auro.re + - sw-fl-serveurs.switch.auro.re + - sw-ff-uplink.switch.auro.re + - sw-fl-core.switch.auro.re + - sw-fd-vcore.switch.auro.re + - sw-fl-vcore.switch.auro.re + - sw-ff-vcore.switch.auro.re + - sw-pn-core.switch.auro.re + - sw-ec-core.switch.auro.re + - sw-gk-core.switch.auro.re + - sw-r3-core.switch.auro.re + prometheus_ipmi_targets: | + {{ groups['aurore_pve'] | list | sort }} + update_motd: prometheus: >- Prometheus (en configuration aurore) est déployé (/etc/prometheus). @@ -185,15 +169,13 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json - prometheus_targets: - - targets: | - {{ groups['ovh_pve'] + groups['ovh_vm'] | list | sort }} - prometheus_postgres_targets: - - targets: - - bdd-ovh.adm.auro.re + prometheus_servers_targets: | + {{ groups['ovh_pve'] + groups['ovh_vm'] | list | sort }} + prometheus_postgresql_targets: + - bdd-ovh.adm.auro.re prometheus_docker_targets: - docker-ovh.adm.auro.re + update_motd: prometheus: >- Prometheus (en configuration ovh) est déployé (/etc/prometheus). @@ -208,8 +190,7 @@ snmp_pdu_user: "{{ vault_snmp_pdu_user }}" snmp_pdu_password: "{{ vault_snmp_pdu_password }}" - # Prometheus targets.json - prometheus_targets: + prometheus_servers_targets: - prometheus-edc.adm.auro.re - prometheus-gs.adm.auro.re - prometheus-fleming.adm.auro.re @@ -217,6 +198,7 @@ - prometheus-rives.adm.auro.re - prometheus-aurore.adm.auro.re - prometheus-ovh.adm.auro.re + update_motd: prometheus_federate: >- Prometheus (en configuration fédération) est déployé (/etc/prometheus). diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index 5d4e7ca..98d0c52 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -63,62 +63,19 @@ - snmp.yml notify: Restart prometheus-snmp-exporter -# We don't need to restart Prometheus when updating nodes -- name: Configure Prometheus nodes - copy: - content: "{{ prometheus_targets | to_nice_json }}\n" - dest: /etc/prometheus/targets.json - mode: 0644 - -# We don't need to restart Prometheus when updating nodes -- name: Configure Prometheus Ubiquity Unifi SNMP devices - copy: - content: "{{ prometheus_unifi_snmp_targets | to_nice_json }}\n" - dest: /etc/prometheus/targets_unifi_snmp.json - mode: 0644 - when: prometheus_unifi_snmp_targets is defined - -- name: Configure Prometheus Switchs - copy: - content: "{{ prometheus_switch_snmp_targets | to_nice_json }}\n" - dest: /etc/prometheus/targets_switch_snmp.json - mode: 0644 - when: prometheus_switch_snmp_targets is defined - -- name: Configure Prometheus UPS SNMP devices - copy: - content: "{{ [{'targets': prometheus_ups_snmp_targets }] | to_nice_json }}\n" - dest: /etc/prometheus/targets_ups_snmp.json - mode: 0644 - when: prometheus_ups_snmp_targets is defined - -- name: Configure Prometheus docker monitoring - copy: - content: "{{ [{'targets': prometheus_docker_targets }] | to_nice_json }}\n" - dest: /etc/prometheus/targets_docker.json - mode: 0644 - when: prometheus_docker_targets is defined - -- name: Configure Prometheus postgres monitoring - copy: - content: "{{ prometheus_postgres_targets | to_nice_json }}\n" - dest: /etc/prometheus/targets_postgres.json - mode: 0644 - when: prometheus_postgres_targets is defined - -- name: Configure Prometheus apc_pdu monitoring - copy: - content: "{{ [{'targets': prometheus_pdu_snmp_targets }] | to_nice_json }}\n" - dest: /etc/prometheus/targets_apc_pdu_snmp.json - mode: 0644 - when: prometheus_pdu_snmp_targets is defined - -- name: Configure Prometheus ipmi targets monitoring - copy: - content: "{{ prometheus_ipmi_targets | to_nice_json }}\n" - dest: /etc/prometheus/targets_ipmi.json - mode: 0644 - when: prometheus_ipmi_targets is defined +- name: Remove old targets + file: + path: "{{ item }}" + state: absent + loop: + - /etc/prometheus/targets.json + - /etc/prometheus/targets_unifi_snmp.json + - /etc/prometheus/targets_switch_snmp.json + - /etc/prometheus/targets_ups_snmp.json + - /etc/prometheus/targets_docker.json + - /etc/prometheus/targets_postgres.json + - /etc/prometheus/targets_apc_pdu_snmp.json + - /etc/prometheus/targets_ipmi.json - name: Activate prometheus service systemd: diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index 05564af..15f12e9 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -25,17 +25,15 @@ rule_files: # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: - # The .json in file_sd_configs is dynamically reloaded - - job_name: prometheus static_configs: - targets: - localhost:9090 +{% if prometheus_servers_targets is defined %} - job_name: servers - file_sd_configs: - - files: - - '/etc/prometheus/targets.json' + static_configs: + - targets: {{ prometheus_servers_targets | to_json }} relabel_configs: # Do not put :9100 in instance name, rather here - source_labels: [__address__] @@ -45,11 +43,12 @@ scrape_configs: - source_labels: [__param_target] target_label: __address__ replacement: '$1:9100' +{% endif %} +{% if prometheus_unifi_snmp_targets is defined %} - job_name: unifi_snmp - file_sd_configs: - - files: - - '/etc/prometheus/targets_unifi_snmp.json' + static_configs: + - targets: {{ prometheus_unifi_snmp_targets | to_json }} metrics_path: /snmp params: module: [ubiquiti_unifi] @@ -60,16 +59,12 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: 127.0.0.1:9116 +{% endif %} - - job_name: django - scheme: https - static_configs: - - targets: [] - +{% if prometheus_ups_snmp_targets is defined %} - job_name: ups_snmp - file_sd_configs: - - files: - - '/etc/prometheus/targets_ups_snmp.json' + static_configs: + - targets: {{ prometheus_ups_snmp_targets | to_json }} metrics_path: /snmp params: module: [eatonups] @@ -80,11 +75,12 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: 127.0.0.1:9116 +{% endif %} +{% if prometheus_switch_snmp_targets is defined %} - job_name: switch_snmp - file_sd_configs: - - files: - - "/etc/prometheus/targets_switch_snmp.json" + static_configs: + - targets: {{ prometheus_switch_snmp_targets | to_json }} metrics_path: /snmp params: module: [procurve_switch] @@ -95,11 +91,12 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: 127.0.0.1:9116 +{% endif %} +{% if prometheus_docker_targets is defined %} - job_name: docker - file_sd_configs: - - files: - - '/etc/prometheus/targets_docker.json' + static_configs: + - targets: {{ prometheus_docker_targets | to_json }} relabel_configs: # Do not put :8087 in instance name, rather here - source_labels: [__address__] @@ -109,11 +106,12 @@ scrape_configs: - source_labels: [__param_target] target_label: __address__ replacement: '$1:8087' +{% endif %} +{% if prometheus_postgresql_targets is defined %} - job_name: postgresql - file_sd_configs: - - files: - - '/etc/prometheus/targets_postgres.json' + static_configs: + - targets: {{ prometheus_postgresql_targets | to_json }} relabel_configs: # Do not put :9187 in instance name, rather here - source_labels: [__address__] @@ -123,11 +121,12 @@ scrape_configs: - source_labels: [__param_target] target_label: __address__ replacement: '$1:9187' +{% endif %} +{% if prometheus_apc_pdu_snmp_targets is defined %} - job_name: apc_pdu_snmp - file_sd_configs: - - files: - - '/etc/prometheus/targets_apc_pdu_snmp.json' + static_configs: + - targets: {{ prometheus_apc_pdu_snmp_targets | to_json }} metrics_path: /snmp params: module: @@ -139,11 +138,12 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: 127.0.0.1:9116 +{% endif %} +{% if prometheus_ipmi_targets is defined %} - job_name: ipmi - file_sd_configs: - - files: - - '/etc/prometheus/targets_ipmi.json' + static_configs: + - targets: {{ prometheus_ipmi_targets | to_json }} metrics_path: /metrics params: module: [default] @@ -156,5 +156,6 @@ scrape_configs: - source_labels: [__param_target] target_label: __address__ replacement: '$1:9290' +{% endif %} ...