prometheus: use static targets

playbooks/prometheus.yml

@@ -8,15 +8,13 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
+    prometheus_servers_targets: |
-    prometheus_targets:
+      {{ groups['fleming_pve'] + groups['fleming_vm'] | list | sort }}
-      - targets: |
+    prometheus_unifi_snmp_targets: |
-          {{ groups['fleming_pve'] + groups['fleming_vm'] | list | sort }}
+      {{ groups['fleming_unifi'] | list | sort }}
-    prometheus_unifi_snmp_targets:
+    prometheus_ipmi_targets: |
-      - targets: "{{ groups['fleming_unifi'] | list | sort }}"
+      {{ groups['fleming_pve'] + groups['fleming_server'] | list | sort }}
-    prometheus_ipmi_targets:
+
-      - targets: |
-          {{ groups['fleming_pve'] + groups['fleming_server'] | list | sort }}
     update_motd:
       prometheus: >-
         Prometheus (en configuration fleming) est déployé (/etc/prometheus).
@@ -32,18 +30,16 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
+    prometheus_servers_targets: |
-    prometheus_targets:
+      {{ groups['pacaterie_pve'] + groups['pacaterie_vm'] | list | sort }}
-      - targets: |
+    prometheus_unifi_snmp_targets: |
-          {{ groups['pacaterie_pve'] + groups['pacaterie_vm'] | list | sort }}
+      {{ groups['pacaterie_unifi'] | list | sort }}
-    prometheus_unifi_snmp_targets:
-      - targets: "{{ groups['pacaterie_unifi'] | list | sort }}"
     prometheus_ups_snmp_targets:
       - ups-pn-1.ups.auro.re
       - ups-ps-1.ups.auro.re
-    prometheus_ipmi_targets:
+    prometheus_ipmi_targets: |
-      - targets: |
+      {{ groups['pacaterie_pve'] | list | sort }}
-          {{ groups['pacaterie_pve'] | list | sort }}
+
     update_motd:
       prometheus: >-
         Prometheus (en configuration pacaterie) est déployé (/etc/prometheus).
@@ -59,20 +55,17 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
     prometheus_ups_snmp_targets:
       - ups-ec-1.ups.auro.re
       - ups-ec-2.ups.auro.re
       - ups-ec-3.ups.auro.re
+    prometheus_servers_targets: |
+      {{ groups['edc_pve'] + groups['edc_vm'] + groups['edc_server'] | list | sort }}
+    prometheus_unifi_snmp_targets: |
+      {{ groups['edc_unifi'] | list | sort }}
+    prometheus_ipmi_targets: |
+      {{ groups['edc_pve'] + groups['edc_server'] | list | sort }}
 
-    prometheus_targets:
-      - targets: |
-          {{ groups['edc_pve'] + groups['edc_vm'] + groups['edc_server'] | list | sort }}
-    prometheus_unifi_snmp_targets:
-      - targets: "{{ groups['edc_unifi'] | list | sort }}"
-    prometheus_ipmi_targets:
-      - targets: |
-          {{ groups['edc_pve'] + groups['edc_server'] | list | sort }}
     update_motd:
       prometheus: >-
         Prometheus (en configuration edc) est déployé (/etc/prometheus).
@@ -88,19 +81,17 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
+    prometheus_servers_targets: |
-    prometheus_targets:
+      {{ groups['gs_pve'] + groups['gs_vm'] | list | sort }}
-      - targets: |
+    prometheus_unifi_snmp_targets: |
-          {{ groups['gs_pve'] + groups['gs_vm'] | list | sort }}
+      {{ groups['gs_unifi'] | list | sort }}
-    prometheus_unifi_snmp_targets:
-      - targets: "{{ groups['gs_unifi'] | list | sort }}"
     prometheus_ups_snmp_targets:
       - ups-gk-1.ups.auro.re
-    prometheus_pdu_snmp_targets:
+    prometheus_apc_pdu_snmp_targets:
       - pdu-ga-1.ups.auro.re
-    prometheus_ipmi_targets:
+    prometheus_ipmi_targets: |
-      - targets: |
+      {{ groups['gs_pve'] | list | sort }}
-          {{ groups['gs_pve'] | list | sort }}
+
     update_motd:
       prometheus: >-
         Prometheus (en configuration gs) est déployé (/etc/prometheus).
@@ -116,19 +107,16 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
     prometheus_ups_snmp_targets:
       - ups-r3-1.ups.auro.re
       - ups-r1-1.ups.auro.re
+    prometheus_servers_targets: |
+      {{ groups['rives_pve'] + groups['rives_vm'] | list | sort }}
+    prometheus_unifi_snmp_targets: |
+      {{ groups['rives_unifi'] | list | sort }}
+    prometheus_ipmi_targets: |
+      {{ groups['rives_pve'] | list | sort }}
 
-    prometheus_targets:
-      - targets: |
-          {{ groups['rives_pve'] + groups['rives_vm'] | list | sort }}
-    prometheus_unifi_snmp_targets:
-      - targets: "{{ groups['rives_unifi'] | list | sort }}"
-    prometheus_ipmi_targets:
-      - targets: |
-          {{ groups['rives_pve'] | list | sort }}
     update_motd:
       prometheus: >-
         Prometheus (en configuration rives) est déployé (/etc/prometheus).
@@ -144,32 +132,28 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
+    prometheus_servers_targets: |
-    prometheus_targets:
+      {{ groups['aurore_pve'] + groups['aurore_vm'] | list | sort }}
-      - targets: |
+    prometheus_postgresql_targets: |
-          {{ groups['aurore_pve'] + groups['aurore_vm'] | list | sort }}
+      {{ groups['bdd'] + groups['radius'] | list | sort }}
-    prometheus_postgres_targets:
-      - targets: |
-          {{ groups['bdd'] + groups['radius'] | list | sort }}
     prometheus_switch_snmp_targets:
-      - targets:
+      - yggdrasil.switch.auro.re
-          - yggdrasil.switch.auro.re
+      - sw-pn-serveurs.switch.auro.re
-          - sw-pn-serveurs.switch.auro.re
+      - sw-ec-serveurs.switch.auro.re
-          - sw-ec-serveurs.switch.auro.re
+      - sw-gk-serveurs.switch.auro.re
-          - sw-gk-serveurs.switch.auro.re
+      - sw-fl-serveurs.switch.auro.re
-          - sw-fl-serveurs.switch.auro.re
+      - sw-ff-uplink.switch.auro.re
-          - sw-ff-uplink.switch.auro.re
+      - sw-fl-core.switch.auro.re
-          - sw-fl-core.switch.auro.re
+      - sw-fd-vcore.switch.auro.re
-          - sw-fd-vcore.switch.auro.re
+      - sw-fl-vcore.switch.auro.re
-          - sw-fl-vcore.switch.auro.re
+      - sw-ff-vcore.switch.auro.re
-          - sw-ff-vcore.switch.auro.re
+      - sw-pn-core.switch.auro.re
-          - sw-pn-core.switch.auro.re
+      - sw-ec-core.switch.auro.re
-          - sw-ec-core.switch.auro.re
+      - sw-gk-core.switch.auro.re
-          - sw-gk-core.switch.auro.re
+      - sw-r3-core.switch.auro.re
-          - sw-r3-core.switch.auro.re
+    prometheus_ipmi_targets: |
-    prometheus_ipmi_targets:
+      {{ groups['aurore_pve'] | list | sort }}
-      - targets: |
+
-          {{ groups['aurore_pve'] | list | sort }}
     update_motd:
       prometheus: >-
         Prometheus (en configuration aurore) est déployé (/etc/prometheus).
@@ -185,15 +169,13 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
+    prometheus_servers_targets: |
-    prometheus_targets:
+      {{ groups['ovh_pve'] + groups['ovh_vm'] | list | sort }}
-      - targets: |
+    prometheus_postgresql_targets:
-          {{ groups['ovh_pve'] + groups['ovh_vm'] | list | sort }}
+      - bdd-ovh.adm.auro.re
-    prometheus_postgres_targets:
-      - targets:
-          - bdd-ovh.adm.auro.re
     prometheus_docker_targets:
       - docker-ovh.adm.auro.re
+
     update_motd:
       prometheus: >-
         Prometheus (en configuration ovh) est déployé (/etc/prometheus).
@@ -208,8 +190,7 @@
     snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
     snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
-    # Prometheus targets.json
+    prometheus_servers_targets:
-    prometheus_targets:
       - prometheus-edc.adm.auro.re
       - prometheus-gs.adm.auro.re
       - prometheus-fleming.adm.auro.re
@@ -217,6 +198,7 @@
       - prometheus-rives.adm.auro.re
       - prometheus-aurore.adm.auro.re
       - prometheus-ovh.adm.auro.re
+
     update_motd:
       prometheus_federate: >-
         Prometheus (en configuration fédération) est déployé (/etc/prometheus).

roles/prometheus/tasks/main.yml

@@ -63,62 +63,19 @@
     - snmp.yml
   notify: Restart prometheus-snmp-exporter
 
-# We don't need to restart Prometheus when updating nodes
+- name: Remove old targets
-- name: Configure Prometheus nodes
+  file:
-  copy:
+    path: "{{ item }}"
-    content: "{{ prometheus_targets | to_nice_json }}\n"
+    state: absent
-    dest: /etc/prometheus/targets.json
+  loop:
-    mode: 0644
+    - /etc/prometheus/targets.json
-
+    - /etc/prometheus/targets_unifi_snmp.json
-# We don't need to restart Prometheus when updating nodes
+    - /etc/prometheus/targets_switch_snmp.json
-- name: Configure Prometheus Ubiquity Unifi SNMP devices
+    - /etc/prometheus/targets_ups_snmp.json
-  copy:
+    - /etc/prometheus/targets_docker.json
-    content: "{{ prometheus_unifi_snmp_targets | to_nice_json }}\n"
+    - /etc/prometheus/targets_postgres.json
-    dest: /etc/prometheus/targets_unifi_snmp.json
+    - /etc/prometheus/targets_apc_pdu_snmp.json
-    mode: 0644
+    - /etc/prometheus/targets_ipmi.json
-  when: prometheus_unifi_snmp_targets is defined
-
-- name: Configure Prometheus Switchs
-  copy:
-    content: "{{ prometheus_switch_snmp_targets | to_nice_json }}\n"
-    dest: /etc/prometheus/targets_switch_snmp.json
-    mode: 0644
-  when: prometheus_switch_snmp_targets is defined
-
-- name: Configure Prometheus UPS SNMP devices
-  copy:
-    content: "{{ [{'targets': prometheus_ups_snmp_targets }] | to_nice_json }}\n"
-    dest: /etc/prometheus/targets_ups_snmp.json
-    mode: 0644
-  when: prometheus_ups_snmp_targets is defined
-
-- name: Configure Prometheus docker monitoring
-  copy:
-    content: "{{ [{'targets': prometheus_docker_targets }] | to_nice_json }}\n"
-    dest: /etc/prometheus/targets_docker.json
-    mode: 0644
-  when: prometheus_docker_targets is defined
-
-- name: Configure Prometheus postgres monitoring
-  copy:
-    content: "{{ prometheus_postgres_targets | to_nice_json }}\n"
-    dest: /etc/prometheus/targets_postgres.json
-    mode: 0644
-  when: prometheus_postgres_targets is defined
-
-- name: Configure Prometheus apc_pdu monitoring
-  copy:
-    content: "{{ [{'targets': prometheus_pdu_snmp_targets }] | to_nice_json }}\n"
-    dest: /etc/prometheus/targets_apc_pdu_snmp.json
-    mode: 0644
-  when: prometheus_pdu_snmp_targets is defined
-
-- name: Configure Prometheus ipmi targets monitoring
-  copy:
-    content: "{{ prometheus_ipmi_targets | to_nice_json }}\n"
-    dest: /etc/prometheus/targets_ipmi.json
-    mode: 0644
-  when: prometheus_ipmi_targets is defined
 
 - name: Activate prometheus service
   systemd:

roles/prometheus/templates/prometheus.yml.j2

@@ -25,17 +25,15 @@ rule_files:
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
 scrape_configs:
-  # The .json in file_sd_configs is dynamically reloaded
-
   - job_name: prometheus
     static_configs:
       - targets:
         - localhost:9090
 
+{% if prometheus_servers_targets is defined %}
   - job_name: servers
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_servers_targets | to_json }}
-        - '/etc/prometheus/targets.json'
     relabel_configs:
       # Do not put :9100 in instance name, rather here
       - source_labels: [__address__]
@@ -45,11 +43,12 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: __address__
         replacement: '$1:9100'
+{% endif %}
 
+{% if prometheus_unifi_snmp_targets is defined %}
   - job_name: unifi_snmp
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_unifi_snmp_targets | to_json }}
-        - '/etc/prometheus/targets_unifi_snmp.json'
     metrics_path: /snmp
     params:
       module: [ubiquiti_unifi]
@@ -60,16 +59,12 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 127.0.0.1:9116
+{% endif %}
 
-  - job_name: django
+{% if prometheus_ups_snmp_targets is defined %}
-    scheme: https
-    static_configs:
-    - targets: []
-
   - job_name: ups_snmp
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_ups_snmp_targets | to_json }}
-        - '/etc/prometheus/targets_ups_snmp.json'
     metrics_path: /snmp
     params:
       module: [eatonups]
@@ -80,11 +75,12 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 127.0.0.1:9116
+{% endif %}
 
+{% if prometheus_switch_snmp_targets is defined %}
   - job_name: switch_snmp
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_switch_snmp_targets | to_json }}
-        - "/etc/prometheus/targets_switch_snmp.json"
     metrics_path: /snmp
     params:
       module: [procurve_switch]
@@ -95,11 +91,12 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 127.0.0.1:9116
+{% endif %}
 
+{% if prometheus_docker_targets is defined %}
   - job_name: docker
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_docker_targets | to_json }}
-        - '/etc/prometheus/targets_docker.json'
     relabel_configs:
       # Do not put :8087 in instance name, rather here
       - source_labels: [__address__]
@@ -109,11 +106,12 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: __address__
         replacement: '$1:8087'
+{% endif %}
 
+{% if prometheus_postgresql_targets is defined %}
   - job_name: postgresql
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_postgresql_targets | to_json }}
-        - '/etc/prometheus/targets_postgres.json'
     relabel_configs:
       # Do not put :9187 in instance name, rather here
       - source_labels: [__address__]
@@ -123,11 +121,12 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: __address__
         replacement: '$1:9187'
+{% endif %}
 
+{% if prometheus_apc_pdu_snmp_targets is defined %}
   - job_name: apc_pdu_snmp
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_apc_pdu_snmp_targets | to_json }}
-        - '/etc/prometheus/targets_apc_pdu_snmp.json'
     metrics_path: /snmp
     params:
       module:
@@ -139,11 +138,12 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 127.0.0.1:9116
+{% endif %}
 
+{% if prometheus_ipmi_targets is defined %}
   - job_name: ipmi
-    file_sd_configs:
+    static_configs:
-      - files:
+      - targets: {{ prometheus_ipmi_targets | to_json }}
-        - '/etc/prometheus/targets_ipmi.json'
     metrics_path: /metrics
     params:
       module: [default]
@@ -156,5 +156,6 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: __address__
         replacement: '$1:9290'
+{% endif %}
 
 ...