Kepp federated datas 4 months (120 days)

monitoring.yml

@@ -5,6 +5,8 @@
     prometheus_alertmanager: docker-ovh.adm.auro.re:9093
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
     snmp_switch_community: "{{ vault_snmp_switch_community }}"
+    snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
+    snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
     # Prometheus targets.json
     prometheus_targets:
@@ -20,6 +22,8 @@
     prometheus_alertmanager: docker-ovh.adm.auro.re:9093
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
     snmp_switch_community: "{{ vault_snmp_switch_community }}"
+    snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
+    snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
     # Prometheus targets.json
     prometheus_targets:
@@ -38,6 +42,8 @@
     prometheus_alertmanager: docker-ovh.adm.auro.re:9093
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
     snmp_switch_community: "{{ vault_snmp_switch_community }}"
+    snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
+    snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
     # Prometheus targets.json
     prometheus_ups_snmp_targets:
@@ -79,6 +85,8 @@
     prometheus_alertmanager: docker-ovh.adm.auro.re:9093
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
     snmp_switch_community: "{{ vault_snmp_switch_community }}"
+    snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
+    snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
     # Prometheus targets.json
     prometheus_ups_snmp_targets:
@@ -98,6 +106,8 @@
     prometheus_alertmanager: docker-ovh.adm.auro.re:9093
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
     snmp_switch_community: "{{ vault_snmp_switch_community }}"
+    snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
+    snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
     # Prometheus targets.json
     prometheus_targets:
@@ -130,6 +140,8 @@
     prometheus_alertmanager: docker-ovh.adm.auro.re:9093
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
     snmp_switch_community: "{{ vault_snmp_switch_community }}"
+    snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
+    snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
     # Prometheus targets.json
     prometheus_targets:
@@ -148,6 +160,8 @@
   vars:
     prometheus_alertmanager: docker-ovh.adm.auro.re:9093
     snmp_unifi_password: "{{ vault_snmp_unifi_password }}"
+    snmp_pdu_user: "{{ vault_snmp_pdu_user }}"
+    snmp_pdu_password: "{{ vault_snmp_pdu_password }}"
 
     # Prometheus targets.json
     prometheus_targets:

roles/prometheus/tasks/main.yml

@@ -41,6 +41,7 @@
     - django.rules.yml
     - ups.rules.yml
     - postgres.rules.yml
+    - environmental.rules.yml
   notify: Restart Prometheus
 
 - name: Make Prometheus snmp-exporter listen on localhost only

roles/prometheus/templates/environmental.rules.yml.j2

@@ -0,0 +1,52 @@
+---
+{{ ansible_managed | comment }}
+
+{% macro raw(string) -%}
+{{ "{{" }} {{ string }} {{ "}}" }}
+{%- endmacro %}
+
+groups:
+
+  - name: environmental.rules
+    rules:
+
+      - alert: EnvironmentalTemperature
+        expr: rPDU2SensorTempHumidityStatusTempC / 10 > 30
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: >-
+            Température environnementale à {{ raw('$value') }}°
+
+      - alert: EnvironmentalTemperature
+        expr: rPDU2SensorTempHumidityStatusTempC / 10 > 40
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: >-
+            Température environnementale à {{ raw('$value') }}° 
+
+
+      - alert: EnvironmentalTemperature
+        expr: xupsEnvRemoteTemp > 30
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: >-
+            Température environnementale à {{ raw('$value') }}°
+
+      - alert: EnvironmentalTemperature
+        expr: xupsEnvRemoteTemp > 40
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: >-
+            Température environnementale à {{ raw('$value') }}° 
+
+
+
+...

roles/prometheus_federate/tasks/main.yml

@@ -20,6 +20,13 @@
     - alert.rules.yml
   notify: Restart Prometheus
 
+- name: Define Prometheus retention time
+  lineinfile:
+    path: /etc/default/prometheus
+    regexp: '^ARGS=\"--log.level=debug\"'
+    line: "ARGS=\"--log.level=debug --storage.tsdb.retention.time=120d\""
+  notify: Restart Prometheus
+
 # We don't need to restart Prometheus when updating nodes
 - name: Configure Prometheus Federate devices
   copy:

roles/prometheus_federate/templates/prometheus.yml.j2

@@ -36,7 +36,7 @@ scrape_configs:
         - '{job="unifi_snmp"}'
         - '{job="django"}'
         - '{job="ups_snmp"}'
-        - '{job="django"}'
+        - '{job="apc_pdu_snmp"}'
         - '{job="docker"}'
         - '{job="switch_snmp"}'
 ...