Cleanup Prometheus alerts #45
3 changed files with 150 additions and 124 deletions
|
|
@ -8,3 +8,4 @@
|
||||||
service:
|
service:
|
||||||
name: prometheus-snmp-exporter
|
name: prometheus-snmp-exporter
|
||||||
state: restarted
|
state: restarted
|
||||||
|
...
|
||||||
|
|
|
||||||
|
|
@ -1,138 +1,161 @@
|
||||||
# {{ ansible_managed }}
|
---
|
||||||
{# As this is also Jinja2 it will conflict without a raw block #}
|
{{ ansible_managed | comment }}
|
||||||
{# Depending of Prometheus Node exporter version, rules can change depending of version #}
|
|
||||||
{% raw %}
|
{% macro raw(string) -%}
|
||||||
|
{{ string }}
|
||||||
|
{%- endmacro %}
|
||||||
|
|
||||||
|
{% set instance = '[{{ raw("$label.instance") }}]' %}
|
||||||
|
|
||||||
groups:
|
groups:
|
||||||
- name: alert.rules
|
- name: alert.rules
|
||||||
rules:
|
rules:
|
||||||
|
|
||||||
# Alert for any instance that is unreachable for >3 minutes.
|
- alert: InstanceDown
|
||||||
- alert: InstanceDown
|
expr: up == 0
|
||||||
expr: up == 0
|
for: 3m
|
||||||
for: 3m
|
labels:
|
||||||
labels:
|
severity: critical
|
||||||
severity: critical
|
annotations:
|
||||||
annotations:
|
summary: >-
|
||||||
summary: "{{ $labels.instance }} est invisible depuis plus de 3 minutes !"
|
{{ instance }} Invisible depuis plus de 3 minutes
|
||||||
|
|
||||||
# Alert for out of memory
|
- alert: OutOfMemory
|
||||||
- alert: OutOfMemory
|
expr: >-
|
||||||
expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 < 10
|
(
|
||||||
for: 5m
|
node_memory_MemFree_bytes
|
||||||
labels:
|
+ node_memory_Cached_bytes
|
||||||
severity: warning
|
+ node_memory_Buffers_bytes
|
||||||
annotations:
|
) / node_memory_MemTotal_bytes * 100 < 10
|
||||||
summary: "Mémoire libre de {{ $labels.instance }} à {{ humanize $value }}%."
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: >-
|
||||||
|
{{ instance }} {{ raw("$value | round") }}% de mémoire libre
|
||||||
|
|
||||||
# Alert for out of disk space
|
- alert: OutOfDiskSpace
|
||||||
- alert: OutOfDiskSpace
|
expr: >-
|
||||||
expr: node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10
|
node_filesystem_free_bytes / node_filesystem_size_bytes * 100 < 10
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: "Espace libre de {{ $labels.mountpoint }} sur {{ $labels.instance }} à {{ humanize $value }}%."
|
summary: >-
|
||||||
|
{{ instance }} Espace libre de {{ raw("$labels.mountpoint") }} sur
|
||||||
|
à {{ raw("$value | round") }}%
|
||||||
|
|
||||||
# Alert for out of inode space on disk
|
- alert: OutOfInodes
|
||||||
- alert: OutOfInodes
|
expr: node_filesystem_files_free / node_filesystem_files * 100 < 10
|
||||||
expr: node_filesystem_files_free{fstype="ext4"} / node_filesystem_files{fstype="ext4"} * 100 < 10
|
for: 5m
|
||||||
for: 5m
|
labels:
|
||||||
labels:
|
severity: warning
|
||||||
severity: warning
|
annotations:
|
||||||
annotations:
|
summary: >-
|
||||||
summary: "Presque plus d'inodes disponibles ({{ $value }}% restant) dans {{ $labels.mountpoint }} sur {{ $labels.instance }}."
|
{{ instance }} {{ raw("$value | round") }}% d'inodes restants
|
||||||
|
pour {{ raw("$labels.mountpoint") }}
|
||||||
|
|
||||||
# Alert for high CPU usage
|
- alert: CpuUsage
|
||||||
- alert: CpuUsage
|
expr: >-
|
||||||
expr: (100 - avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 75
|
(
|
||||||
for: 10m
|
100 - avg by (instance) (
|
||||||
labels:
|
irate(node_cpu_seconds_total{mode="idle"}[5m])
|
||||||
severity: warning
|
) * 100
|
||||||
annotations:
|
) > 75
|
||||||
summary: "CPU sur {{ $labels.instance }} à {{ humanize $value }}%."
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: >-
|
||||||
|
{{ instance }} CPU à {{ raw("$value | round") }}%
|
||||||
|
|
||||||
# Check systemd unit (> buster)
|
- alert: SystemdServiceFailed
|
||||||
- alert: SystemdServiceFailed
|
expr: node_systemd_unit_state{state="failed"} == 1
|
||||||
expr: node_systemd_unit_state{state="failed"} == 1
|
for: 10m
|
||||||
for: 10m
|
labels:
|
||||||
labels:
|
severity: warning
|
||||||
severity: warning
|
annotations:
|
||||||
annotations:
|
summary: >-
|
||||||
summary: "{{ $labels.name }} a échoué sur {{ $labels.instance }}"
|
{{ instance }} {{ raw("$label.name") }} a échoué
|
||||||
|
|
||||||
# Check load of instance
|
- alert: LoadUsage
|
||||||
- alert: LoadUsage
|
expr: node_load1 > 5
|
||||||
expr: node_load1 > 5
|
for: 2m
|
||||||
for: 2m
|
labels:
|
||||||
labels:
|
severity: warning
|
||||||
severity: warning
|
annotations:
|
||||||
annotations:
|
summary: >
|
||||||
summary: "La charge de {{ $labels.instance }} est à {{ $value }} !"
|
{{ instance }} Charge à {{ raw("$value") }}
|
||||||
|
|
||||||
# Check UPS
|
- alert: UpsOutputSourceChanged
|
||||||
- alert: UpsOutputSourceChanged
|
expr: upsOutputSource != 3
|
||||||
expr: upsOutputSource != 3
|
for: 1m
|
||||||
for: 1m
|
labels:
|
||||||
labels:
|
severity: critical
|
||||||
severity: warning
|
annotations:
|
||||||
annotations:
|
summary: >-
|
||||||
summary: "La source d'alimentation de {{ $labels.instance }} a changé !"
|
{{ instance }} Source d'alimentation changée
|
||||||
|
|
||||||
- alert: UpsBatteryStatusWarning
|
- alert: UpsBatteryStatusWarning
|
||||||
expr: upsBatteryStatus == 3
|
expr: upsBatteryStatus == 3
|
||||||
for: 2m
|
for: 2m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: "L'état de la batterie de {{ $labels.instance }} est faible !"
|
summary: >-
|
||||||
|
{{ instance }} État de la batterie faible
|
||||||
|
|
||||||
- alert: UpsBatteryStatusCritical
|
- alert: UpsBatteryStatusCritical
|
||||||
expr: upsBatteryStatus == 4
|
expr: upsBatteryStatus == 4
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: critical
|
||||||
annotations:
|
annotations:
|
||||||
summary: "L'état de la batterie de {{ $labels.instance }} est affaibli !"
|
summary: >-
|
||||||
|
{{ instance }} État de la batterie critique
|
||||||
|
|
||||||
- alert: UpsHighLoad
|
- alert: UpsHighLoad
|
||||||
expr: upsOutputPercentLoad > 70
|
expr: upsOutputPercentLoad > 70
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
annotations:
|
annotations:
|
||||||
summary: "La charge de {{ $labels.instance }} est de {{ $value }}% !"
|
summary: >-
|
||||||
|
{{ instance }} Charge de {{ raw("$value | round") }}%
|
||||||
|
|
||||||
- alert: UpsWrongInputVoltage
|
- alert: UpsWrongInputVoltage
|
||||||
expr: (upsInputVoltage < 210) or (upsInputVoltage > 250)
|
expr: (upsInputVoltage < 210) or (upsInputVoltage > 250)
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: "La tension d'entrée de {{ $labels.instance }} est de {{ $value }}V."
|
summary: >-
|
||||||
|
{{ instance }} Tension d'entrée de {{ raw("$value") }}V
|
||||||
|
|
||||||
- alert: UpsWrongOutputVoltage
|
- alert: UpsWrongOutputVoltage
|
||||||
expr: (upsOutputVoltage < 220) or (upsOutputVoltage > 240)
|
expr: (upsOutputVoltage < 220) or (upsOutputVoltage > 240)
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: "La tension de sortie de {{ $labels.instance }} est de {{ $value }}V."
|
summary: >-
|
||||||
|
{{ instance }} Tension de sortie de {{ raw("$value") }}V
|
||||||
|
|
||||||
- alert: UpsTimeRemainingWarning
|
- alert: UpsTimeRemainingWarning
|
||||||
expr: upsEstimatedMinutesRemaining < 8
|
expr: upsEstimatedMinutesRemaining < 8
|
||||||
for: 1m
|
for: 1m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: "L'autonomie restante sur {{ $labels.instance }} est de {{ $value }} min."
|
summary: >-
|
||||||
|
{{ instance }} Autonomie restante de {{ raw("$value") }} min
|
||||||
|
|
||||||
- alert: UpsTimeRemainingCritical
|
- alert: UpsTimeRemainingCritical
|
||||||
expr: upsEstimatedMinutesRemaining < 5
|
expr: upsEstimatedMinutesRemaining < 5
|
||||||
for: 1m
|
for: 1m
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
annotations:
|
annotations:
|
||||||
summary: "L'autonomie restante sur {{ $labels.instance }} est de {{ $value }} min."
|
summary: >-
|
||||||
|
{{ instance }} Autonomie restante de {{ raw("$value") }} min
|
||||||
|
...
|
||||||
{% endraw %}
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
# {{ ansible_managed }}
|
---
|
||||||
|
{{ ansible_managed | comment }}
|
||||||
|
|
||||||
global:
|
global:
|
||||||
# scrape_interval is set to the global default (60s)
|
# scrape_interval is set to the global default (60s)
|
||||||
|
|
@ -100,3 +101,4 @@ scrape_configs:
|
||||||
file_sd_configs:
|
file_sd_configs:
|
||||||
- files:
|
- files:
|
||||||
- '/etc/prometheus/targets_docker.json'
|
- '/etc/prometheus/targets_docker.json'
|
||||||
|
...
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue