Cleanup Prometheus alerts #51

Closed
jeltz wants to merge 2 commits from add_prometheus_alerts into master
1 changed files with 69 additions and 27 deletions

View File

@ -1,7 +1,10 @@
# {{ ansible_managed }}
{# As this is also Jinja2 it will conflict without a raw block #}
{# Depending of Prometheus Node exporter version, rules can change depending of version #}
{% raw %}
---
{{ ansible_managed | comment }}
{% macro raw(string) -%}
{{ '{{' }} {{ string }} {{ '}}' }}
{%- endmacro %}
groups:
- name: alert.rules
rules:
@ -13,43 +16,59 @@ groups:
labels:
severity: critical
annotations:
summary: "{{ $labels.instance }} est invisible depuis plus de 3 minutes !"
summary: >-
{{ raw('$labels.instance') }} est invisible depuis plus de 3 minutes !
# Alert for out of memory
- alert: OutOfMemory
expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 < 10
expr: >-
(node_memory_MemFree_bytes +
node_memory_Cached_bytes +
node_memory_Buffers_bytes) /
node_memory_MemTotal_bytes * 100 < 10
for: 5m
labels:
severity: warning
annotations:
summary: "Mémoire libre de {{ $labels.instance }} à {{ humanize $value }}%."
summary: >-
Mémoire libre de {{ raw('$labels.instance') }} à
{{ raw('$value | humanize') }}%
# Alert for out of disk space
- alert: OutOfDiskSpace
expr: node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10
expr: >-
node_filesystem_free_bytes / node_filesystem_size_bytes * 100 < 10
for: 5m
labels:
severity: warning
annotations:
summary: "Espace libre de {{ $labels.mountpoint }} sur {{ $labels.instance }} à {{ humanize $value }}%."
summary: >-
Espace libre de {{ raw('$labels.mountpoint') }} sur
{{ raw('$labels.instance') }} à {{ raw('$value | humanize') }}%
# Alert for out of inode space on disk
- alert: OutOfInodes
expr: node_filesystem_files_free{fstype="ext4"} / node_filesystem_files{fstype="ext4"} * 100 < 10
expr: >-
node_filesystem_files_free / node_filesystem_files * 100 < 10
for: 5m
labels:
severity: warning
annotations:
summary: "Presque plus d'inodes disponibles ({{ $value }}% restant) dans {{ $labels.mountpoint }} sur {{ $labels.instance }}."
summary: >-
Presque plus d'inodes disponibles ({{ raw('$value') }}%)
dans {{ raw('$labels.mountpoint') }} sur {{ raw('$labels.instance') }}
# Alert for high CPU usage
- alert: CpuUsage
expr: (100 - avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 75
expr: >-
(100 - avg by (instance)
(irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 75
for: 10m
labels:
severity: warning
annotations:
summary: "CPU sur {{ $labels.instance }} à {{ humanize $value }}%."
summary: >-
CPU sur {{ raw('$labels.instance') }} à {{ raw('$value | humanize') }}%
# Check systemd unit (> buster)
- alert: SystemdServiceFailed
@ -58,7 +77,8 @@ groups:
labels:
severity: warning
annotations:
summary: "{{ $labels.name }} a échoué sur {{ $labels.instance }}"
summary: >-
{{ raw('$labels.name') }} a échoué sur {{ raw('$labels.instance') }}
# Check load of instance
- alert: LoadUsage
@ -67,8 +87,19 @@ groups:
labels:
severity: warning
annotations:
summary: "La charge de {{ $labels.instance }} est à {{ $value }} !"
summary: >-
La charge de {{ raw('$labels.instance') }} est à {{ raw('$value') }} !
- alert: HostMemoryUnderMemoryPressure
expr: rate(node_vmstat_pgmajfault[1m]) > 1000
for: 2m
labels:
severity: warning
annotations:
summary: >-
La pression sur la mémoire de {{ raw('$labels.instance') }}
est importante
# Check UPS
- alert: UpsOutputSourceChanged
expr: upsOutputSource != 3
@ -76,7 +107,8 @@ groups:
labels:
severity: warning
annotations:
summary: "La source d'alimentation de {{ $labels.instance }} a changé !"
summary: >-
La source d'alimentation de {{ raw('$labels.instance') }} a changé !
- alert: UpsBatteryStatusWarning
expr: upsBatteryStatus == 3
@ -84,7 +116,8 @@ groups:
labels:
severity: warning
annotations:
summary: "L'état de la batterie de {{ $labels.instance }} est faible !"
summary: >-
L'état de la batterie de {{ raw('$labels.instance') }} est faible !
- alert: UpsBatteryStatusCritical
expr: upsBatteryStatus == 4
@ -92,7 +125,8 @@ groups:
labels:
severity: warning
annotations:
summary: "L'état de la batterie de {{ $labels.instance }} est affaibli !"
summary: >-
L'état de la batterie de {{ '$labels.instance' }} est affaibli !
- alert: UpsHighLoad
expr: upsOutputPercentLoad > 70
@ -100,7 +134,9 @@ groups:
labels:
severity: critical
annotations:
summary: "La charge de {{ $labels.instance }} est de {{ $value }}% !"
summary: >-
La charge de {{ raw('$labels.instance') }}
est de {{ raw('$value') }}%
- alert: UpsWrongInputVoltage
expr: (upsInputVoltage < 210) or (upsInputVoltage > 250)
@ -108,7 +144,9 @@ groups:
labels:
severity: warning
annotations:
summary: "La tension d'entrée de {{ $labels.instance }} est de {{ $value }}V."
summary: >-
La tension d'entrée de {{ raw('$labels.instance') }}
est de {{ raw('$value') }}V
- alert: UpsWrongOutputVoltage
expr: (upsOutputVoltage < 220) or (upsOutputVoltage > 240)
@ -116,7 +154,9 @@ groups:
labels:
severity: warning
annotations:
summary: "La tension de sortie de {{ $labels.instance }} est de {{ $value }}V."
summary: >-
La tension de sortie de {{ raw('$labels.instance') }}
est de {{ raw('$value') }}V
- alert: UpsTimeRemainingWarning
expr: upsEstimatedMinutesRemaining < 8
@ -124,7 +164,9 @@ groups:
labels:
severity: warning
annotations:
summary: "L'autonomie restante sur {{ $labels.instance }} est de {{ $value }} min."
summary: >-
L'autonomie restante sur {{ raw('$labels.instance') }}
est de {{ raw('$value') }} min
- alert: UpsTimeRemainingCritical
expr: upsEstimatedMinutesRemaining < 5
@ -132,7 +174,7 @@ groups:
labels:
severity: critical
annotations:
summary: "L'autonomie restante sur {{ $labels.instance }} est de {{ $value }} min."
{% endraw %}
summary: >-
L'autonomie restante sur {{ raw('$labels.instance') }}
est de {{ raw('$value') }} min
...