aurore/ansible
aurore/ansible
11
2
Fork 0
Code Issues 1 Pull requests 13 Releases Wiki Activity

Uniform labels for alerts

Browse source
This commit is contained in:
jeltz 2021-04-01 05:21:08 +02:00
parent 424aa80d8f
commit e247aa3f70
3 changed files with 150 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
roles/prometheus/handlers/main.yml
View file

@ -8,3 +8,4 @@
service: service:
name: prometheus-snmp-exporter name: prometheus-snmp-exporter
state: restarted state: restarted
...

269
roles/prometheus/templates/prometheus/alert.rules.yml.j2
View file

@ -1,138 +1,161 @@
# {{ ansible_managed }} ---
{# As this is also Jinja2 it will conflict without a raw block #} {{ ansible_managed | comment }}
{# Depending of Prometheus Node exporter version, rules can change depending of version #}
{% raw %} {% macro raw(string) -%}
{{ string }}
{%- endmacro %}
{% set instance = '[{{ raw("$label.instance") }}]' %}
groups: groups:
- name: alert.rules - name: alert.rules
rules: rules:
# Alert for any instance that is unreachable for >3 minutes. - alert: InstanceDown
- alert: InstanceDown expr: up == 0
expr: up == 0 for: 3m
for: 3m labels:
labels: severity: critical
severity: critical annotations:
annotations: summary: >-
summary: "{{ $labels.instance }} est invisible depuis plus de 3 minutes !" {{ instance }} Invisible depuis plus de 3 minutes
# Alert for out of memory - alert: OutOfMemory
- alert: OutOfMemory expr: >-
expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 < 10 (
for: 5m node_memory_MemFree_bytes
labels: + node_memory_Cached_bytes
severity: warning + node_memory_Buffers_bytes
annotations: ) / node_memory_MemTotal_bytes * 100 < 10
summary: "Mémoire libre de {{ $labels.instance }} à {{ humanize $value }}%." for: 5m
labels:
severity: warning
annotations:
summary: >-
{{ instance }} {{ raw("$value | round") }}% de mémoire libre
# Alert for out of disk space - alert: OutOfDiskSpace
- alert: OutOfDiskSpace expr: >-
expr: node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10 node_filesystem_free_bytes / node_filesystem_size_bytes * 100 < 10
for: 5m for: 5m
labels: labels:
severity: warning severity: warning
annotations: annotations:
summary: "Espace libre de {{ $labels.mountpoint }} sur {{ $labels.instance }} à {{ humanize $value }}%." summary: >-
{{ instance }} Espace libre de {{ raw("$labels.mountpoint") }} sur
à {{ raw("$value | round") }}%
# Alert for out of inode space on disk - alert: OutOfInodes
- alert: OutOfInodes expr: node_filesystem_files_free / node_filesystem_files * 100 < 10
expr: node_filesystem_files_free{fstype="ext4"} / node_filesystem_files{fstype="ext4"} * 100 < 10 for: 5m
for: 5m labels:
labels: severity: warning
severity: warning annotations:
annotations: summary: >-
summary: "Presque plus d'inodes disponibles ({{ $value }}% restant) dans {{ $labels.mountpoint }} sur {{ $labels.instance }}." {{ instance }} {{ raw("$value | round") }}% d'inodes restants
pour {{ raw("$labels.mountpoint") }}
# Alert for high CPU usage - alert: CpuUsage
- alert: CpuUsage expr: >-
expr: (100 - avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 75 (
for: 10m 100 - avg by (instance) (
labels: irate(node_cpu_seconds_total{mode="idle"}[5m])
severity: warning ) * 100
annotations: ) > 75
summary: "CPU sur {{ $labels.instance }} à {{ humanize $value }}%." for: 10m
labels:
severity: warning
annotations:
summary: >-
{{ instance }} CPU à {{ raw("$value | round") }}%
# Check systemd unit (> buster) - alert: SystemdServiceFailed
- alert: SystemdServiceFailed expr: node_systemd_unit_state{state="failed"} == 1
expr: node_systemd_unit_state{state="failed"} == 1 for: 10m
for: 10m labels:
labels: severity: warning
severity: warning annotations:
annotations: summary: >-
summary: "{{ $labels.name }} a échoué sur {{ $labels.instance }}" {{ instance }} {{ raw("$label.name") }} a échoué
# Check load of instance - alert: LoadUsage
- alert: LoadUsage expr: node_load1 > 5
expr: node_load1 > 5 for: 2m
for: 2m labels:
labels: severity: warning
severity: warning annotations:
annotations: summary: >
summary: "La charge de {{ $labels.instance }} est à {{ $value }} !" {{ instance }} Charge à {{ raw("$value") }}
# Check UPS - alert: UpsOutputSourceChanged
- alert: UpsOutputSourceChanged expr: upsOutputSource != 3
expr: upsOutputSource != 3 for: 1m
for: 1m labels:
labels: severity: critical
severity: warning annotations:
annotations: summary: >-
summary: "La source d'alimentation de {{ $labels.instance }} a changé !" {{ instance }} Source d'alimentation changée
- alert: UpsBatteryStatusWarning - alert: UpsBatteryStatusWarning
expr: upsBatteryStatus == 3 expr: upsBatteryStatus == 3
for: 2m for: 2m
labels: labels:
severity: warning severity: warning
annotations: annotations:
summary: "L'état de la batterie de {{ $labels.instance }} est faible !" summary: >-
{{ instance }} État de la batterie faible
- alert: UpsBatteryStatusCritical - alert: UpsBatteryStatusCritical
expr: upsBatteryStatus == 4 expr: upsBatteryStatus == 4
for: 10m for: 10m
labels: labels:
severity: warning severity: critical
annotations: annotations:
summary: "L'état de la batterie de {{ $labels.instance }} est affaibli !" summary: >-
{{ instance }} État de la batterie critique
- alert: UpsHighLoad - alert: UpsHighLoad
expr: upsOutputPercentLoad > 70 expr: upsOutputPercentLoad > 70
for: 5m for: 5m
labels: labels:
severity: critical severity: critical
annotations: annotations:
summary: "La charge de {{ $labels.instance }} est de {{ $value }}% !" summary: >-
{{ instance }} Charge de {{ raw("$value | round") }}%
- alert: UpsWrongInputVoltage - alert: UpsWrongInputVoltage
expr: (upsInputVoltage < 210) or (upsInputVoltage > 250) expr: (upsInputVoltage < 210) or (upsInputVoltage > 250)
for: 10m for: 10m
labels: labels:
severity: warning severity: warning
annotations: annotations:
summary: "La tension d'entrée de {{ $labels.instance }} est de {{ $value }}V." summary: >-
{{ instance }} Tension d'entrée de {{ raw("$value") }}V
- alert: UpsWrongOutputVoltage - alert: UpsWrongOutputVoltage
expr: (upsOutputVoltage < 220) or (upsOutputVoltage > 240) expr: (upsOutputVoltage < 220) or (upsOutputVoltage > 240)
for: 10m for: 10m
labels: labels:
severity: warning severity: warning
annotations: annotations:
summary: "La tension de sortie de {{ $labels.instance }} est de {{ $value }}V." summary: >-
{{ instance }} Tension de sortie de {{ raw("$value") }}V
- alert: UpsTimeRemainingWarning - alert: UpsTimeRemainingWarning
expr: upsEstimatedMinutesRemaining < 8 expr: upsEstimatedMinutesRemaining < 8
for: 1m for: 1m
labels: labels:
severity: warning severity: warning
annotations: annotations:
summary: "L'autonomie restante sur {{ $labels.instance }} est de {{ $value }} min." summary: >-
{{ instance }} Autonomie restante de {{ raw("$value") }} min
- alert: UpsTimeRemainingCritical - alert: UpsTimeRemainingCritical
expr: upsEstimatedMinutesRemaining < 5 expr: upsEstimatedMinutesRemaining < 5
for: 1m for: 1m
labels: labels:
severity: critical severity: critical
annotations: annotations:
summary: "L'autonomie restante sur {{ $labels.instance }} est de {{ $value }} min." summary: >-
{{ instance }} Autonomie restante de {{ raw("$value") }} min
...
{% endraw %}

4
roles/prometheus/templates/prometheus/prometheus.yml.j2
View file

@ -1,4 +1,5 @@
# {{ ansible_managed }} ---
{{ ansible_managed | comment }}
global: global:
# scrape_interval is set to the global default (60s) # scrape_interval is set to the global default (60s)
@ -100,3 +101,4 @@ scrape_configs:
file_sd_configs: file_sd_configs:
- files: - files:
- '/etc/prometheus/targets_docker.json' - '/etc/prometheus/targets_docker.json'
...