|
|
|
@ -7,29 +7,29 @@ prometheus__scraping_node:
|
|
|
|
|
prometheus__rules_node:
|
|
|
|
|
- alert: OutOfMemory
|
|
|
|
|
expr:
|
|
|
|
|
100 * (
|
|
|
|
|
(
|
|
|
|
|
node_memory_MemFree_bytes
|
|
|
|
|
+ node_memory_Cached_bytes
|
|
|
|
|
+ node_memory_Buffers_bytes
|
|
|
|
|
) / node_memory_MemTotal_bytes < 10
|
|
|
|
|
) / node_memory_MemTotal_bytes < 0.1
|
|
|
|
|
for: 5m
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
FreeMemory: !unsafe '{{ printf "%.0f" $value }} %'
|
|
|
|
|
FreeMemory: !unsafe "{{ $value | humanizePercentage }} %"
|
|
|
|
|
- alert: HostSwapIsFillingUp
|
|
|
|
|
expr:
|
|
|
|
|
100 * (
|
|
|
|
|
(
|
|
|
|
|
1 - (
|
|
|
|
|
node_memory_SwapFree_bytes
|
|
|
|
|
/ node_memory_SwapTotal_bytes
|
|
|
|
|
)
|
|
|
|
|
) >= 50
|
|
|
|
|
) >= 0.5
|
|
|
|
|
for: 3m
|
|
|
|
|
labels:
|
|
|
|
|
severity: critical
|
|
|
|
|
annotations:
|
|
|
|
|
UsedSwap: !unsafe '{{ printf "%.0f" $value }} %'
|
|
|
|
|
UsedSwap: !unsafe "{{ $value | humanizePercentage }} %"
|
|
|
|
|
- alert: HostPhysicalComponentTooHot
|
|
|
|
|
expr:
|
|
|
|
|
node_hwmon_temp_celsius > 79
|
|
|
|
@ -37,7 +37,7 @@ prometheus__rules_node:
|
|
|
|
|
labels:
|
|
|
|
|
severity: critical
|
|
|
|
|
annotations:
|
|
|
|
|
Temperature: !unsafe '{{ printf "%.0f" $value }} °C'
|
|
|
|
|
Temperature: !unsafe "{{ $value | humanize }} °C"
|
|
|
|
|
Chip: !unsafe "{{ $labels.chip }}"
|
|
|
|
|
Sensor: !unsafe "{{ $labels.sensor }}"
|
|
|
|
|
- alert: HostNodeOvertemperatureAlarm
|
|
|
|
@ -92,8 +92,8 @@ prometheus__rules_node:
|
|
|
|
|
- alert: OutOfDiskSpace
|
|
|
|
|
expr:
|
|
|
|
|
(
|
|
|
|
|
100 * node_filesystem_free_bytes
|
|
|
|
|
/ node_filesystem_size_bytes < 10
|
|
|
|
|
node_filesystem_free_bytes
|
|
|
|
|
/ node_filesystem_size_bytes < 0.1
|
|
|
|
|
)
|
|
|
|
|
and on (instance, device, mountpoint) (
|
|
|
|
|
node_filesystem_readonly
|
|
|
|
@ -102,19 +102,19 @@ prometheus__rules_node:
|
|
|
|
|
labels:
|
|
|
|
|
severity: critical
|
|
|
|
|
annotations:
|
|
|
|
|
Mountpoint: !unsafe '{{ $labels.mountpoint }}'
|
|
|
|
|
FreeSpace: !unsafe '{{ printf "%.0f" $value }} %'
|
|
|
|
|
Mountpoint: !unsafe "{{ $labels.mountpoint }}"
|
|
|
|
|
FreeSpace: !unsafe "{{ $value | humanizePercentage }} %"
|
|
|
|
|
- alert: HostConntrackLimit
|
|
|
|
|
expr:
|
|
|
|
|
100 * (
|
|
|
|
|
(
|
|
|
|
|
node_nf_conntrack_entries
|
|
|
|
|
/ node_nf_conntrack_entries_limit
|
|
|
|
|
) > 80
|
|
|
|
|
) > 0.8
|
|
|
|
|
for: 5m
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
Filled: !unsafe '{{ printf "%.0f" $value }} %'
|
|
|
|
|
Filled: !unsafe "{{ $value | humanizePercentage }} %"
|
|
|
|
|
- alert: HostClockSkew
|
|
|
|
|
expr:
|
|
|
|
|
(
|
|
|
|
@ -142,26 +142,26 @@ prometheus__rules_node:
|
|
|
|
|
severity: warning
|
|
|
|
|
- alert: OutOfInodes
|
|
|
|
|
expr:
|
|
|
|
|
100 * node_filesystem_files_free
|
|
|
|
|
/ node_filesystem_files < 10
|
|
|
|
|
node_filesystem_files_free
|
|
|
|
|
/ node_filesystem_files < 0.1
|
|
|
|
|
for: 3m
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
Mountpoint: !unsafe "{{ $labels.mountpoint }}"
|
|
|
|
|
FreeInodes: !unsafe '{{ printf "%.0f" $value }} %'
|
|
|
|
|
FreeInodes: !unsafe "{{ $value | humanizePercentage }} %"
|
|
|
|
|
- alert: CpuUsage
|
|
|
|
|
expr:
|
|
|
|
|
100 * (
|
|
|
|
|
(
|
|
|
|
|
1 - avg by (instance) (
|
|
|
|
|
irate(node_cpu_seconds_total{mode="idle"}[5m])
|
|
|
|
|
)
|
|
|
|
|
) > 75
|
|
|
|
|
) > 0.75
|
|
|
|
|
for: 10m
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
Usage: !unsafe '{{ printf "%.0f" $value }} %'
|
|
|
|
|
Usage: !unsafe "{{ $value | humanizePercentage }} %"
|
|
|
|
|
- alert: SystemdServiceFailed
|
|
|
|
|
expr:
|
|
|
|
|
node_systemd_unit_state{state="failed"} == 1
|
|
|
|
@ -177,7 +177,7 @@ prometheus__rules_node:
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
Load1: !unsafe '{{ printf "%.0f" $value }}'
|
|
|
|
|
Load1: !unsafe "{{ $value | humanize }}"
|
|
|
|
|
- alert: UnhealthyDisk
|
|
|
|
|
expr:
|
|
|
|
|
smartmon_device_smart_healthy < 1
|
|
|
|
@ -188,12 +188,13 @@ prometheus__rules_node:
|
|
|
|
|
Disk: !unsafe "{{ $labels.disk }}"
|
|
|
|
|
- alert: HostCpuStealNoisyNeighbor
|
|
|
|
|
expr:
|
|
|
|
|
100 * avg by (instance) (
|
|
|
|
|
avg by (instance) (
|
|
|
|
|
rate(node_cpu_seconds_total{mode="steal"}[5m])
|
|
|
|
|
) > 10
|
|
|
|
|
) > 0.1
|
|
|
|
|
for: 5m
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
Disk: !unsafe "{{ $labels.disk }}"
|
|
|
|
|
Steal: !unsafe "{{ $value | humanizePercentage }} %"
|
|
|
|
|
...
|
|
|
|
|