323 lines
8.9 KiB
YAML
323 lines
8.9 KiB
YAML
---
|
|
prometheus__alertmanager_targets:
|
|
- docker-ovh.adm.auro.re:9093
|
|
|
|
prometheus__tsdb_retention_time: 90d
|
|
|
|
prometheus__scraping:
|
|
node:
|
|
targets: "{{ groups.vm_network + groups.pve_network }}"
|
|
address:
|
|
port: 9100
|
|
prometheus:
|
|
targets: "{{ groups.prom }}"
|
|
address:
|
|
port: 9090
|
|
kresd:
|
|
targets: "{{ groups.dns }}"
|
|
address:
|
|
port: 8453
|
|
bird:
|
|
targets: "{{ groups.router }}"
|
|
address:
|
|
port: 9324
|
|
quanta:
|
|
targets: "{{ groups.quanta }}"
|
|
address: 127.0.0.1:9116
|
|
path: /snmp
|
|
params:
|
|
module:
|
|
- quanta
|
|
snmp:
|
|
targets: "{{ groups.prom }}"
|
|
address:
|
|
port: 9116
|
|
|
|
prometheus__alert_rules_prometheus:
|
|
- alert: PrometheusTsdbCompactionFailed
|
|
expr: "increase(prometheus_tsdb_compactions_failed_total[1m]) > 0"
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
|
|
prometheus__alert_rules_common:
|
|
- alert: CollectorDown
|
|
expr: 'up == 0'
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Job: !unsafe "{{ $labels.job }}"
|
|
|
|
prometheus__alert_rules_node:
|
|
- alert: OutOfMemory
|
|
expr: "( node_memory_MemFree_bytes
|
|
+ node_memory_Cached_bytes
|
|
+ node_memory_Buffers_bytes )
|
|
/ node_memory_MemTotal_bytes * 100 < 10"
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
FreeMemory: !unsafe '{{ printf "%.0f" $value }} %'
|
|
- alert: HostSwapIsFillingUp
|
|
expr: "( 1 - ( node_memory_SwapFree_bytes
|
|
/ node_memory_SwapTotal_bytes ) )
|
|
* 100 >= 50"
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
UsedSwap: !unsafe '{{ printf "%.0f" $value }} %'
|
|
- alert: HostPhysicalComponentTooHot
|
|
expr: "node_hwmon_temp_celsius > 79"
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Temperature: !unsafe '{{ printf "%.0f" $value }} °C'
|
|
Chip: !unsafe "{{ $labels.chip }}"
|
|
Sensor: !unsafe "{{ $labels.sensor }}"
|
|
- alert: HostNodeOvertemperatureAlarm
|
|
expr: "node_hwmon_temp_crit_alarm_celsius == 1"
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Chip: !unsafe "{{ $labels.chip }}"
|
|
Sensor: !unsafe "{{ $labels.sensor }}"
|
|
- alert: HostRaidArrayGotInactive
|
|
expr: 'node_md_state{state="inactive"} > 0'
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Device: !unsafe "{{ $labels.device }}"
|
|
- alert: HostRaidDiskFailure
|
|
expr: 'node_md_disks{state="failed"} > 0'
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
severity: !unsafe "{{ $labels.md_device }}"
|
|
- alert: HostOomKillDetected
|
|
expr: "increase(node_vmstat_oom_kill[1m]) > 0"
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
PID: !unsafe "{{ $value }}"
|
|
- alert: HostEdacCorrectableErrorsDetected
|
|
expr: "increase(node_edac_correctable_errors_total[1m]) > 0"
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
CorrectedErrors: !unsafe "{{ $value }}"
|
|
- alert: HostEdacUncorrectableErrorsDetected
|
|
expr: "increase(node_edac_uncorrectable_errors_total[1m]) > 0"
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
DetectedErrors: !unsafe "{{ $value }}"
|
|
- alert: OutOfDiskSpace
|
|
expr: "( node_filesystem_free_bytes
|
|
/ node_filesystem_size_bytes * 100 < 10 )
|
|
and on (instance, device, mountpoint)
|
|
node_filesystem_readonly == 0"
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Mountpoint: !unsafe '{{ $labels.mountpoint }}'
|
|
FreeSpace: !unsafe '{{ printf "%.0f" $value }} %'
|
|
- alert: HostConntrackLimit
|
|
expr: "( node_nf_conntrack_entries
|
|
/ node_nf_conntrack_entries_limit ) * 100 > 80"
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Filled: !unsafe '{{ printf "%.0f" $value }} %'
|
|
- alert: HostClockSkew
|
|
expr: "(node_timex_offset_seconds > 0.05
|
|
and deriv(node_timex_offset_seconds[5m]) >= 0)
|
|
or (node_timex_offset_seconds < -0.05
|
|
and deriv(node_timex_offset_seconds[5m]) <= 0)"
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
- alert: HostClockNotSynchronising
|
|
expr: "min_over_time(node_timex_sync_status[1m]) == 0
|
|
and node_timex_maxerror_seconds >= 16"
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
- alert: HostRequiresReboot
|
|
expr: "node_reboot_required > 0"
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
- alert: OutOfInodes
|
|
expr: "node_filesystem_files_free
|
|
/ node_filesystem_files * 100 < 10"
|
|
for: 3m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Mountpoint: !unsafe "{{ $labels.mountpoint }}"
|
|
FreeInodes: !unsafe '{{ printf "%.0f" $value }} %'
|
|
- alert: CpuUsage
|
|
expr: '( 100 - avg by (instance)
|
|
( irate(node_cpu_seconds_total{mode="idle"}[5m]) )
|
|
* 100 ) > 75'
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Usage: !unsafe '{{ printf "%.0f" $value }} %'
|
|
- alert: SystemdServiceFailed
|
|
expr: 'node_systemd_unit_state{state="failed"} == 1'
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Service: !unsafe "{{ $labels.name }}"
|
|
- alert: LoadUsage
|
|
expr: "node_load1 > 5"
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Load1: !unsafe '{{ printf "%.0f" $value }}'
|
|
- alert: UnhealthyDisk
|
|
expr: "smartmon_device_smart_healthy < 1"
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Disk: !unsafe "{{ $labels.disk }}"
|
|
- alert: HostCpuStealNoisyNeighbor
|
|
expr: 'avg by(instance)
|
|
(rate(node_cpu_seconds_total{mode="steal"}[5m]))
|
|
* 100 > 10'
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Disk: !unsafe "{{ $labels.disk }}"
|
|
|
|
prometheus__alert_rules_keepalived:
|
|
- alert: KeepalivedVrrpFault
|
|
expr: 'keepalived_vrrp_state{state="fault"} > 0'
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Instance: !unsafe "{{ $labels.instance }}"
|
|
- alert: KeepalivedMasterChange
|
|
expr: 'changes(
|
|
keepalived_vrrp_state
|
|
{keepalived_vvrp_state="master"}[1m]) > 1'
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Instance: !unsafe "{{ $labels.instance }}"
|
|
|
|
prometheus__alert_rules_bird:
|
|
- alert: BirdProtocolDown
|
|
expr: "bird_protocol_up == 0"
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Protocol: !unsafe "{{ $labels.name }}"
|
|
State: !unsafe "{{ $labels.state }}"
|
|
|
|
prometheus__alert_rules_quanta:
|
|
- alert: QuantaQueueOverflow
|
|
expr: 'snAgGblQueueOverflow == 1'
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
- alert: QuantaCpuUsage
|
|
expr: 'snAgGblCpuUtil1MinAvg > 50'
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Usage: !unsafe "{{ $value }} %"
|
|
- alert: QuantaCpuUsage
|
|
expr: 'snAgGblCpuUtil1MinAvg > 80'
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Usage: !unsafe "{{ $value }} %"
|
|
- alert: QuantaMemoryUsage
|
|
expr: '100 * (1 - (snAgGblDynMemFree / snAgGblDynMemTotal)) > 50'
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
UsedMemory: !unsafe "{{ $value }} %"
|
|
- alert: QuantaMemoryUsage
|
|
expr: '100 * (1 - (snAgGblDynMemFree / snAgGblDynMemTotal)) > 80'
|
|
for: 5m
|
|
labels:
|
|
severity: alert
|
|
annotations:
|
|
UsedMemory: !unsafe "{{ $value }} %"
|
|
- alert: QuantaFanHealth
|
|
expr: 'snChasFanOperStatus{snChasFanOperStatus="normal"} == 0'
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Description: !unsafe "{{ $labels.shChasFanDescription }}"
|
|
Status: !unsafe "{{ $labels.snChasFanOperStatus }}"
|
|
- alert: QuantaTemp
|
|
expr: '(snAgentTempValue / 2) > 45'
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Temperature: !unsafe "{{ $value }} °C"
|
|
Description: !unsafe "{{ $labels.snAgentTempSensorDescr }}"
|
|
- alert: QuantaTemp
|
|
expr: '(snAgentTempValue / 2) > 60'
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
Temperature: !unsafe "{{ $value }} °C"
|
|
Description: !unsafe "{{ $labels.snAgentTempSensorDescr }}"
|
|
- alert: QuantaPowerRedundancyFailure
|
|
expr: 'count by (instance)
|
|
(snChasPwrSupplyOperStatus{snChasPwrSupplyOperStatus="normal"})
|
|
< 2'
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
|
|
prometheus__alert_rules_switch:
|
|
- alert: SwitchPromiscuousChange
|
|
expr: "changes(ifPromiscuousMode[5m]) > 0"
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
Interface: !unsafe "{{ $labels.ifName }}
|
|
{{ if $labels.ifAlias }}- {{ $labels.ifAlias }}{{ end }}"
|
|
|
|
prometheus__alert_rules:
|
|
common: "{{ prometheus__alert_rules_common }}"
|
|
switch: "{{ prometheus__alert_rules_switch }}"
|
|
prometheus: "{{ prometheus__alert_rules_prometheus }}"
|
|
node: "{{ prometheus__alert_rules_node }}"
|
|
keepalived: "{{ prometheus__alert_rules_keepalived }}"
|
|
quanta: "{{ prometheus__alert_rules_quanta }}"
|
|
#bird: "{{ prometheus__alert_rules_bird }}"
|
|
...
|