prometheus: cleanup bird alerts

group_vars/prom/prometheus/bird.yml

@@ -5,17 +5,21 @@ prometheus__scraping_bird:
     port: 9324
 
 prometheus__rules_bird:
-  - record: bird:protocol_up:bgp
+  - record: bird:protocol_up:bgp_all
     expr:
       label_replace(
-        bird_protocol_up{proto="BGP",}
+        bird_protocol_up{proto="BGP"},
-        unless bird_protocol_up{
-          proto="BGP",
-          name=~"^(viarezo|isp[12]|rezel)[46]$"
-        },
         "group", "$1",
         "instance", "^([^0-9\\.]+)-[0-9]+.*"
       )
+  # FIXME: sessions en cours d'installation, pas encore monitorées
+  - record: bird:protocol_up:bgp
+    expr:
+      bird:protocol_up:bgp_all
+      unless bird:protocol_up:bgp_all{
+        group="edge",
+        name=~"^(viarezo|isp[12]|rezel)[46]$"
+      }
   # Sessions qui ne sont volontairement pas redondées
   # au sein d'un groupe
   - record: bird:protocol_up:bgp:non_redundant
@@ -35,10 +39,10 @@ prometheus__rules_bird:
       (
         count by (group, name) (
           bird:protocol_up:bgp:redundant{state="Established"}
-        ) or (
+        ) + (
           count by (group, name) (
-            bird:protocol_up:bgp:redundant{state!="Established"}
+            bird:protocol_up:bgp:redundant{state!="Established"} * 0
-          ) * 0
+          )
         )
       ) < 2
     for: 0m
@@ -53,10 +57,10 @@ prometheus__rules_bird:
       (
         count by (group, name) (
           bird:protocol_up:bgp{state="Established"}
-        ) or (
+        ) + (
           count by (group, name) (
-            bird:protocol_up:bgp{state!="Established"}
+            bird:protocol_up:bgp{state!="Established"} * 0
-          ) * 0
+          )
         )
       ) == 0
     for: 0m
@@ -65,6 +69,7 @@ prometheus__rules_bird:
     annotations:
       Session: !unsafe "{{ $labels.name }}"
       Group: !unsafe "{{ $labels.group }}"
+  # TODO: warning pour redondant ?
   - alert: BirdBGPNoExportedPrefixRedundant
     expr:
       bird_protocol_prefix_export_count{

group_vars/prom/prometheus/common.yml

@@ -1,7 +1,7 @@
 ---
 prometheus__rules_common:
   - alert: CollectorDown
-    expr: >
+    expr:
       up == 0
     for: 3m
     labels:

group_vars/prom/prometheus/keepalived.yml

@@ -14,7 +14,7 @@ prometheus__rules_keepalived:
         keepalived_vrrp_state{
           keepalived_vvrp_state="master"
         }[1m]
-      ) > 1
+      ) > 0
     for: 0m
     labels:
       severity: warning

group_vars/prom/prometheus/quanta.yml

@@ -58,7 +58,8 @@ prometheus__rules_quanta:
   - alert: QuantaTemp
     expr:
       0.5 * snAgentTempValue > 45
-    for: 0m
+    for: 10m
+    keep_firing_for: 30m
     labels:
       severity: warning
     annotations:
@@ -67,7 +68,8 @@ prometheus__rules_quanta:
   - alert: QuantaTemp
     expr:
       0.5 * snAgentTempValue > 60
-    for: 0m
+    for: 10m
+    keep_firing_for: 30m
     labels:
       severity: critical
     annotations:

group_vars/prom/prometheus/switch.yml

@@ -44,4 +44,48 @@ prometheus__rules_switch:
       ErrorRate: !unsafe '{{ printf "%.2f" $value }} %'
       Interface: !unsafe "{{ $labels.ifName }}
         {{ if $labels.ifAlias }}- {{ $labels.ifAlias }}{{ end }}"
+  - alert: SwitchInLinkUsage
+    expr:
+      100 * rate(ifHCInOctets[5m]) / (ifHighSpeed * 1000000 / 8) > 50
+    for: 5m
+    keep_firing_for: 10m
+    labels:
+      severity: warning
+    annotations:
+      Usage: !unsafe '{{ printf "%.2f" $value }} %'
+      Interface: !unsafe "{{ $labels.ifName }}
+        {{ if $labels.ifAlias }}- {{ $labels.ifAlias }}{{ end }}"
+  - alert: SwitchInLinkUsage
+    expr:
+      100 * rate(ifHCInOctets[5m]) / (ifHighSpeed * 1000000 / 8) > 80
+    for: 5m
+    keep_firing_for: 10m
+    labels:
+      severity: critical
+    annotations:
+      Usage: !unsafe '{{ printf "%.2f" $value }} %'
+      Interface: !unsafe "{{ $labels.ifName }}
+        {{ if $labels.ifAlias }}- {{ $labels.ifAlias }}{{ end }}"
+  - alert: SwitchOutLinkUsage
+    expr:
+      100 * rate(ifHCOutOctets[5m]) / (ifHighSpeed * 1000000 / 8) > 50
+    for: 5m
+    keep_firing_for: 10m
+    labels:
+      severity: warning
+    annotations:
+      Usage: !unsafe '{{ printf "%.2f" $value }} %'
+      Interface: !unsafe "{{ $labels.ifName }}
+        {{ if $labels.ifAlias }}- {{ $labels.ifAlias }}{{ end }}"
+  - alert: SwitchOutLinkUsage
+    expr:
+      100 * rate(ifHCOutOctets[5m]) / (ifHighSpeed * 1000000 / 8) > 80
+    for: 5m
+    keep_firing_for: 10m
+    labels:
+      severity: warning
+    annotations:
+      Usage: !unsafe '{{ printf "%.2f" $value }} %'
+      Interface: !unsafe "{{ $labels.ifName }}
+        {{ if $labels.ifAlias }}- {{ $labels.ifAlias }}{{ end }}"
 ...

host_vars/ns-master.int.infra.auro.re/knotd.yml

@@ -420,6 +420,18 @@ knotd__zones:
       ff-1.core.sw:
         #- 2a09:6840:207::1:1
         - 10.207.1.1
+      ff-2.core.sw:
+        #- 2a09:6840:207::1:2
+        - 10.207.1.2
+      fl-1.core.sw:
+        #- 2a09:6840:207::1:3
+        - 10.207.1.3
+      fl-2.core.sw:
+        #- 2a09:6840:207::1:4
+        - 10.207.1.4
+      fd-1.core.sw:
+        #- 2a09:6840:207::1:5
+        - 10.207.1.5
       gk-1.core.sw:
         #- 2a09:6840:207::2:1
         - 10.207.2.1

hosts

@@ -5,6 +5,10 @@ eb-1.acs.sw.infra.auro.re
 
 [quanta]
 ff-1.core.sw.infra.auro.re
+ff-2.core.sw.infra.auro.re
+fl-1.core.sw.infra.auro.re
+fl-2.core.sw.infra.auro.re
+fd-1.core.sw.infra.auro.re
 gk-1.core.sw.infra.auro.re
 ec-1.core.sw.infra.auro.re
 r3-1.core.sw.infra.auro.re