From 958eaa1bcb9eb66c3e06cfd8e585571921dc1a36 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Fri, 5 Mar 2021 00:54:44 +0100 Subject: [PATCH 01/13] Use label federated_instance instead of instance --- .../prometheus_federate/templates/prometheus/prometheus.yml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 b/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 index 52e5a92..ee7f4a6 100644 --- a/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 +++ b/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 @@ -31,6 +31,7 @@ scrape_configs: - job_name: federate scrape_interval: 15s metrics_path: '/federate' + honor_labels: true file_sd_configs: - files: - '/etc/prometheus/targets.json' @@ -39,7 +40,7 @@ scrape_configs: - source_labels: [__address__] target_label: __param_target - source_labels: [__param_target] - target_label: instance + target_label: federated_instance - source_labels: [__param_target] target_label: __address__ replacement: '$1:9090' From 802bfcc69828e702b092e23fbee7dc6263748d14 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 00:38:36 +0100 Subject: [PATCH 02/13] 'prometheus-federate' must not retrieve its own federated metrics --- monitoring.yml | 1 - .../templates/prometheus/django.rules.yml.j2 | 106 ----- .../templates/prometheus/snmp.yml.j2 | 387 ------------------ 3 files changed, 494 deletions(-) delete mode 100644 roles/prometheus_federate/templates/prometheus/django.rules.yml.j2 delete mode 100644 roles/prometheus_federate/templates/prometheus/snmp.yml.j2 diff --git a/monitoring.yml b/monitoring.yml index 53bdae7..f83c2e0 100755 --- a/monitoring.yml +++ b/monitoring.yml @@ -121,7 +121,6 @@ - prometheus-rives.adm.auro.re - prometheus-aurore.adm.auro.re - prometheus-ovh.adm.auro.re - - prometheus-federate.adm.auro.re roles: - prometheus_federate diff --git a/roles/prometheus_federate/templates/prometheus/django.rules.yml.j2 b/roles/prometheus_federate/templates/prometheus/django.rules.yml.j2 deleted file mode 100644 index fddd398..0000000 --- a/roles/prometheus_federate/templates/prometheus/django.rules.yml.j2 +++ /dev/null @@ -1,106 +0,0 @@ -# {{ ansible_managed }} -{# As this is also Jinja2 it will conflict without a raw block #} -{% raw %} -groups: -- name: django.rules - rules: - - record: job:django_http_requests_before_middlewares_total:sum_rate30s - expr: sum(rate(django_http_requests_before_middlewares_total[30s])) BY (job) - - record: job:django_http_requests_unknown_latency_total:sum_rate30s - expr: sum(rate(django_http_requests_unknown_latency_total[30s])) BY (job) - - record: job:django_http_ajax_requests_total:sum_rate30s - expr: sum(rate(django_http_ajax_requests_total[30s])) BY (job) - - record: job:django_http_responses_before_middlewares_total:sum_rate30s - expr: sum(rate(django_http_responses_before_middlewares_total[30s])) BY (job) - - record: job:django_http_requests_unknown_latency_including_middlewares_total:sum_rate30s - expr: sum(rate(django_http_requests_unknown_latency_including_middlewares_total[30s])) - BY (job) - - record: job:django_http_requests_body_total_bytes:sum_rate30s - expr: sum(rate(django_http_requests_body_total_bytes[30s])) BY (job) - - record: job:django_http_responses_streaming_total:sum_rate30s - expr: sum(rate(django_http_responses_streaming_total[30s])) BY (job) - - record: job:django_http_responses_body_total_bytes:sum_rate30s - expr: sum(rate(django_http_responses_body_total_bytes[30s])) BY (job) - - record: job:django_http_requests_total:sum_rate30s - expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job) - - record: job:django_http_requests_total_by_method:sum_rate30s - expr: sum(rate(django_http_requests_total_by_method[30s])) BY (job, method) - - record: job:django_http_requests_total_by_transport:sum_rate30s - expr: sum(rate(django_http_requests_total_by_transport[30s])) BY (job, transport) - - record: job:django_http_requests_total_by_view:sum_rate30s - expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job, - view) - - record: job:django_http_requests_total_by_view_transport_method:sum_rate30s - expr: sum(rate(django_http_requests_total_by_view_transport_method[30s])) BY (job, - view, transport, method) - - record: job:django_http_responses_total_by_templatename:sum_rate30s - expr: sum(rate(django_http_responses_total_by_templatename[30s])) BY (job, templatename) - - record: job:django_http_responses_total_by_status:sum_rate30s - expr: sum(rate(django_http_responses_total_by_status[30s])) BY (job, status) - - record: job:django_http_responses_total_by_charset:sum_rate30s - expr: sum(rate(django_http_responses_total_by_charset[30s])) BY (job, charset) - - record: job:django_http_exceptions_total_by_type:sum_rate30s - expr: sum(rate(django_http_exceptions_total_by_type[30s])) BY (job, type) - - record: job:django_http_exceptions_total_by_view:sum_rate30s - expr: sum(rate(django_http_exceptions_total_by_view[30s])) BY (job, view) - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "50" - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "95" - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99" - - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s - expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99.9" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "50" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "95" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99" - - record: job:django_http_requests_latency_seconds:quantile_rate30s - expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_seconds_bucket[30s])) - BY (job, le)) - labels: - quantile: "99.9" - - record: job:django_model_inserts_total:sum_rate1m - expr: sum(rate(django_model_inserts_total[1m])) BY (job, model) - - record: job:django_model_updates_total:sum_rate1m - expr: sum(rate(django_model_updates_total[1m])) BY (job, model) - - record: job:django_model_deletes_total:sum_rate1m - expr: sum(rate(django_model_deletes_total[1m])) BY (job, model) - - record: job:django_db_new_connections_total:sum_rate30s - expr: sum(rate(django_db_new_connections_total[30s])) BY (alias, vendor) - - record: job:django_db_new_connection_errors_total:sum_rate30s - expr: sum(rate(django_db_new_connection_errors_total[30s])) BY (alias, vendor) - - record: job:django_db_execute_total:sum_rate30s - expr: sum(rate(django_db_execute_total[30s])) BY (alias, vendor) - - record: job:django_db_execute_many_total:sum_rate30s - expr: sum(rate(django_db_execute_many_total[30s])) BY (alias, vendor) - - record: job:django_db_errors_total:sum_rate30s - expr: sum(rate(django_db_errors_total[30s])) BY (alias, vendor, type) - - record: job:django_migrations_applied_total:max - expr: max(django_migrations_applied_total) BY (job, connection) - - record: job:django_migrations_unapplied_total:max - expr: max(django_migrations_unapplied_total) BY (job, connection) -{% endraw %} diff --git a/roles/prometheus_federate/templates/prometheus/snmp.yml.j2 b/roles/prometheus_federate/templates/prometheus/snmp.yml.j2 deleted file mode 100644 index d4dc51c..0000000 --- a/roles/prometheus_federate/templates/prometheus/snmp.yml.j2 +++ /dev/null @@ -1,387 +0,0 @@ -# {{ ansible_managed }} -# TODOlist : -# - Faire fonctionner le monitoring des switchs défini ici -# * Configurer tous les switchs avec un compte SNMPv3 -# * Mettre l'inventaire des switchs dans Ansible -# - Optimiser les règles pour les bornes Unifi, -# on pourrait indexer avec les SSID - -eatonups: - walk: - - 1.3.6.1.2.1.33.1.2 - - 1.3.6.1.2.1.33.1.3 - - 1.3.6.1.2.1.33.1.4 - - 1.3.6.1.4.1.534.1.6 - get: - - 1.3.6.1.2.1.1.3.0 - metrics: - - name: sysUpTime - oid: 1.3.6.1.2.1.1.3 - type: gauge - help: The time (in hundredths of a second) since the network management portion - of the system was last re-initialized. - 1.3.6.1.2.1.1.3 - - name: upsBatteryStatus - oid: 1.3.6.1.2.1.33.1.2.1 - type: gauge - help: The indication of the capacity remaining in the UPS system's batteries - - 1.3.6.1.2.1.33.1.2.1 - - name: upsEstimatedMinutesRemaining - oid: 1.3.6.1.2.1.33.1.2.3 - type: gauge - help: An estimate of the time to battery charge depletion under the present load - conditions if the utility power is off and remains off, or if it were to be - lost and remain off. - 1.3.6.1.2.1.33.1.2.3 - - name: upsInputVoltage - oid: 1.3.6.1.2.1.33.1.3.3.1.3 - type: gauge - help: The magnitude of the present input voltage. - 1.3.6.1.2.1.33.1.3.3.1.3 - indexes: - - labelname: upsInputLineIndex - type: gauge - - name: upsOutputSource - oid: 1.3.6.1.2.1.33.1.4.1 - type: gauge - help: The present source of output power - 1.3.6.1.2.1.33.1.4.1 - - name: upsOutputVoltage - oid: 1.3.6.1.2.1.33.1.4.4.1.2 - type: gauge - help: The present output voltage. - 1.3.6.1.2.1.33.1.4.4.1.2 - indexes: - - labelname: upsOutputLineIndex - type: gauge - - name: upsOutputPower - oid: 1.3.6.1.2.1.33.1.4.4.1.4 - type: gauge - help: The present output true power. - 1.3.6.1.2.1.33.1.4.4.1.4 - indexes: - - labelname: upsOutputLineIndex - type: gauge - - name: upsOutputPercentLoad - oid: 1.3.6.1.2.1.33.1.4.4.1.5 - type: gauge - help: The percentage of the UPS power capacity presently being used on this output - line, i.e., the greater of the percent load of true power capacity and the percent - load of VA. - 1.3.6.1.2.1.33.1.4.4.1.5 - indexes: - - labelname: upsOutputLineIndex - type: gauge - - name: xupsEnvRemoteTemp - oid: 1.3.6.1.4.1.534.1.6.5 - type: gauge - help: The reading of an EMP's temperature sensor. - 1.3.6.1.4.1.534.1.6.5 - - name: xupsEnvRemoteHumidity - oid: 1.3.6.1.4.1.534.1.6.6 - type: gauge - help: The reading of an EMP's humidity sensor. - 1.3.6.1.4.1.534.1.6.6 - version: 1 - auth: - community: public - - -procurve_switch: - walk: - - 1.3.6.1.2.1.31.1.1.1.10 - - 1.3.6.1.2.1.31.1.1.1.6 - get: - - 1.3.6.1.2.1.1.3.0 - - 1.3.6.1.2.1.1.5.0 - - 1.3.6.1.2.1.1.6.0 - metrics: - - name: sysUpTime - oid: 1.3.6.1.2.1.1.3 - type: gauge - help: The time (in hundredths of a second) since the network management portion - of the system was last re-initialized. - 1.3.6.1.2.1.1.3 - - name: sysName - oid: 1.3.6.1.2.1.1.5 - type: DisplayString - help: An administratively-assigned name for this managed node - 1.3.6.1.2.1.1.5 - - name: sysLocation - oid: 1.3.6.1.2.1.1.6 - type: DisplayString - help: The physical location of this node (e.g., 'telephone closet, 3rd floor') - - 1.3.6.1.2.1.1.6 - - name: ifHCOutOctets - oid: 1.3.6.1.2.1.31.1.1.1.10 - type: counter - help: The total number of octets transmitted out of the interface, including framing - characters - 1.3.6.1.2.1.31.1.1.1.10 - indexes: - - labelname: ifIndex - type: gauge - - name: ifHCInOctets - oid: 1.3.6.1.2.1.31.1.1.1.6 - type: counter - help: The total number of octets received on the interface, including framing - characters - 1.3.6.1.2.1.31.1.1.1.6 - indexes: - - labelname: ifIndex - type: gauge - version: 3 - auth: - username: prometheus - -ubiquiti_unifi: - walk: - - 1.3.6.1.4.1.41112.1.6 - get: - - 1.3.6.1.2.1.1.5.0 - - 1.3.6.1.2.1.1.6.0 - metrics: -# Pour faire une WifiMap un jour, on peut entrer la location dans la conf des bornes -# - name: sysLocation -# oid: 1.3.6.1.2.1.1.6 -# type: DisplayString -# help: The physical location of this node (e.g., 'telephone closet, 3rd floor') -# - 1.3.6.1.2.1.1.6 - - name: unifiVapIndex - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.1 - type: gauge - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.1' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapChannel - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.4 - type: gauge - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.4' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapEssId - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.6 - type: DisplayString - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.6' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapName - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.7 - type: DisplayString - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.7' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifi_vap_num_stations - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.8 - type: gauge - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.8' - indexes: - - labelname: unifi_vap_index - type: gauge - lookups: - - labels: [unifi_vap_index] - labelname: unifi_vap_essid - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.6 - type: DisplayString - - labels: [unifi_vap_index] - labelname: unifi_vap_radio - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.9 - type: DisplayString - - labels: [] - labelname: unifi_vap_index -# - name: unifiVapNumStations -# oid: 1.3.6.1.4.1.41112.1.6.1.2.1.8 -# type: gauge -# help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.8' -# indexes: -# - labelname: unifiVapIndex -# type: gauge - - name: unifiVapRadio - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.9 - type: DisplayString - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.9' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapRxBytes - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.10 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.10' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapRxCrypts - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.11 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.11' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapRxDropped - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.12 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.12' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapRxErrors - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.13 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.13' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapRxFrags - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.14 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.14' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapRxPackets - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.15 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.15' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapTxBytes - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.16 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.16' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapTxDropped - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.17 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.17' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapTxErrors - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.18 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.18' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapTxPackets - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.19 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.19' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapTxRetries - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.20 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.20' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapTxPower - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.21 - type: gauge - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.21' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapUp - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.22 - type: gauge - help: ' - 1.3.6.1.4.1.41112.1.6.1.2.1.22' - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiVapUsage - oid: 1.3.6.1.4.1.41112.1.6.1.2.1.23 - type: DisplayString - help: guest or regular user - 1.3.6.1.4.1.41112.1.6.1.2.1.23 - indexes: - - labelname: unifiVapIndex - type: gauge - - name: unifiIfIndex - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.1 - type: gauge - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.1' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfName - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.5 - type: DisplayString - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.5' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfRxBytes - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.6 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.6' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfRxDropped - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.7 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.7' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfRxError - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.8 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.8' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfRxMulticast - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.9 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.9' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfRxPackets - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.10 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.10' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfTxBytes - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.12 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.12' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfTxDropped - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.13 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.13' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfTxError - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.14 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.14' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiIfTxPackets - oid: 1.3.6.1.4.1.41112.1.6.2.1.1.15 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.2.1.1.15' - indexes: - - labelname: unifiIfIndex - type: gauge - - name: unifiApSystemModel - oid: 1.3.6.1.4.1.41112.1.6.3.3 - type: DisplayString - help: ' - 1.3.6.1.4.1.41112.1.6.3.3' - - name: unifiApSystemUptime - oid: 1.3.6.1.4.1.41112.1.6.3.5 - type: counter - help: ' - 1.3.6.1.4.1.41112.1.6.3.5' - version: 3 - auth: - security_level: authPriv - username: snmp_prometheus - password: {{ snmp_unifi_password }} - auth_protocol: SHA - priv_protocol: AES - priv_password: {{ snmp_unifi_password }} From 4ca7ebd1448979147511045e30d11ebce15f8ca3 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 00:40:44 +0100 Subject: [PATCH 03/13] Add a unique exported label (useful for federation) --- roles/prometheus/templates/prometheus/prometheus.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/prometheus/templates/prometheus/prometheus.yml.j2 b/roles/prometheus/templates/prometheus/prometheus.yml.j2 index 75c8be9..937f64f 100644 --- a/roles/prometheus/templates/prometheus/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus/prometheus.yml.j2 @@ -8,7 +8,7 @@ global: # Attach these labels to any time series or alerts when communicating with # external systems (federation, remote storage, Alertmanager). external_labels: - monitor: 'example' + federated_instance: '{{ inventory_hostname }}' # Alertmanager configuration # Use prometheus alertmanager installed on the same machine From 32669e1fb1b6a8a6dee6430a8d8cbc6922392c16 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 00:44:22 +0100 Subject: [PATCH 04/13] Don't load Django rules prometheus-federate --- roles/prometheus_federate/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/prometheus_federate/tasks/main.yml b/roles/prometheus_federate/tasks/main.yml index 33feb90..f168f1f 100644 --- a/roles/prometheus_federate/tasks/main.yml +++ b/roles/prometheus_federate/tasks/main.yml @@ -23,12 +23,11 @@ notify: Restart Prometheus loop: - alert.rules.yml - - django.rules.yml # We don't need to restart Prometheus when updating nodes - name: Configure Prometheus Federate devices copy: - content: "{{ [{'targets': prometheus_targets }] | to_nice_json }}" + content: "{{ [{'targets': prometheus_targets }] | to_nice_json }}" dest: /etc/prometheus/targets.json mode: 0644 when: prometheus_targets is defined @@ -44,3 +43,4 @@ src: update-motd.d/05-service.j2 dest: /etc/update-motd.d/05-prometheus mode: 0755 +... From 7d527be1c0db243d81c97d255e6c4c465531a994 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 00:45:43 +0100 Subject: [PATCH 05/13] Remove duplicate alerts from 'prometheus-federate' --- .../templates/prometheus/alert.rules.yml.j2 | 152 ++---------------- 1 file changed, 15 insertions(+), 137 deletions(-) diff --git a/roles/prometheus_federate/templates/prometheus/alert.rules.yml.j2 b/roles/prometheus_federate/templates/prometheus/alert.rules.yml.j2 index f78df48..d30511f 100644 --- a/roles/prometheus_federate/templates/prometheus/alert.rules.yml.j2 +++ b/roles/prometheus_federate/templates/prometheus/alert.rules.yml.j2 @@ -1,138 +1,16 @@ -# {{ ansible_managed }} -{# As this is also Jinja2 it will conflict without a raw block #} -{# Depending of Prometheus Node exporter version, rules can change depending of version #} -{% raw %} +--- +{{ ansible_managed | comment }} + groups: -- name: alert.rules - rules: - - # Alert for any instance that is unreachable for >3 minutes. - - alert: InstanceDown - expr: up == 0 - for: 3m - labels: - severity: critical - annotations: - summary: "Federate : {{ $labels.exported_instance }} est invisible depuis plus de 3 minutes !" - - # Alert for out of memory - - alert: OutOfMemory - expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 < 10 - for: 5m - labels: - severity: warning - annotations: - summary: "Federate : Mémoire libre de {{ $labels.exported_instance }} à {{ humanize $value }}%." - - # Alert for out of disk space - - alert: OutOfDiskSpace - expr: node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10 - for: 5m - labels: - severity: warning - annotations: - summary: "Espace libre de {{ $labels.mountpoint }} sur {{ $labels.exported_instance }} à {{ humanize $value }}%." - - # Alert for out of inode space on disk - - alert: OutOfInodes - expr: node_filesystem_files_free{fstype="ext4"} / node_filesystem_files{fstype="ext4"} * 100 < 10 - for: 5m - labels: - severity: warning - annotations: - summary: "Federate : Presque plus d'inodes disponibles ({{ $value }}% restant) dans {{ $labels.mountpoint }} sur {{ $labels.exported_instance }}." - - # Alert for high CPU usage - - alert: CpuUsage - expr: (100 - avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 75 - for: 10m - labels: - severity: warning - annotations: - summary: "Federate : CPU sur {{ $labels.exported_instance }} à {{ humanize $value }}%." - - # Check systemd unit (> buster) - - alert: SystemdServiceFailed - expr: node_systemd_unit_state{state="failed"} == 1 - for: 10m - labels: - severity: warning - annotations: - summary: "Federate : {{ $labels.name }} a échoué sur {{ $labels.exported_instance }}" - - # Check load of instance - - alert: LoadUsage - expr: node_load1 > 5 - for: 2m - labels: - severity: warning - annotations: - summary: "Federate : la charge de {{ $labels.exported_instance }} est à {{ $value }} !" - - # Check UPS - - alert: UpsOutputSourceChanged - expr: upsOutputSource != 3 - for: 1m - labels: - severity: warning - annotations: - summary: "Federate : La source d'alimentation de {{ $labels.exported_instance }} a changé !" - - - alert: UpsBatteryStatusWarning - expr: upsBatteryStatus == 3 - for: 2m - labels: - severity: warning - annotations: - summary: "Federate : L'état de la batterie de {{ $labels.exported_instance }} est faible !" - - - alert: UpsBatteryStatusCritical - expr: upsBatteryStatus == 4 - for: 10m - labels: - severity: warning - annotations: - summary: "L'état de la batterie de {{ $labels.exported_instance }} est affaibli !" - - - alert: UpsHighLoad - expr: upsOutputPercentLoad > 70 - for: 5m - labels: - severity: critical - annotations: - summary: "Federate : La charge de {{ $labels.exported_instance }} est de {{ $value }}% !" - - - alert: UpsWrongInputVoltage - expr: (upsInputVoltage < 210) or (upsInputVoltage > 250) - for: 10m - labels: - severity: warning - annotations: - summary: "Federate : La tension d'entrée de {{ $labels.exported_instance }} est de {{ $value }}V." - - - alert: UpsWrongOutputVoltage - expr: (upsOutputVoltage < 220) or (upsOutputVoltage > 240) - for: 10m - labels: - severity: warning - annotations: - summary: "Federate : La tension de sortie de {{ $labels.exported_instance }} est de {{ $value }}V." - - - alert: UpsTimeRemainingWarning - expr: upsEstimatedMinutesRemaining < 15 - for: 1m - labels: - severity: warning - annotations: - summary: "Federate : L'autonomie restante sur {{ $labels.exported_instance }} est de {{ $value }} min." - - - alert: UpsTimeRemainingCritical - expr: upsEstimatedMinutesRemaining < 5 - for: 1m - labels: - severity: critical - annotations: - summary: "Federate : L'autonomie restante sur {{ $labels.exported_instance }} est de {{ $value }} min." - - -{% endraw %} + - name: alert.rules + rules: + - alert: FederateInstanceDown + expr: up{job="federate"} == 0 + for: 3m + labels: + severity: critical + annotations: + summary: >- + Federate : {{ "{{" }} $labels.instance {{ "}}" }} est invisible + depuis plus de 3 minutes ! +... From 52124d2cad39bde043aab51051c5375540e3e6c6 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 00:46:13 +0100 Subject: [PATCH 06/13] Cleanup prometheus_federate's prometheus.yml.j2 --- .../templates/prometheus/prometheus.yml.j2 | 36 ++++++------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 b/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 index ee7f4a6..5d9a31a 100644 --- a/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 +++ b/roles/prometheus_federate/templates/prometheus/prometheus.yml.j2 @@ -1,51 +1,35 @@ -# {{ ansible_managed }} +--- +{{ ansible_managed | comment }} global: - # scrape_interval is set to the global default (60s) - # evaluation_interval is set to the global default (60s) - # scrape_timeout is set to the global default (10s). - - # Attach these labels to any time series or alerts when communicating with - # external systems (federation, remote storage, Alertmanager). external_labels: - monitor: 'example' + monitor: '{{ ansible_fqdn }}' -# Alertmanager configuration -# Use prometheus alertmanager installed on the same machine alerting: alertmanagers: - - static_configs: - - targets: ['{{ prometheus_alertmanager }}'] + - static_configs: + - targets: ['{{ prometheus_alertmanager }}'] -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: - - "alert.rules.yml" # Monitoring alerts, this is the file you may be searching! - - "django.rules.yml" # Custom rules specific for Django project monitoring + - 'alert.rules.yml' -# A scrape configuration containing exactly one endpoint to scrape: -# Here it's Prometheus itself. scrape_configs: - # The .json in file_sd_configs is dynamically reloaded - - - job_name: federate - scrape_interval: 15s + scrape_interval: 30s metrics_path: '/federate' honor_labels: true + honor_timestamps: true file_sd_configs: - files: - '/etc/prometheus/targets.json' relabel_configs: - # Do not put :9100 in instance name, rather here - source_labels: [__address__] target_label: __param_target - - source_labels: [__param_target] - target_label: federated_instance - source_labels: [__param_target] target_label: __address__ replacement: '$1:9090' params: - 'match[]': + match[]: - '{job="servers"}' - '{job="prometheus"}' - '{job="unifi_snmp"}' @@ -54,4 +38,4 @@ scrape_configs: - '{job="django"}' - '{job="docker"}' - '{job="switch"}' - +... From 21fed6ae3f0efaaf1decf0f2fcbff424389cab76 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 00:55:27 +0100 Subject: [PATCH 07/13] Add useful lookups for switchs interfaces --- .../templates/prometheus/snmp.yml.j2 | 105 +++++++++++------- 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/roles/prometheus/templates/prometheus/snmp.yml.j2 b/roles/prometheus/templates/prometheus/snmp.yml.j2 index d4dc51c..dc2addf 100644 --- a/roles/prometheus/templates/prometheus/snmp.yml.j2 +++ b/roles/prometheus/templates/prometheus/snmp.yml.j2 @@ -1,4 +1,6 @@ -# {{ ansible_managed }} +--- +{{ ansible_managed | comment }} + # TODOlist : # - Faire fonctionner le monitoring des switchs défini ici # * Configurer tous les switchs avec un compte SNMPv3 @@ -77,49 +79,73 @@ eatonups: auth: community: public - procurve_switch: walk: - - 1.3.6.1.2.1.31.1.1.1.10 - - 1.3.6.1.2.1.31.1.1.1.6 + - 1.3.6.1.2.1.31.1.1.1.10 + - 1.3.6.1.2.1.31.1.1.1 + - 1.3.6.1.2.1.2.2.1.2 + - 1.3.6.1.2.1.31.1.1.1.18 get: - - 1.3.6.1.2.1.1.3.0 - - 1.3.6.1.2.1.1.5.0 - - 1.3.6.1.2.1.1.6.0 + - 1.3.6.1.2.1.1.3.0 + - 1.3.6.1.2.1.1.5.0 + - 1.3.6.1.2.1.1.6.0 metrics: - - name: sysUpTime - oid: 1.3.6.1.2.1.1.3 - type: gauge - help: The time (in hundredths of a second) since the network management portion - of the system was last re-initialized. - 1.3.6.1.2.1.1.3 - - name: sysName - oid: 1.3.6.1.2.1.1.5 - type: DisplayString - help: An administratively-assigned name for this managed node - 1.3.6.1.2.1.1.5 - - name: sysLocation - oid: 1.3.6.1.2.1.1.6 - type: DisplayString - help: The physical location of this node (e.g., 'telephone closet, 3rd floor') - - 1.3.6.1.2.1.1.6 - - name: ifHCOutOctets - oid: 1.3.6.1.2.1.31.1.1.1.10 - type: counter - help: The total number of octets transmitted out of the interface, including framing - characters - 1.3.6.1.2.1.31.1.1.1.10 - indexes: - - labelname: ifIndex + - name: sysUpTime + oid: 1.3.6.1.2.1.1.3 type: gauge - - name: ifHCInOctets - oid: 1.3.6.1.2.1.31.1.1.1.6 - type: counter - help: The total number of octets received on the interface, including framing - characters - 1.3.6.1.2.1.31.1.1.1.6 - indexes: - - labelname: ifIndex - type: gauge - version: 3 - auth: - username: prometheus + help: The time (in hundredths of a second) since the network management + portion of the system was last re-initialized. - 1.3.6.1.2.1.1.3 + - name: sysName + oid: 1.3.6.1.2.1.1.5 + type: DisplayString + help: An administratively-assigned name for this managed node + - 1.3.6.1.2.1.1.5 + - name: sysLocation + oid: 1.3.6.1.2.1.1.6 + type: DisplayString + help: The physical location of this node (e.g., 'telephone closet, 3rd + floor') - 1.3.6.1.2.1.1.6 + - name: ifHCOutOctets + oid: 1.3.6.1.2.1.31.1.1.1.10 + type: counter + help: The total number of octets transmitted out of the interface, + including framing characters - 1.3.6.1.2.1.31.1.1.1.10 + indexes: + - labelname: ifIndex + type: gauge + lookups: + - labels: + - ifIndex + labelname: ifDescr + oid: 1.3.6.1.2.1.2.2.1.2 + type: DisplayString + - labels: + - ifIndex + labelname: ifName + oid: 1.3.6.1.2.1.31.1.1.1.1 + type: DisplayString + - name: ifHCInOctets + oid: 1.3.6.1.2.1.31.1.1.1.6 + type: counter + help: The total number of octets received on the interface, including + framing characters - 1.3.6.1.2.1.31.1.1.1.6 + indexes: + - labelname: ifIndex + type: gauge + lookups: + - labels: + - ifIndex + labelname: ifDescr + oid: 1.3.6.1.2.1.2.2.1.2 + type: DisplayString + - labels: + - ifIndex + labelname: ifName + oid: 1.3.6.1.2.1.31.1.1.1.1 + type: DisplayString + version: 2 + auth: + community: "{{ snmp_switch_community }}" ubiquiti_unifi: walk: @@ -385,3 +411,4 @@ ubiquiti_unifi: auth_protocol: SHA priv_protocol: AES priv_password: {{ snmp_unifi_password }} +... From 8ae94fa8f8d6b3c371865a9bef7340ce8d1f5fe1 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 01:08:51 +0100 Subject: [PATCH 08/13] Rename vault_snmp_switch{s,}_community --- group_vars/all/vault.yml | 358 ++++++++++++++++++++------------------- 1 file changed, 180 insertions(+), 178 deletions(-) diff --git a/group_vars/all/vault.yml b/group_vars/all/vault.yml index 3666f5b..1a6af68 100644 --- a/group_vars/all/vault.yml +++ b/group_vars/all/vault.yml @@ -1,179 +1,181 @@ $ANSIBLE_VAULT;1.1;AES256 -32313562646230353138303964366135656361616532343933353732313961323339653964353130 -3938346666633565356134343835633964626261363365370a663664663938383731343733386136 -33356531323762313463326339333963336636353933326537333665313334616563626632336663 -6537363033663935660a613366613962626563643035663330343061353836646561623031323236 -65313633383063373064613930623530656365396335663363643330636239643937373163623932 -61373136303737333739316565323934376433316362353935363637373264616238373831666438 -35343135383233653963333237393232353631636566373766366664656666313436323535393736 -62323731343261373331393062633030356235313834373861323138663930613332643432386436 -38383038616536316465343561643639353434396631643033633537393265646532613161343732 -32363265643963386538326639353233363438643833306637336431303533396562613863633537 -30303334643137313136633039393463346562306236353566333563633238313865313534326137 -33623036376439653532313833633135326631643361333463633162303065623633636331666661 -62303636653233666164383463356530633464306564383236373832616263653165373937303030 -31323865656436366265303537306438303434613135396166313635656566373539303463393830 -65383636363064333730623161316162373734626433346564333835393030616437636665316566 -37353937626465383439633534316336313931663561336335653761396230393031393839336264 -37623037663032646631656637386366333131356562376665333964393264643133626532653564 -32353235633434656334663233303664613865343039613330663833396162646430623735653434 -66633466306338373061326636366330643639383632353564353865623637303832306332653131 -37343566393965326635613135613134316264616336303233616162313839626235386137343435 -33633336636434343531633362633834376135303337363637303039323038313937646236366265 -34303434373566313730623664653263653466366133363562333736393836393363326665353434 -30333263323366326436623238353335323936346637646130623265366535653737343665373165 -63336166633831623464343862353065653162613934646539396364353162633063303332313266 -65656163396463363737663931353765376337643065646131303264363961366336343432653537 -65306437623535393132343962333666366665316362366536663431646435633166333731303232 -63313337353334623330623862386661306333366638306433373437623835636631376231373636 -66666539363561313166396438343730656230663532633031353336636565343964366136663466 -38316364663936303231633633613832313163646262313238346666336661613236343966353130 -62656237663865306632333130653933633332623061633062363964643130383430613864663935 -63663765356434626661346165653163626565336437613539653536306432376332616430393737 -34366139336363383761366338623236383135373634613239616665343061396633383231663230 -63653331336366666234626662356461663263626465663036326162343239373734346661626665 -61666231613565356633343030343935393135653261376239303037373634386138393463363239 -30356365663133646634333863616230646235656135336330393836353462323630376537366334 -31306330363232326661616666623131383837353139643838326430653561346565393762323936 -31623136656361383039653763613162356530653933376539336130376237396661663664393733 -36396433303339613965316230613237303331646331383239356638333366653961303138343663 -33393664303637333863313364356666383836633063643539333262633565623534323866316537 -38623630363139643837396330353463303932383231663831363763656537386531383531303165 -37366338343063346230656461393832383736636662656666636434363731623437303862636366 -33613333393139613637623963373262323637653531336265333033333135613330313166633738 -36353935383931363535656539333130653164613431616438613432313532373063353738656162 -36616563383133623336396633343762376537663432356238653766666636323232623065313537 -39636632326166323130646633626431323831373963313837613465356436326430616433303662 -65343834663937306539663330366538643265626665613631323036616463313266303237613938 -30613565306636306561643238326138623366343365303934306561623234313332636462383363 -30623432326336396364636164366463326533613665333830656564626663383331323661663934 -35353135323930656138373830623932396138626335343265623738383532333861306561323430 -66333532333961636463656535636132323535313730333762633139306235373031363831363266 -33646635316137616663653461393566303432386330623936633330373461333762356532663062 -39666437363931313861356331653932303132353364623664656364316430653933653935616230 -38376631316463646663626562366233626334323235633235653364623936643131356130343261 -36396535393335366532313930623363663032386635396262363430303466373737633739626435 -30636136396562336561393936353763383732653166353266376165663233626266353638363131 -65323462633039323334613566373434343363633532656534663635363763396265663137636331 -38613736353635613437663133616431396666316230393066343431336535626335373437393039 -63666135353937313765316134326338376161353862373161653039333631306264343464353035 -65353639313134346239646362663836643734373465353866373238613162303336306438376237 -35363934333536376136666561333636653136316435316530366461306636333063313739626630 -37633333333766613663636466373364663132613266343136376138663461383832356631303132 -30363434336161393962363636313364663839383734373533356663343733333731613535646433 -64396361643736653931336365313338313633383038306131333863306437386362633263646364 -36656566326333333136636566613066623362363263373435356162396431396334386237383231 -30326465646334613235666435613462633230353434653666336364646466613066346366376262 -66633863333461626631383961663930383663666538613162643730323565653732386330613538 -38666164353130386530376332643637333931313661633634303636643639613561643338373331 -63333932306634313933366533623837613934366334396637396361623439383964333665383435 -62316265356537616137643537366666336634393935613034393737313930333364323031653234 -37366561356332666439623462396266623961653039626562393065393336643962373064343563 -36346665666338623931343739386531343833386135356164303532643463346565316163656633 -32616365623065626139383362613466633332666133313263393062373338653834363830333039 -62626230343362393533633061663432363836616539643065643839623065633363393134643534 -63343935376537393739333063333333386239663763383435633234376434366362616433363162 -34363539633661633333306133363433313761303138363864373266333461303139613362663937 -39626332356139396330393361613364643363366164376234316266316164393035386334366362 -36373065626530333237636139336163623766623561656234333239646263626164323134633434 -63326635393665333533383562633438303036616262366435373739386430353964333265393732 -66643838303566626131323834646564613830333937616264383864316666343333396636303836 -38633335656536653334626530303835623531666665326533303535313164323836373365636265 -65393061363933373931396134623264643065633534313566346336343862346537343437363765 -62663264376266326538616330376633353832353234653661613964373231666562326466663934 -38393931643736626332623461613737383463663935656263656233306437653331343838343865 -64343239636166343134336261656162393938396633376663366466653634373566336165323237 -34386137313961653739393231616532346664366138356631353030623236343535363435636462 -32323564306339396437633763613535393230386631616166656539373861386633363464653439 -34323134626334356631623764356232366337646236313031336138333636633834353463363961 -32316664383038633330383765356563353062303133333133336365346561643234386161383461 -39323964303061313461386333613961396533646161663230666466616231386239386666306233 -39343239323739323738373263313662336237346663663432343861343034633463386163303366 -38333537626232663438383230623032623765336164653438653434396362633063333437366338 -34373431323539306531323536363238333037643337626131336631356537626237656630393964 -38393736633433306632323334613232303162313962616334376130353931336337303462363266 -39643137643034396564303531346361336134353461653535336165323032323238663631653935 -38366339366436376166333335663230306663633634336434323532316664666134313365323834 -31363964346561373262393632366637396633323332393162666166326631383164643265353135 -34303664353434373131653530346634386333663732373966613761616261323032336266646163 -32663966656464633565356337653534623962663939333033613933633965666339653764663134 -38363965393730633638653561393432303835303164396462366435353030643966316665333061 -39643634646137626338323537393031356532616637666634333139396630663930636235333735 -66336465666439356636623037653564393161393432346534656132346631396462356463336566 -30303833386638333866396462633330306439613139636331636331333663386438623461343133 -30643164366434353765633738356536643861303232393362343131353730376364623463326361 -37363061623333653466636438666465616133396233616430393265626362663736613031383764 -63353065306166646461623763643062383738376266353765643134376538393233383663346237 -37643639663063383266373536323533343936633134386263616163343637613636303134343037 -34626232303335393532643134646132323463396333386664333731646331343937363661323539 -65663936366464643162633432666537393439313664643638343237653566613235353165663336 -32373037346239356337633036306138343366666463363538373836616530313565613562383433 -64616263626165343938363230613039356137643665653734366533393033316363663036363738 -66323663663366666162623734363465663939383830396533383665393139633530616263663136 -64333132633031623835373831636366643831626235303831313761653734666365386462393534 -66303332656561653162636636313439663633396638353638363465663138353866376636326634 -63613865613466326230323564323439393061653664393261373531306235333663373434636262 -62353132653333313635653633346461323165373862343839316539653038633664353830643234 -36633763653738323732386263643461333761306532303534663763323735636563366266653464 -66636236393033613736656562663661346162316164616663306465623431613133633130383136 -35313434346164653163396137383064656538353766653237646237663639663039663665666236 -62346139633234343735303762653030326333333764356562656435623330663066353333326239 -39646465393362323537343766366432323765363139643361643037373739643636623437386636 -32353233303337623136343062623633306361383737303431613663633163643832343434656335 -39633434393466646366376534333865633361333861653366316238626637363537303335363662 -61353830303733623665643864333134623062356334616331363565333235666261653732633264 -62663238663461343738303764303636366638393830623264613730303635623635626364646464 -35623239356235316136343532616638663930313565383264663936633733386663326161623830 -62626634313963323866653432343561303233343035353433613731353538356438613033346638 -33613466656633626261326465336437613630376335663933303061393731313065636131393762 -65613037653363636235613838613535316635613066393436356537633662313539323163613361 -36356632323634363335366665376663346565393439313031636331633235333664663830636135 -64653266616262336437623731383161383437613461323837653066656233643230663064616432 -65383337323333633465316533623465303735396430326334643634626436303263396534356335 -34373134653232303866386433643864363536643138353965323130616338353731633434326361 -66303133353264343664323435653133383431626263373237613631616235666465616333343937 -37323333653565363665376236396232393132336137346461613831623063326631636335333365 -65376538396265313732323932383061633464393630393563386163393230623238633938396535 -34333330386131353336646361313634353862663762653234373235366565343232306432653731 -61383863306632626463653831383735636233623966353130626634366638626236383864316531 -37353062336539626531356133313132663330663135393930356565323364353761393439373533 -61366465313462313033306631333432646163653832363564313838643362316263353562373262 -33343664666230303065373836306663643135303439356362336634346637353438633364306365 -30623332363436353865633738663464636132306134386465306164363333386338323433643163 -37626235303062393933393363656339636139323464373439363765316266646536316336666163 -34306262326238343937623432643262646263666266623933623565363535326235623637396237 -64623961663037653033383933333062393932613933303962326538333739303731363137623365 -30363030353433646133666166383938356232396331656165343531343232613934663834633464 -36353331373233393861636131393238363031383135613633373665613364373466356663376431 -66303331383837663261313838363266656164633836623661326331356566653938306266376632 -63613238356135373938663030343634393566653963306237303138626461613931356565663835 -64386433613937643730396130663333646334386336613864333533626661626166346232333964 -66316664346231376639393132613936323261383131633737386331343966363961633237666334 -38353363383761333439373437623937393534626435386262383732363833346166656233666332 -62636130323536663432633434646666303664393130626437636132316264613535306463623964 -30633030613665343631373366363737313130666337326230633631646461356362363963306361 -64393639353339303436346438313833333432356666666339613666623132636235383866343838 -36666263343538633537303665616366656363373736306235333264336466313939356131303561 -33363030653966316232313933323665663330303338366333656536623861623537313266383565 -65633866663665393635646531353539623362646663356664333866623432333465333335333333 -31616262356537646261373166343665633238633235373335343134393366663462393465643135 -35326336613835663132343233386564373462353561333066323631313664373865323233653336 -65333731336565633664636562326365343263373263373162653239633964396138616335616230 -63376562383064663330363562306338346465666563306365306639353632396633323830353337 -65666233376239333436633566623535383065646235353832363030303565623531333539613864 -63393339656238323466343564333134636164383062613138656138373936636531636166393062 -32613431636233316533353937326234663336343231313630393037313663383034383238346562 -36383264626366383835623261643562323037303661383832323939363939623038626664393530 -65353061313266633764353331313532383766613735333131373365366336306139343265306634 -66313435313965633362356563313763653634643362616138633832633136333362343731346166 -34613431653134363732353833643962636431623036393935666237663833373934373438666434 -36633538306632383439323465636665303863646532653165666638316137633738363736386633 -33303234306531356136316463353232303737323661333430333137636633306131316434376665 -64323633383735313536373534626331356631316464643530363866633730353239346633396364 -36323437306165363465613365383666353037313333653230316234626439623964343336343762 -66343831343133343330336536613134303836626434663731343636613835623364633236653962 -63356635363239663533336265306261393337313136313937356662616231636461373230376232 -64313738333966633265626166653266313932666134356235373238376530303437646464333364 -31613631386335356561363938323831313061373566323638663864393266656361366463353736 -63386361373737383837336435633562626566656666373737313464323466313364626466633537 -6661656232313066363235616364646663623039386561636332 +31333135363439623565336435306566656236316266316330623236643862616666326332633937 +3130356563393965626334373564623336633061303264660a646662636364313031316438396439 +36653532656533626535376662393061666430666536356430623030376335636530313237623038 +6133333636323234320a323433376539333265333837303631316637346561383861656662633536 +32313839336431663466633764363937316439636439373732646163663664363665623065616335 +39376463656566376666303635323830613533363638623436396536626634613137353232656434 +63646233636666363932616432663536386630613466386437643534643136383362633264383232 +38386335376635383465393465343065636330383664386435643762353435623161613235643934 +35393038633163343862616633313836626138663237313062356330343562383964383031663834 +31313338633063343665323438313162643039653136373163623533626630356534636436396237 +65363238613665326437386233613861666638316362343337323462623736383263383231383663 +35393832363231363163646430363263326538306136303864393830373163663036383364666563 +64356561656537353965396631343261643431323034386263313139303932663932306631373932 +65616132666631666266353935363262343733646431333436396438313466366566366231313236 +61346539616131316562343939623432303236386662303036643131323632366231626330333333 +62353533613264313434643834386462386139306362373039663730323263326239386130376162 +33383161666538643535646132306262366535666563366661386132376162336237613164313634 +66316530393164376537666135636632373735336636303634323866333930613362643439373331 +65353736333264333636336238653064376538623135313730393634623939636365383932326338 +30333231376366323033316364326330323065643364613538306661303032303239613335646237 +39333335366163383561336233643830626432623336333864343239643534376339303231323563 +31343539616266383461613666356665353036343533626332376661303565376162313666333338 +37323332303765396362643732343935656664396136396361336533663335353663326233373664 +63396265656538623238363532613532623633386531336236363363383765366335396532653161 +63613736653863336235356336393839336530333930663539663536353963336331646630333334 +30313436313335383365306463316338623033306634316332373732346630646430613630616236 +61373430653238373161626436303264363662313565623862356266313930623162626535363765 +64653136356364326238313136336335396131393637363432383961616636373839303132353932 +30623366626236373861303838396562653131633436303862393166353930336636303765383438 +36663863323337393061333735306663636564313966326561666436393562626562653764336162 +62383235646539323861663438373365373239323731626236333263383037333065646433363464 +66386435363938646538343262636331353064646330623439303166643538306130616236313532 +31663238326465646562306265346132346138346363313532363566373436356630323130306363 +35346463326136303532386433336137656639616165646562373533653663643031343533346565 +65633237333431643038653830383836313538343033386665346431336235636133353637643234 +32666361656137323130656331643562353135643861323438396135376532373437616333336631 +35393366336438383666656165313634336133653238653464343730333739323539386331336461 +32653563353032643730633530303636636439316139653231343738616131626438633561363239 +37383533616565373162373832653730646265663864316335393938303465653035306138633437 +62306433656365333561643333656564666565633638353730353764336561333536666136313235 +35333133396332626637373962376130303838633131656538333634653861613639653165363930 +32313436396161306235323861363265626134656566626132353362313830383638643739333765 +65346636306366343863386238316530623338613038643933353839366434316339333430336230 +63353839356536666165326138376163386534616661323834336163306632343536613034316335 +64386433386637666430303338626635386131373134393530643963613966363965623763393133 +38333337663630656366393338383230396431616535323864383235636337363336313036323139 +32626566353261653734366237363131323431396665333638376637316536656434303434343133 +61316564333536643331613437623764396232636461326435343735383563353236313238643638 +32366463363530343866333334356166393433376563333563353135633336666434643435373766 +33663130383436333961373535626334333931643030303834373330396530643837386364393933 +33623835663833376263303237653063313861386333393262326335333365653164343561643135 +66393962613933653762396566316338333861656361303939393631323163373133653135396237 +34376232383638303861383462366564316537663064373865343037303638363337373435613433 +38653137383561646634306334393766343837666333313135616365373633326230303437306566 +31373161666463313131383434333530363163376637373637353531383463613364613032393638 +31333938626464656465366639353631316337376135393765383562306166633364396536356633 +63346434636636393736386234616534393462306335303135386566666635623431306133366665 +36333762353130303264613232323362383762653837616432353561643030656439616163396664 +66313735366266643364333665343765393264646630653638363563313338316633303639646563 +36306538616237343064366139313533386639333734333131636636343064623237626365313163 +30323438613564633739613733386430633839303331303238303762643664343132346637386465 +31323732313166383562323161316139333532636238613661373639383539366231663436346664 +33363439313964613938666338333135396633396635383964613736306436636364616131656533 +62326263313935393339373837633132653339366133653136653032666563613839643334396131 +38353032316336343765366439326230333535653630313065366632643133393063353438633362 +39346132623530336437623530623662333438393361633861356535356535306137666334373232 +64313465303961616133666335643338316331373164663966333762643964363538323637643338 +66323563653364303538633630316562386230303634323966313935363439666464396262396564 +37633636623835373832396462613665316461396662356664383938333963333566643861306363 +34663461663633386464643961393137636365343566373262376432613264393536326136613763 +66396139663861336432386233333335383165663434356563373031656330386664353635616363 +32363963336463623131383536313233646462643137366365366337303430313131393764396537 +65376438663733626636653564386631396337343236353631353638626330313763343632303963 +38323035333737366562356431383535333639353866613264656336636330323637623033336165 +36326136333232643539336230353165623835356665623736313166613262336136326432323163 +39626337333464376331616664653366333430363631623138383861393366646437353364623838 +34363063653830316339303131306537346137653166316633343764363630646133313062363962 +36336530626162393733303432656133663233633765656239353161303431633164653331346664 +36666465656264643737383031623834643334346637653162343436376631366534623863626431 +38313766396366646634613533623231393632366562323263326461653239663037656539366333 +38303562653839313266616266383666346438343963633231326266653562663462633534383063 +62663665383237623632376461363834396637626537613834626132643165323735646139386232 +32353034313330383835366464663838306133613430316237303831343935653238353565666666 +37666364323939303164323463313636613861623963333534323033326662313662336132333565 +63643935353432643332323932333735643564313231313563366464666132646333336230653066 +37393564643734323634383266616632336430396530633732306566376564656434313762323863 +35346439393330616234666638653036363034643064366436363638383664626530653463636536 +39356135633836613430346532323732396538363735343232653666323963366433643238346134 +39363637353939356464373064373063663730386334366433386134656265613731353231393333 +31333830303966636130303463336566386266326338333663656635356139373039666435316137 +34353934646136383263353336363134383331656333366463646630646264343533653735303161 +65613666353339663565623230656162653135323163656435633132663265386233363930386165 +31663861373162303338313265613536393533313738626464336238303830366636303131623863 +37656330616336633437383937383235616439376532343166643966623437303137306262653036 +61636237656332613232356130333237346239616639616234656534303837343832316330333266 +30353235626666646136326666373030303963326533336365366466313637626466376261386132 +39303164396236303232346234303265333531353031633937633236626664346130336535646133 +36333833386362393935303663353062653836356564346338613938313532313637306230326465 +33393464366231643033363230356566383938353434613737346233376530336537306562353034 +31313938613732383330383239323564643531363733633666326664353330386137383263623330 +61623534646463663231333561333431643563633065663864613562326464386330633839363065 +32396266356462613562373533306432663866636165326330396634653332323165313565353635 +32343533306564306631626331303634633036613730326636643961663064353138633362356630 +33373235656561303738303263663436396162643532303264333137626138353938643261353564 +36363465386339326634663563616662633634363337333462653166333233343038633062663334 +32356435663738643531333539663936316535333836323934663831623039323264356562663463 +63323165663836616364393932333137363037323034323632323165396664353766343138306431 +65393165373561373064396530633230383963333562636661653062616437333037316462663335 +34613661653663376663613531383764646438396666623661363461653133303939626665366430 +37613331666333356566356432313831663737386362363436636239306431373534366164346631 +32306361336363376337376333303035333763306138333238303030346434363662653766323635 +36653336663635323939623430316330643837376332326136643136653039613366336438383437 +66623530363333313339646233373430646163306165353430363663353331613762623031623935 +33633062343562646235333130393263373936633731663362346631653731363765646361353436 +63663532313130653634316530663361323536373031373031316561656238653333333331376436 +31646636393062656166316161343438326564373333356465626463346139633836366634316238 +34313166613666356535306534646163363664363066393331666564613230303362613666313938 +37383065633064303661663666386566613664616163343034383766376638623364396136373839 +37303035383733336662643932393636393037643861613633333265643333333332386366626164 +63633436396264383035666334633336633065326339303062366334633865336134623134316238 +64396266323139396238616137623166373362323364313936396266366532613261353735336133 +64303935323231656237653134396134316130376262323633333863333238363137646131386634 +31616133306531643338356235613837356135303938626539323139326633633939613330653434 +37333263633739326230666339646463633933343163323365363761353166353066373430646432 +31386535633661643862356663353166396236633266333635353864303532363362643561313334 +38303364396661316266626335396265363234373532353130353639646331626230303932393136 +38613463306561653765666562656136353531313433643162313137633535313263373035616233 +62363339313662373833333630376466396265336638373361646434666636396333383063636462 +33396236653130363862336630313362643565613962396439326531336339373466383138303861 +64306135386337323437386331346436653466653863353836623339663037666235653962373832 +36393036353533313437333033366266653964613766663331393363626532343866616538633264 +63336461623838376234343265663933613965646666633132346433353463383839383263613530 +36396361346534353834633835346362336164343932386363383637326433326439623532623634 +38613335343734613335373761663935626539373534383335613966373334353763323135643666 +32353530613163303866343832323131363363306638613336366132343633623861626630613866 +31393261316637323764656535643837333265616562336634383464393561363932313237613162 +32616562336335613864363063336564393536373734373930353436653563643935346636333631 +33633766666635393232636464313063646463303564656663613666303234363138613533376138 +30613935663362626432303266356435653566363439663763306233663261633465323933326437 +32663962663932326263666461613365623264346539343033663566366137313732383839343633 +39616363656434623666316639616136333431613136386439373739363862663466383234393962 +35373966386235343535663362343464333531623136326565333633386561633132623762306466 +32626166333430303161366638346336386534303838653737393731333238346435383134396561 +32376435663363373037613332653333656166353530393635636466666438306531633535346630 +63623831303435643761313836316435656465366331363930343637663439613738653363366438 +35316337626234393932306432363631656364356237316233306638396438326339373866613063 +64346338336239636535386130396465323333313337366263623362353838653938663161333539 +64303063323838303030303635653038353432313333346632626330313539613266613362333837 +63643735613363313163346233353535333434316337643961353635353465363238616432366434 +35383736343765643166633033643462396634306163346235306438323264383438653635346433 +38326431666336623635633531366562343830646434343964303837393864303836313434626230 +35613437383034303761333763303432666630356135653637373936363331643032376437646630 +64663561666238386536616630343763333661643762396462616239396536386634353431376266 +34316433623037366330346633656365356136626363343461353137646632343533323361636336 +62323939323732346232626264373964303134626532323937666537356361393461366133633731 +62383037326563386266656464313331383733303837343465393234343730646138636263396636 +38353262323064313131623738633835316338666530383335363565306432376132656634313264 +33663237316437396432386130646664623065353361346263623637383630323136643262343865 +62616230623631333864643831393438373964383037616265316337623335313265323135353935 +33396530333935646437613931383137646233346664636363623561393336623062623039306362 +62653966623636363562393336646164663631366334346361663039313161663765326634656238 +35613239653536663639666137393963613231323134343639343061663935626162316664316539 +32313639363335613063303664646163663264333565323934323264656438343831643964393635 +62333061323861373433363638336363653265613331653665396563386362326336313430646438 +30343936393166323033666131653734366439623937616665656133646639333739323538343764 +62373335363038616239626638643933336433663631353263346365616366343061306430333031 +39623466373633363565626331303463363964313638356632626663633533373764626664626434 +34366336626466356332393335626433636438623866383232663937653465313439316635643334 +30663861383261363830303863373961653135393764613435363939356266363836363066333835 +30626263376438353765376235663339366336653337333638343666373235646339313139333966 +66373262306164643536376539653432633265623437306634636132303934313036623736613337 +64353638373262346564653563653966636137383633366264306630326337396132633231343831 +31663439643737636661343834663364313734326437373430306430303134613933326633383838 +38356630386566313830363464313262653038353064363466323064656433636663633637656631 +62663233623766353962666332663064653266393937346162643731633139326362316134353438 +38383765343631396132636663363465633533636532393835383730393066306633326364646337 +32363238363365366331303665313634653635303032356137663364343161326132613039343162 +35386633366464383138646630653365636133313964333435373533313163343235643036343531 +63356139353739633565363165356464396332663564646466383637643837623565613837376431 +61643538326230623763 From eaa0d2e0fc1e67191c5c7d1e0da0a3f548574b0f Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 01:56:18 +0100 Subject: [PATCH 09/13] Fix bad indent in snmp.yml.j2 --- roles/prometheus/templates/prometheus/snmp.yml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/prometheus/templates/prometheus/snmp.yml.j2 b/roles/prometheus/templates/prometheus/snmp.yml.j2 index dc2addf..8757d79 100644 --- a/roles/prometheus/templates/prometheus/snmp.yml.j2 +++ b/roles/prometheus/templates/prometheus/snmp.yml.j2 @@ -143,9 +143,9 @@ procurve_switch: labelname: ifName oid: 1.3.6.1.2.1.31.1.1.1.1 type: DisplayString - version: 2 - auth: - community: "{{ snmp_switch_community }}" + version: 2 + auth: + community: "{{ snmp_switch_community }}" ubiquiti_unifi: walk: From 763cc2eb514ed7e926ac1beabc2b973d40545000 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 01:57:08 +0100 Subject: [PATCH 10/13] Generate targets_switch_snmp.json --- roles/prometheus/tasks/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index f215930..a820190 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -57,6 +57,13 @@ mode: 0644 when: prometheus_unifi_snmp_targets is defined +- name: Configure Prometheus Switchs + copy: + content: "{{ prometheus_switch_snmp_targets | to_nice_json }}" + dest: /etc/prometheus/targets_switch_snmp.json + mode: 0644 + when: prometheus_switch_snmp_targets is defined + - name: Configure Prometheus UPS SNMP devices copy: content: "{{ [{'targets': prometheus_ups_snmp_targets }] | to_nice_json }}\n" From 8abca7916fc134c8f6bcb4476cde697d8aac7f01 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 01:57:32 +0100 Subject: [PATCH 11/13] Add switch_snmp job for prometheus --- .../templates/prometheus/prometheus.yml.j2 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/roles/prometheus/templates/prometheus/prometheus.yml.j2 b/roles/prometheus/templates/prometheus/prometheus.yml.j2 index 937f64f..7399f48 100644 --- a/roles/prometheus/templates/prometheus/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus/prometheus.yml.j2 @@ -81,6 +81,21 @@ scrape_configs: - target_label: __address__ replacement: 127.0.0.1:9116 + - job_name: switch_snmp + file_sd_configs: + - files: + - "/etc/prometheus/targets_switch_snmp.json" + metrics_path: /snmp + params: + module: [procurve_switch] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9116 + - job_name: docker file_sd_configs: - files: From e5be09656bd55611b60ee3ca4a79e0465725487c Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 01:58:02 +0100 Subject: [PATCH 12/13] Monitor yggdrasil from prometheus-aurore --- monitoring.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monitoring.yml b/monitoring.yml index f83c2e0..c7c9927 100755 --- a/monitoring.yml +++ b/monitoring.yml @@ -84,11 +84,15 @@ vars: prometheus_alertmanager: docker-ovh.adm.auro.re:9093 snmp_unifi_password: "{{ vault_snmp_unifi_password }}" + snmp_switch_community: "{{ vault_snmp_switch_community }}" # Prometheus targets.json prometheus_targets: - targets: | {{ groups['aurore_pve'] + groups['aurore_vm'] | list | sort }} + prometheus_switch_snmp_targets: + - targets: + - yggdrasil.switch.auro.re roles: - prometheus From b9269f3967a08fb3ac50bd004a0af6319c42d1f8 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sat, 6 Mar 2021 02:00:56 +0100 Subject: [PATCH 13/13] Fix monitoring.yml indentation (yamllint warning) --- monitoring.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring.yml b/monitoring.yml index c7c9927..76282b9 100755 --- a/monitoring.yml +++ b/monitoring.yml @@ -92,7 +92,7 @@ {{ groups['aurore_pve'] + groups['aurore_vm'] | list | sort }} prometheus_switch_snmp_targets: - targets: - - yggdrasil.switch.auro.re + - yggdrasil.switch.auro.re roles: - prometheus