misc: prometheus_snmp + various minor fixes

group_vars/all/prometheus.yml

@@ -0,0 +1,3 @@
+---
+prometheus_node__text_dir: /var/run/prometheus-node-exporter
+...

group_vars/all/vault.yml

@@ -1,289 +1,291 @@
 $ANSIBLE_VAULT;1.1;AES256
-61326136373937643366636537333432613961386637656233356236643966353738386239376564
+62363962336164326561646163363130616431306666373036613833303934373266653230396136
-6538663532356263663537323031313165633464336562340a316433343232373238366631306538
+3830663065383337333938663735346262633537643563330a313833613937663764313630353762
-62653832633061623664626537346333633437393961663762386365656563636137616232663735
+33656633386438333834396465353063623662646634396262636261613436313431383433323163
-6365653736626130660a616235363366646531633663636635626362326636626235663037653166
+3865346266326631350a656233346335643333623434646236626535376434376632346465353231
-34633833366131346333313064663762356262623465343461653864646163363530623331376634
+39356231303831663936303938303132633135363165393064313333656166303133653866343764
-65316231356661616533656139623961666331623134613430306566336661396564636638373062
+61653536326463646332386533626431623631353161613336383137636435623532336462336134
-35346236656435616165393563646139643436333966306336323531653337346137313739363364
+34666465666164336161383835633964303933656133363735356637396331343763363262633038
-66383636346333656531626634616564623862366434363335653739663539303035613065393161
+30396561393838643863616639646539663837356164616438653264333334623035353537306238
-62636366633263663830646331653963366262346563323437636362646366333339333866306561
+66376363656532653139313935643266386365353432613963636334366365313238356430613733
-30336639623264323064316666366363363931396134343639306663396236383639373164386431
+32323865613030343238666462613265623461326463633136393530666439333936363362656637
-35336565396463323630326136373139386438623430663735323238333731653239386337306330
+31393361323439333536353838303530326134306632323339366465363332336363643636373539
-33346134653830653637343838383635653966633161303738343433323137616137313761323162
+36653336303939333663393263393131633837376337343537333863633738663336343030303166
-61306431656434646337613230333234626534343236663262623863393366633234346233663333
+63313766663763366330303764373064333730346536373035356161376333646135373465383036
-36313737393861393634373564313730346135323365656136623636653534396564303066333831
+63623066656164386538373866363236393662643638636535613634396632373564643266336263
-65333565343963393832323336383263336164373838663537656637663335663962616635316234
+37623933393138356632613965636463343162393434623565393932333064643964663531626130
-66343237386264643134316433393365633766623935373131386636363764326163383433346263
+39613534336537663338396465353964623235333638393837613637623439653133306431653063
-34616166343030323162666539383939316232353636366435313464393938316232333538663762
+64663232306338383034613334643233336261363832396437316139623865636136663563373063
-66643365626135376237333163373237386435666466643265393830313364663166303063303565
+65633939376139336439386361643234356130323033643830356133656534636465396165356634
-37623761333133613061323038303133333738623865653938316434663465646134613931346334
+63633233366234323033353565376131663766373338636536306330653935393632633334313464
-64396238343334636538623730666130346439343036323864623964393932353466663865303736
+63646231326239393830376439333162373530323831393663656663373434663263306264333930
-66623332363662396431353038366634383666633762616637643061376137663330393237313562
+61653738626133383766346139653936643931656263383236313035333135393235343135653335
-61376266626365366334356561336365633230333530633036353538323763663563313333396233
+39356562393162366139386633623064646462623239383561326565643838346630643933616138
-66663162643831313963346164383365383166333666383133366434356535636638666461663232
+32313564643634363232656566316339306333353762316335613733346331313938386362393766
-34393035613531613763663830633230376235393336306332636135653934353139383963626666
+38333565376563643239613438313636623964613137613766346131636132376562396634343161
-32363366663238636666393735653165613030353964393865623561656230336533363335303839
+66363361393963336331653033366661343936663762333030333630626363643331303036633635
-63613231396438393731643137356234653465303037613966663633313164393733393033336131
+32393536346266373336326537636139373762343262666130633538333631373565623834653635
-37613139383766653462323032363636353566666331333530303466393636393537383838643538
+37616439396462383263643531336163396261646462323931303161373566653066643062303231
-32636138323466616366313864336337656235393838306637343161663966666564343565363334
+36616564376238653966333239316333333735626138646166336331656234353431313461663861
-35376535333539623463646438616638626237346633363533653438653632313734646562646565
+65663661343837396337616466383661626633316330653034643162333630353734393434393836
-37323931393863343239323234663062653238363639316338633863366666363363643061626135
+65626365363464383464386533306265353864633634356232643362356266666239303238373366
-63383838646536616536363737643035326162393164636163666463353165643264666265636265
+35646563666565373934323366383562613064346431643733363834373032653165346462343238
-39326430636162323161353239643461343635666430613439313731356366356564613834306530
+65316231333932623164653665626364613831306335376330303934623132393338633238613162
-33343631313265343438613362383036643638613864663830303666346336363533613961613462
+36353466356463323239663165363439333365376131383034393336643763313338616263306439
-66626630636636646562336364636339646235643936373231313337656261646265383861656366
+35666336333866363537356266333931653131623236353962653166333663363634353866666236
-61613835636639386461343563646230613266666637343537306361336635646531316131373533
+62366562363061386432376139376361303637363764386239323364353635393237663363313464
-33643338303563613261633935626636376662653836613065313132643561323834663263613532
+65333165383264396538613738623838623830336634313261376635383638313937666633313061
-62636164343261313031326133353466616431613538356161626264303334303333643932616533
+66356239393433326231336265343538613736346134306435303036653363363661656266333665
-66373463393366393535383238373937663933313864366165313862653633333766336634623763
+66383861326364653539636362343637613630323065303639633462613638353037366533376630
-63333532646438323834656138376134663036356465613938656430323533383361313664363231
+36643064623639346464613131616332653162393738306434633033306239353439613634646532
-66343038366362646133356131303763613463383739353737303530626266383833316263363338
+31356337636663663332376336646630633238633131643862636132383337666130306438303138
-64646635666633376337656464336535303838636632376333303162386265326132313261663665
+65633931356133393466353839393539343932656532386165326637383663633835653233646639
-37613738303238336438353763396163383664626263353863643538373631643735396334393734
+63383462346362643332393036333862636332326530373837623131323563666136343336633133
-30343035646631613230623338633238663333386433626236613131346563306632653936303666
+38643466393438373031373163313136396134313461383265323332663739613937313938623864
-33626232303761376133343534336633636335636530356237353534643865643266643231373032
+34626534323631626331633361383235626562383534623336323033396637663262333637376538
-35646435336532643538646430353531626136663861353731623761653939383433633061333232
+37633066383262323261306639353361343661396236353461323133666530613333613638383237
-65633230313135326230396361396666653339643464346366663866656433653965346265383464
+34333965343130386166313430653330643139643130303235396333613138393031653264386463
-35376261383961303663353138343364613334313663373539323639616535623335633061646262
+61626539646232386434303934633639363330333265613132393061366266656637393537646435
-35623137363137666633613862653234353961316431616337396265303666636230353931663966
+31333138353864333038633263303535623135316464616336323939326335386265643931353266
-36633435613231326336396439653166643835623737356566303565313965616535653563663132
+31393735643063313831356232303239386436343865633135326131636333643630663739343966
-64316565386639333937633761623034623133623630303038653239643438653136313231616233
+35633762356239326632653234353034323564346431363432366561303966306165333966356437
-36366661626464333938626163373836363431373336363930323261626661653730383662323133
+64656230346464393337633562623864656132346132643736643336653734623333636433613037
-37613031333230343339336463353433363637366438636166303736373765643436633236653831
+33306666346333386662303861353938333636306366306238386166646336396535333235333063
-32353830306639663664626230356463393631353537396666323638306161363834383833663838
+61383439313030623263386137343231306136633833633230653436346231663030343964373233
-32393564383430333237373530376164616166366566656462326164613535363936626430353138
+62353662353632323637386466383532646530386638356564666339636263653766306233656635
-62626433333836376639366238616430303666386335356634653030376263333234616335373465
+33393637636264376364396234303434346233306638336430373261323830383036346633646635
-65313431323236383538316338376537396439373637656630303063346562653764623332333339
+66636235386166303632343037363233643063373838373866363234666136613838663665333233
-65636364373063303864613433346363623135623838343964643031353637373037356536383366
+36306432313361316364363834323666663732616261386430323863353565633761323063333938
-33336666653034333561336365633466326337613836313732316464666531343234643166366634
+33636464383235316665333133636164303139333763653564343138623562396231366364386338
-32626432303832346438636636323735653536393033656630636431356665373738383164383531
+37303832336137383939386234343231393131393136373235316364623762333762396133653338
-61303238663034303831346564663034663236383638383338333132333163333637666632323836
+32633331363066366263646161316265393234653730653463653866653934653534663936363433
-39306234316266613365353236306265326136666134356566393533393333653532643964343961
+36366264346164616133633164373831326230336536633964313635353461643864366536643164
-36383532363437366263383465346165343731643661363435336436653661323237613261613632
+62333730313030373235373739393464633164366466383364646239343034333339666535663564
-31366438353837636534646132373438306464663066353233326436323234613034666432653439
+64383437393666366338613463376635303333343036313565663163313433643661656566626562
-39363733653335626433666163346437396566623266633630333931366462656231366363366331
+35653335613130383961623462383230356233306666353764626566623865306130613862616238
-33323466656539363063333962336337333136623539386336626232396333363065646533316361
+31336235386261613836376663633832333964656231383762303562353233653763343935303037
-34616265383461363739393762323061336561376136613865626438636164646662663830313935
+36633061383433663432313139633866343161336461333432653839363238613938313530633931
-38346362376164613131613730356537663332323236366131653734616233336562386633326434
+30366161663639393635313035656639623939333166616238353665613865303265663362623434
-36333666663364626333323062643838323434376164333432353764643333376435613062373131
+66366333633834336231333665316561643434643835663062366539623630663732376231346133
-66396632633031623064313030346234386339316635376436613430633961323231636137653833
+38383634373838353039643132313932373033653234326333336430316564646465313763323431
-38383738386237373335333564323233636534386535303962613838653737616162306266373639
+66356665303363336432393561636539303562633863383236363032323238333162376538636662
-32333966613639333963626365306463616166363336373732386436653137666639366535313262
+37373332653436386163333661393334343935613761373131626330333534613733313066393238
-30363634373633303666363035383133616334303932613766326638353961373162383862656563
+62366237653238353938316266666365616337343435366163303132653364353436386265386432
-62303537643166396230323266303036366238346165653164353634303666353036333133333435
+66306638303162366463316537313432623062383334623566356261373262373230636535323230
-37376563333835363463653734373663623566383136613335313362663662393564643161313431
+34346432636263393439623238383766376430623537306536626237313161316339653763366633
-31326162306539653164343639366233333131346434353231376361653962656133326135313064
+37333961363664643862636166663732366638353363343562616464316663373434303066663036
-36616331303633386131383539666134323063333038376236653530393437356263636538346664
+61376436373065326162643732333465313739653665316438346161333934636461656530386138
-63633433303432666336356631313635633862643938366333316439333439663139356434633162
+39396434313036613236646161363964643231383535616135636166326332633962643532653662
-30373135393639303862646334383530616536346661663532613461363663613232373066383861
+64303332623238303861306532643466393835633536373066356665363264653632613231363134
-32343366636631656134636131616230613132623631316337393432323639656237353937366639
+38363230306663666365663164373538343531353636633738646637363736303537636361396634
-31616564323766373166646433306137386631623432623134336163326236346437306562613434
+63333966343263323363333630613836333964336137653638656461346136396632646434336262
-39663164303738653264623162356366373334313861303434613538396338653933363437363534
+39626131666461303138366436386339643638626635616636343361316364663831623931373036
-64303131623532306137343665396135623038613363643963386337396463356437353331613936
+34363231353038353261396133633537376531626437383937643163633763326431343230633736
-62663261646635626265313234393561616361393534636662653531616630623337326235336466
+37656438633837643763373737353964336363613462366433303162636262343733656164383262
-35393961313434383666613265613463393439346634316631373937623038333331316531326163
+64623033316430656635393933393336323139393038376130383764393737393565383338656232
-36356639313662363935613139383364316332333964316362363665373138646566633761316461
+33613734306330633233643064346166323633346363383964376630353462626338323762633737
-33323061343765333362646130396338656265303562663931333961306664623036313662346635
+39313835633634363333636436346634396131376561303037346537326433376264383236393837
-32646533383661363834313733313330616562636634613064303231626139386536393663666361
+39616238626537313666636137356631653033653665653064653730303237313731616266386265
-36313430316635323765326231323664393937653162353532626534333763386135663663376334
+30663465306136306563613530643032663336616334653738653965393561393632373537323638
-31653636633534616265326431343066613862346565353164343963396435383362313265376439
+63633066333934323962373965633937636333653134643438356464373630366534343134633532
-65636164326231646538316138633339353337383138306136636431633162373936616533333866
+65353465386431623736383464666231373464613964313732336334393337366537646239643439
-39613539396465366336623332313964393438386337333836633933333832353666323232343962
+64646237316261326263643330333438343334383066326631313266386137646537643262356665
-62663733343863393833353937306435626661623761666638373938653766353630353433316133
+39646538343063643738616438316161666464616232393066363631373931383131366661663162
-64363935643564383864343666333031386262313866303431316265633861393830356535613464
+38633764623238323865613239643061656130313133383736623363313163343566633136396562
-36323835623831633664386130306363393965633661653163656533616431343836316530666139
+39386462653463643865663464626366326135346231366531326135393462393031653035346430
-38616564653162356331623862316361616438343464643730396361653334343638393661363530
+33656332646238326637396239306661316436323865363537643065643830653665376165343561
-65653462373764626439363637303135653738646362653233643033653561366363326164356131
+37356362343263636130303333376439633131323761356661306135336662623264623665613164
-38343362366235626462303932376237343535653461366535383465663435306439616239656361
+64626634313437336535643436383538366662343865396465653034333936653663353461616238
-37313738363839383466383535643362393930373831633433636566396433333563663537633362
+34383733653236633963663533366135653832363461623462303435373063336165666661343433
-61316535613866373635363162383030343438386635323065303762336331653835653239363039
+31353664623337646131383237343432663739356135353833386434396330383830353265383238
-64383932383730373231663932313039653738613265356331663735363634316532393962376337
+63363630353131363734666334386532656333366238323666653266623566323763393432333665
-39623031343331373035393934623238346665613237386262376139303832653634393133316131
+32636362323535333136373033343566303230613834386264653034646231333339366238633864
-38333666663161643166623431313539303130633835633032663338363663653038333133343965
+30363235613235326439663961643433636130653365656662396232633034646233393839343561
-37363263353735626462396230303664396437363539333336626664623737313563613239666532
+66323964653935303839616365643239393439303663616530646562313031646438396330656134
-61646636303535656531333763656163363033613837323139646239303134613266343737626137
+62306663646366323338383861663130373631633766316531396165313536353961366631346537
-66653335666562306663323364666535333663666132663738613637653232613837373530376536
+32376236653564303238343837396365653264343165636538373432326436383032613664373338
-39306635386238326363366661353432333538306262353132653639393835393535636135373031
+37636661373333616632313739393733653363313939356662646536343035393264303131353238
-37303135303532313033303461333637656562616365326533643065386432363965616139303235
+33343430303430613834393761373231386338656238326461326336636261373465353963326634
-34353639363630306437653764386161393731663938653965383965633731386234393538333336
+35373535623630363565386663646563306434666136363834323033336331663633313066633238
-65383831623439633037386265393963303735343937363738376131373930653464303635313666
+31653538323235623564336635613637396433643462613135363336376132653533336563663634
-37366333656130373935376364373735316562366637323733633963666439303363343365363536
+66396532636265353061656630363536613131393936333139363830323838626634303761326462
-65336133316136386135356636646364626132346436626635656662356136623064313830643361
+61643233313234653231326230383763663364306635383363313938386566363039346635376230
-64656361656133353366663463373761323236306134623966306362373238323661376362636630
+33613934373036356635346434643964386563343938616430623932616537396365353062316563
-31616334323533626535393038393539366339333637326538626637343765373030613163626133
+36373664373332383234373266383439336638333833373135623565386436336236373864313964
-34613536623064323365636364663961313334666235656133636439396662386166303761353731
+37643662316637623533313939306335356134393030393239343539656536666666346439326162
-61613930323631356237356232353066386134313031623363623136313536396563626136343233
+63656433323364333762633033346463643038646434643562363533623961653730303935353637
-34636233633731313937356264343536653837653863626361376632663831616538373938323437
+33623537373435343361653731383930663266623565383165336533623062376164396462366661
-31356635633966333261663137373365613435373334626337326137643835383834626435303339
+31383738386335336432393537333665373264336532343061626439353530356566636433313330
-66303663643731666331316333306239343632323062313064333238643834613666343365383132
+61343464393538653561636562356463386166626563373036653639656336623533373335336134
-32396564333661363135646534363730616161386238613132396231303964666162323965636233
+65346138363762306233383439626431373334386531616165356532393734303635613036303263
-30353438366532656462393763363330363033383365373039336461626138343732363561333038
+34373235656636326133663966393632353734303866356664333232306135366630613135376265
-64363137333133353066636137383838393138343237373234306465313838663833613762636662
+35616630643563336439343239663961373737323436373264336536376130653533393435366237
-30356333306562663235386138366364376330333261653963333438373435613835653965373037
+30383461623462316465386437653630626136356530396633356535623839383533643237333736
-36313439353435306265313061373465353764653639383764663735376535646630646462356465
+34633339616433323064373336616264613035346330333433643038623164653738643933626136
-65623932343730666334326232343763646333623630623530663939656263316438626632333261
+38386562663966623635343066613765343834656637633935663562386230303737636561323763
-33633239323062353032383130333366303836316437376231353066306134356334613535653465
+66643861356435363363623166326332343738666430393138316238396632343462323563393065
-30653837326430376664386362373732643434303433343939616434653138333737303264613538
+36393434373739643531653737343564363966623636613065636632616635656464303964656431
-30343731613736366339343139623166623236326666336336356638336664326161363331313634
+64666133323163313861656563623932316131333439636439316630383262666666383732393530
-31626436303461373536393661323139613032663563613062353662646633363662636261393032
+66376166393837663361353534663663646332366465313731333537366163363439386462386561
-35356239613332373761306631343134376663633237643031396465653634643066383530393165
+64653063393730303137333231363838636333616262303461323361613439383462393639343338
-33383633346366636561356334336330336631646332326663343838363439383730306633393439
+34623531663332373133656161663566313639326536373564353162633538333435353430326431
-39616133656336326464316537616263383832313536323462363765313639396139363065616564
+62623836656535383833303665336637336665386130346437303864333664373966656539636437
-38333561366265333433633937303532376135393863653536366136653836323163626536623939
+63366163303462633439323831336361386639616335663132313839316531343564353561663364
-34373131353331363232303236313664633862393831646231306261653065356632383463383336
+65613239306433663766333438666533616534363439393538333662333531643663376662653731
-38643164393365623431626532373738383536346362303936626238643039376666373530366236
+62393231623137666537636464336461616162393465643362646332376639333236313037343238
-32633663386562643763343031383161383632306630303439613234623231343761646531653863
+30666466666130633866656665396339646262633539623761396163333361313164306361373662
-63656532653839356434303835386636376634656433656337396666346234613565313363633739
+36353762613537316234373139383936653430333266343131383834316534383662653463376534
-36363830643030383163313466376462373334663539363766623931643533336334616466343832
+62346264653937363036326534633366626333643331663131316237643163656433373865653434
-66613461396539646638303031353034363664393439356166633537326439623662313461623530
+65316435393836633035333237313830646466666539613364333064373064323266316563633031
-65653961623434653139393364613234343034613638313064343265333262633536623636646238
+62616436393434363565353130393262356236343865346333353264623438353237643239373561
-32343334663134346433636433353733313761353439376432386264333862316564616631313239
+32313462313830656234653035613135333337346162646162373064623736656165313766616666
-66383637386538396461666661623033386264323264653665346630316532653462646131343838
+33633065343839373432326536623365363862356535393836326635353239393133653531633362
-33316463343133383334613035613234303664363633643232613466633335613635313663636236
+65623333373164353638306465643761656665623336383138366233313363646465336437336635
-38373862316161616435623231393232366230613264306666306335663833323531373963616433
+63306333366365303237393039656336303938666164313062363731623233306438343635336339
-39653065636637323433666265323234336365616131323734633732343566303835333537653536
+30383838343964613365353435636264386361366237626461303139346266393062313439356439
-66356431363835363837386335626538663630666138343937316639623632386336363434653861
+65386363613266366336313261643734343062353532303866366437636562363265386162633361
-30313338353565666665373132343862363762633133666364396164386336633166356338666539
+65623664306233303939353764323565303566613463333165663138383834356338383837666330
-63353562613362316433633633373361323532306339386534346339643361343661316331333834
+35633139376630626639343537396531353462356232373133633631313165323864656466313737
-62323662363261666538333363643531633332643837386666366665373966633231373766643464
+65633365313836646630653761623166316538633730353563636532333735633836333432353530
-31616465643361343137346634313437646661393162643032653234346633313437613434643461
+66386164343164666533663131356339353336383739616439383038633462663564663162396136
-62626239643331633932353563623631663061386135356666373262653664353566373938323139
+36363837643437623138386631386534653963663133333934343961303433646632653162336663
-65356666316235346562343233663038376633356634643562646131613466363637666535323238
+35386363666232383932643835313232653834633539373238663666326166653135353438306265
-36396263306362353733306562313562323762616536633134343536313466623662633230396263
+37376538626266653333626662396538326336333738376337613735336535313733383138383936
-31353330626163346638383434303934333535653536623532313733303964363634313763633830
+38663865313962386365343862646237653234656364613638303735636334313763333137623365
-37333634373639313261393639323261633666323930343666653734306136346336623734353661
+62393666626230366634313631353233346561303433643066656665333738356262643332326363
-66346535323531393430353632326366333665633233303661326631646237353465333961353835
+62346566323530353336333534626536396338363761653465336433663062373131613735353734
-63396435313437376635383761313164373038643166633139613732353933386237633136306130
+61363833643366363361323666333366323763363631356431656565383961373836656433383562
-39306364663237666639306138366537366664306232656465333263636462376131366465373663
+38363632666162373563363737353730373936363837313364306539633332666433653131623033
-65343231633164343864636336653263396366316161656661633964626335383038313964646261
+36393436656232386365313766326336393462653564326236303638366464336665613063343766
-31363761653430666133333939656232356462616539303864383063386261386132396164616130
+37363164373963393263343730393066383762666565613765646137313364396166333631303138
-34396263613962336632653836393037303261363438333964666234613862343031313861613363
+38666263643333303963393036396631633163626461636364643666373634316237343336383366
-61613166643133373938383734656336396437646333383638383339303237633036646534656465
+36666238376331393738633434336565623362303334306334636366383664616131336435353330
-64393964653435323437303838313632633532376263626437336563323264313835346262323630
+66653566653035346335353133643438336632316435663639393934393965353237386264663830
-65333132303530323536313265353330363434306630383832383863663061633231316162356230
+65313937633033646462313430383436663063363565323564303934646533323132323331636266
-62373338333030363034623364376335353465613432353938346430366366323034646462346666
+31653366356231616332336333393464616664323262353739656561373830303634663565323663
-64303538653365623735333135326630333833323137623265383331333430623766643563393631
+37393835343065376432646266336331653962376239623330386430353031636665333339346135
-64303539663861306466396436323066633364393838366464366564313664666363316238396536
+35623436663164613964333637343563643530336533353832353532613861306430336136396633
-65353739313634366336316338396533643936396236303862613833616464346166366338613164
+32663430666566323335383233633430346662376261656139326162653336383732366439383131
-66343166373664633630353539356162623166373963613736353363383033353763656263316333
+64643231393235353363333830653131376664336137366336633662643462313232636235633233
-61343631626262336662356533306432353666396631636562666566373636313864323361336631
+32306435356465303263343537346338303236616464346365303136356438333835326264343938
-64343162646462653866373263393937353830333434323639623233626538656339663638303061
+65666335373238373236366635333330656664346462306331646633643132303837316366393430
-32643764333831646566383664333164313866373632343433333437633637396637353261333839
+64336231663432303565376662363536613436323966346462613330633965666165393463303535
-64336334366162316461376234316133363939643564663134303537306636623732633461316634
+62393136353262623834393032393732643239323432656430393863636337383764343039366138
-61646638383761376564393638393033626534633566636335396333323433313665346534653834
+33306337376135613234643863383639653831633465633733303630353761396335396532363165
-61386136336338303231343162366136636536353164396235323262653337356534616632653934
+34386136393262303231353962326361373564303137383035613131633265376534646466643036
-30386263653464613034363764636336666165333966373163353239393932363039626364623636
+35646330383236356431303162353836303236383131613238343263636139373938356364623935
-63393664313237613939383864663566366562303936393633623837653662316336623139666130
+63343361643562626539316137623365393238656434343130376239666264353437323261633362
-65633561386530383832623639386537383866646366323566633534333832316662333539383032
+39303461393163343130653438613937613130633363643339366136386330336466343132396331
-66366632396663356238323835353266376436353461313533386538326463333832326333343565
+31316338393261353939616362303666336634623236373538653062383962386166636532353436
-37393337656462326666646238636334613761346636373632646130643164636362353062656237
+35643139663962303262643933656132663063333166383538623666393731613037373562666134
-30393066333061623132343936343332346130633736376465653464373739383630313739616666
+35343663333038353236646165663331613462313935356638343932386138666436313538623466
-36646264383739626163653566336465393662316334383439346530353830343939353866336261
+62383965613961306330633436323033333637396532363235643761326666323639376638376532
-34383739303333666532353366643039343637363135666637663836626632623931346262353535
+37633663613837373565363265633137316161333063333335363463383061656538643131623433
-65373661643638303935303766396639303062623633316237303061306166373966393462653630
+39373430626531336533363861306137336531316334333065316563613731616136353138383134
-32386231393038656336333230373633313164353235663836626664393838393039363431396661
+63353134636136653664343962333837663533353637383230313661663366313562613636643466
-38313530636161653133653339376337303639353835633233643066326639373439303039313334
+63653065636163653164323230343765386330633766343434343735666161363664616537326630
-37356639633535646236316164386538666261333663643064313463613630373466626439636338
+31633765326530333830643838303934626431336635333265346338663165653839663433633664
-66336463396131653439383331323865646435333330316130363637646366653532313863313232
+33363836346131336362376364383164636561373464353635336562626464613064323037316564
-39356631613463346230396261623636663862663835353466623563333434353037393732653565
+31346263306338643065396237623461646461313263646362363662346166396464353730393239
-32316163383039396430393537303662646638323666613432303635353939623439633164636562
+34626461393461393461313839666566653930376431643739646135633130353437323864363632
-39323932323837373861613563303934386665653631393230623634316362663138303364373033
+31386266363433343232626438336466666133666239643565376666386333336434326535646234
-32393632366535666630323761323330316132653732343734373235383930386336623565316163
+37326161636431656165323936396433343534613064663733353735616339656132336634336536
-35343831366532303436326566343536326137376565373630343030666138636432646337363237
+63643337353238653264386435323563336538663163353162653034376563346662386531363631
-63663233336234313130313938336262323039656139633231636335613734353036313333616639
+37303833643038333132336636343961346337316538303866666163623732353430656431393139
-32616439623737336436613436646638313863333565353063386233323232633663656364623937
+31363061316465626361346463323564346363613830653162653831326230373165373965356662
-35666237316638383861323863326337323535353365386338323565663331346463626361646239
+39626630336339383665616434663037663730646364323739376430383562333132613833666539
-66383830626334646433633437613063323163343737626166386137323830613563376466666431
+31656461376664363938386666633732663539313839623661613238623130353662326565343236
-61643838303166383332353134643363303465653565313364643836333134626161373362326663
+36643439353363643737316336666165623135616131326638333432323638636634346265643136
-39396536353366323530336532363032626230353839383430623835333939336536636237363965
+66623433623163663132393064373538353532386164363363366639323837346634353531396133
-66616365626166363736616130653563323332373061633834323039613166623438643036353638
+39303632613063376333663030336633636236303133623662383834626461333862306165383364
-37303362643536383137623164336533653233343938316362616134633530653537313264306438
+37353034306235666635333765303834643363313761633738333839313233343462323161326662
-35333231393764653930386536616138303131636363356130343934613232613366326235633266
+35373765326232346538623130656665633662393034323431643965323733333666313262643265
-37653864613534396331306539663538316363666530653134663331356561643839663362353039
+36346463356237303766656532663961336266343232646365376161373439656263323435336139
-63313536326537393932373935396235636636656336383431383731323932383732316466653136
+31643761376363366462616639626531633731396336626532333666636531386265613235303530
-33666638333162663236333436613038326238363330323434396232656336363763333534346439
+38663965373633623265376438306135613562626565306136343331633361316236376530633766
-61616331313935306232653839613566613664616630613362383661336662323162663630373130
+33626133303935383036393263616662343364613666643565666565663532333438373232346263
-35303331666133376639616238373838326339383037653863353664613131313362323965656666
+61306261616366643963326536316366653734346166363132366366623263643437363237346262
-61353164333633386232363132303630653736373238646361613130306165373035326630303466
+62633336626337633437303361623135353765343337333136623436376361616364316365633463
-64396134663466666438656233313963336461353634326163623766623132323339333564306339
+63643364613334373033356433393462626330363631626432303465323361623232616532393938
-30376264343536383165623765303465313534646463346565373864643234666466623832663164
+61636663623637363863306238656339343463383131323663663261373339353862646436373365
-34396538643966346333383766626134656238343036626230366432623639643333316536663431
+30313335623233383233366635663266383837653638623461653965623433376562353262643837
-32383262363738333931653436303031636533366265323137643236346638383330656435616661
+65663062363862323634303965393932333235643032636431303061356262643664343636363938
-65623032313737386432626665656235396166633066393365643833656238323636636663663735
+39633563393663643836653731333064666238646530373031336563353735623664346163393465
-65336435393163393531333036623863396537303535663733373538393265616134366435636432
+61393233363165333164656630363131636231666339353963323365333832393332646163393538
-32306633363164316234333036623730643838636232303865633838353166333835303562376534
+33623762336534666536623638383361323262313364663765646364623331313666623763363262
-36633534386563626562323165663334353832613637303063376532333635626231656262313963
+62376361643938386264396531663665326462336239623862656137643038326466376438616438
-39386261613564323433653862363332653363313832323663356130653232313230623935623936
+39643533356434343334626439373065333139366435386536613633353462633337333563313966
-35386265373034343035336564346336623064633638383463306537316565346366656538356265
+39366233613864313566383565333032336232633538336434636630656137633333306636336230
-35613339323565313932653664366433316163383335303439353637306166613732393531353536
+34646232376639383238636633353235653632333039363936306165386532323935323864383037
-66623933393536613836643235643139346337386537333461313437356463343038656130666666
+66313538393236376234346338643264323361633437636236373830336234366539373363366338
-37346532656365616238633230663732306430363064333439363862306135323266323365386164
+63313565376437336539356665303037396335346133316634323637356239393063653839633439
-65333639373464353138656436353633356136376265363062346232343133346263666434383966
+39666163383365343435383132303330313739333731363233616165653963313633396462303835
-30306265316434306532636637333165356234663735663938646366623632393530383163323063
+39356432656364383339633235353235386132633834626263306565623166666334333531333132
-64656131386330663535393336616564663465316430623061666431653838666438633366663830
+31636438643330313034393665666334316664336331633830343737663864653330393664303931
-62386535353333663237363263393131373364663266323637343166613831333463633333313837
+66393030323164633062663336323462623963656562343736633638373965646138643331343637
-35326637613163653734323435393331336161633032343137613763666339306435653533386432
+63333863623763323536653733323531613161383766363664663832386531313963386565396137
-30633031353336303839353361623139613863666335333131333031363864663366636638323235
+35383338383565366464663065393463383732386264323633313466333536613133373765333531
-30323734396530373737333464396230333936393664373239376632363731633331323062613931
+36333835313733383966383933643462363262396439663237363662663764356336653536636565
-66363433666365376430646466616531653435383633613266316633623535643338343162333030
+65393231336535313563633866346431343666383132663733653234346137353463386437346133
-34653333333033306239353936316238623265616630376136353566653935343232616432633932
+30346134306637346632626331636136393032336537393135383339626163393561373232333732
-61333432303330343064633461623563613734643237646563336136343463663733313439333136
+66333462363066633939343531633161303338306335343065636532396234386262363938613139
-30396334323731363533613039626336643434656661626239396261353865616539346138663339
+61653434653939646463663463616363653666363363613239353265613132633939383162303931
-35613337303062616465373337333432343962336365643362373037326435636362363537373833
+62346638343765393766383266363139623131393066373764333363346435313861343536633737
-65636432343766303036346262626632393730356262633039346161343939646134363036396365
+30353066656539363864313834336132666232306465333461383139386465646533653236363064
-31383033663963326162663535613466306435373963336562343164393364336463313039643732
+66333864626264393230323964653135333232636163343762616362363635656661356661316162
-36663834343064633264366162363139396237653736363632353839623032656435636430373636
+38616662316330363865363261323536636538653431353664396336336363376138613234613064
-37383039666536323665643865646133643438336530303336333764373935313339306537313034
+32353065396165616565626533343838373065633834376130633864323934646263323761363738
-63323939386639373330323033613365623164343535343438333037363662363536653765323437
+31373261363230626434303865633033383964336638333436303135383934663866316630346563
-33663164666439623637346137336237333566366164346664366337363231636630643764643261
+33646438343563333730323466656236663863303336626132383632383839656661323337313131
-31366666356135656535396632366530386363633339326337623537326530383935646364663630
+30353439323030666432663839663934613036393362646637323230343061313964343534663639
-38633735633835663939393333333962396537613563336230333233316135353961366264646437
+35323262346230356637636665343035373966363865326564666365663930383538363263646561
-65623630666266393934393537346164653765396232363861666239626633383766303937643061
+35323533303338343466313361343363323036616239636438633162623730653535663631643062
-62656336316132613161353334613238333765353732306535383934373635363232363064633531
+32333135623139303835326130356634333835623264386462616531633135656639343462383136
-35326330386439653363303137633164633233373937376537383232303437653835383130626631
+33363435613266623534313165386664623033626634393163303166633561303961376566633431
-33656364323666366664623265323566303139363439663433353930386262613062356236313766
+36363230346139343061633137366666393866333263626563346232653437626531353530643662
-37626130323936346531636233356433633831633738663963346433363937636138396133326561
+34353438626638396430346562376531666439656136383166316466303431333062306132326133
-33376638333237303239646639346634323263363265626231333236393138623134663466633035
+62613066623064353461623332613465633939383838393064326661306363343136303633336166
-61633263663362613864326531353432396161343665653266356463613033613163313533343962
+39366439306336326633633838646331663137343663366631626630663136343030386266343037
-63666462313538333063336565386537333763643738316663623030633034303031343331626236
+33323230326431373166363232623132636563663730663066636564363033306330643237626135
-33323030326465386635653837316664396364326266363562326665373332353032633738323363
+66306533326262613537613032636361346537353736333538386237393564363931643631343738
-30353432323137633637346432346663393665376465653566343965663630306332643237353935
+33396633356163303635666663366663303938393537363466386235323538336330316661313963
-61386638633732623039386530326631623964323562386639303966306231643837313637363034
+37303065643561363463326630343835336566626439353161333066383530633932363336353531
-32663837353533346563663163323039313861653366356238346332626533306136613034636630
+35353064626638363330666532636236303533626432613761326337626636323332623365623039
-62323462646336383061313737386361663436373039393130633161383339663665353931313261
+31343637663663646436346135653932343563653562316333373638636238616637663563333631
-35313933336231356361663162356637313065323931333230373932313135633732343735633531
+61386231346262366665343864376166373739343966336464363635663037656232383031343564
-33666538633938333263303039363365376465373061626365613137373835396437346265393638
+39663831336266393062316662363664343866613734626436376230633137313666316239666362
-30626461333837303936616530336537306335333933623662626236373963626163323263376237
+38393636343830316438313162333561663238306664343863333833306239303731613430383562
-39393563376131343739343539633264623036333731383231643961646463646332313837656437
+36393639393462353638303536636563343632666462366564393335303263386361626135656439
-37356339386533396635363835393763663762386231363234323231383433386135383162656666
+32343939393038353463613936663439613033313630373763353564616231363738613232613638
-36363930646362646564653663343332343431636333386565313035633263646230356566666661
+63646463346633333466303638656333326538643235323233343831333665633236346130363931
-64323130663236393738303062396662326430353838653139393438396361396638623737393663
+34343864613731643965323538633233646436333862653533643937643131323635386266633432
-36333961383964323633306631363533646436383762653335306532323064326536613231366531
+39303565323934353863346639376636323230333362613832306465383261326439623861386330
-32376231613661653535623433626139646261616430386331323861326664306539383035656232
+39323330333630653863356366643439383736366163643233626635313332333462653361353935
-62373663656363643837613233393837313934353961623437323239633362366632363364383534
+31393231616662393362636332336532636334333337633062646230373434636534643564626635
-36303334626132393533613038306532363036316364333437323638383430643636646632626536
+66646139396432383166656230663430323932313634366161356332386364666166323530396635
-62353832323963613238663237326536613931346334623339653635643633633634313431386464
+30313562633133653539336538343637343062343062376464326639356438663064633062313964
-39373735336563353331323561346131313861353337333366656634363833623530383464636265
+31336130313863316335313032363938373430383461643036636536323732643238643430353036
-36333335636262613832353332626536656633613434323331646136356331326364646533323766
+36313832643335663632386266373535653065656366373066323166303464363339653038336266
-39396137323233383762363961326233633539363631316432336630373133653339366661623430
+38353762393934653232346332633138396663366264306532616439643365636131313266366439
-31353938336539366236646162386331333462323934653133336535623331646635643838346230
+62316133313734306636373464643935663539663538663637316561646135376437323837643962
-30646636373662373465373662643262636465343765393236373330386566313662336535373166
+64643736316637383736666431313130636661653466663662616365643532356266336362353830
-33326439363038653433356231336635346430306431663765663262323964356132346636356261
+65653638366530366637646439653937613865626165313831616666626664616334663863663130
-35383465343765343336336663626635643939613463653965653033393465653063613438306232
+64363936353165356635343464376639396361626363613135373962376131346561303562633739
-64636263613365653633656334306565663564383263383139643735336339346432393135343532
+35656635306166393866346161626535643538663964303036343237383036333363376130316431
-35623330333332636537643262643135376630623637623939313636303563373838633032663039
+32656634396165613735626333316263323439353062633961653237336233656537363938643135
-65633937626338633065636335353239663130393130386233656332626362303237393534636134
+30363939383439633633336236366462366161316636363566366534303964656663653062653465
-61616133363834653835613532333638383761366461356638376434396265616538313737393030
+35356461326230633738386330383363363765393466306135363265616331313834363662323337
-30323133333961646137356330666139393764623333666337653663653962643834376338316135
+65306461646462623533376636663436333432313866373961666365626561363435323764363964
-65366237383632663337376335373138383133626135623531323438656636616631643235613138
+30303933613537623264333331363131643432313466336130396262323531613763316566643032
-32616630646665313562326232663936623139376264643438643136376137363236306462326338
+38303936666566366134393436656439396537353862663936353934323766323864626639636137
-63633063633966646436646365336364303166636232353331613634313932306662393366363131
+36393338356230383639613065616564356137626661376639613763643464393730313865646165
-61303632343461323466396132663639313133356532356234646138393464373265643435653538
+64633061653630646262343861613266363562323637376336316337393435366161343130363764
-65363731336439356136303633343836323934643365653862623863303865353663323339643962
+30366332386535616638643430333930383134356138333862393938336130316564313963643365
-36646430346636396261636538363633333836363763323035653935366331383439666165316466
+64626238646533656161306566626561363831356130613737363563633166373038613135623366
-64393837636634393465613931316162363233643639643861383663393332363337343432613830
+34613933313434353665306435396437383461333131336263366634623962366539313862363235
-66663138363238386636316439333739346264373336323431336630343436363238653334636262
+36373966386438393031363865666566313332346230623665653763663533336337323865326338
-38326535346230633231363137343236656130643439383539666362346134393063613838363738
+38303465303966386164316264393535623336333136636234313635336362396135383432343233
-61396237316163303239333162663737656535643530343732623266346564336432353239613631
+65643962346563313836663262353161333537393235323236303039343662393930636532636461
-37366536373332323338
+36383935386531623838663432613463336331396537303031363837633537666133366638663961
+38316363613836333439386662663738333261326162316339616166643665633362333432643731
+6131366133663362633964633134353761663564376163656139

group_vars/infra/firewall.yml

@@ -100,7 +100,21 @@ firewall__zones:
       - 10.206.1.6
 
 firewall__input:
-  - verdict: accept
+  - src: back
+    verdict: accept
+  - src: monit
+    protocols:
+      tcp:
+        dport: 9100
+    verdict: accept
+  - protocols:
+      icmp: true
+    verdict: accept
+  - protocols:
+      tcp:
+        dport: 22
+    verdict: accept
+  - verdict: drop
 
 firewall__output:
   - verdict: accept
@@ -117,14 +131,18 @@ firewall__forward:
     protocols:
       icmp: true
     verdict: accept
+  - src: monit
+    dst: sw
+    verdict: accept
   - src: adm-legacy
     dst: bmc
     verdict: accept
+  # Prometheus for Grafana
   - src: grafana.adm
     dst: prometheus.int
     protocols:
       tcp:
-        dport: 9100
+        dport: 9090
     verdict: accept
   # Admin VPN clients
   - src: vpn-clients
@@ -132,11 +150,25 @@ firewall__forward:
     verdict: accept
   # Prometheus node
   - src: monit
-    dst: int
+    dst: infra
     protocols:
       tcp:
         dport: 9100
     verdict: accept
+  # Prometheus bird
+  - src: monit
+    dst: back
+    protocols:
+      tcp:
+        dport: 9324
+    verdict: accept
+  # Prometheus kresd
+  - src: monit
+    dst: dns.int
+    protocols:
+      tcp:
+        dport: 8453
+    verdict: accept
   # Allow DNS from infra to dns-{1,2}
   - src: infra
     dst: dns.int

group_vars/prom/prometheus.yml

@@ -0,0 +1,242 @@
+---
+prometheus__alertmanager_targets:
+  - docker-ovh.adm.auro.re:9093
+
+prometheus__tsdb_retention_time: 90d
+
+prometheus__scraping:
+  node:
+    targets: "{{ groups.vm_network + groups.pve_network }}"
+    address:
+      port: 9100
+  prometheus:
+    targets: "{{ groups.prom }}"
+    address:
+      port: 9090
+  kresd:
+    targets: "{{ groups.dns }}"
+    address:
+      port: 8453
+  bird:
+    targets: "{{ groups.router }}"
+    address:
+      port: 9324
+  quanta:
+    targets: "{{ groups.quanta }}"
+    address: 127.0.0.1:9116
+    path: /snmp
+    params:
+      module:
+        - quanta
+  snmp:
+    targets: "{{ groups.prom }}"
+    address:
+      port: 9116
+
+prometheus__alert_rules_prometheus:
+  - alert: PrometheusTsdbCompactionFailed
+    expr: "increase(prometheus_tsdb_compactions_failed_total[1m]) > 0"
+    for: 0m
+    labels:
+      severity: critical
+
+prometheus__alert_rules_node:
+  - alert: MachineDown
+    expr: "up == 0"
+    for: 3m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Collecteur {{ '$labels.job' | interp }}"
+  - alert: OutOfMemory
+    expr: "( node_memory_MemFree_bytes
+             + node_memory_Cached_bytes
+             + node_memory_Buffers_bytes )
+           / node_memory_MemTotal_bytes * 100 < 10"
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Mémoire libre à {{ '$value' | interp_float }}%"
+  - alert: HostSwapIsFillingUp
+    expr: "( 1 - ( node_memory_SwapFree_bytes
+                   / node_memory_SwapTotal_bytes ) )
+           * 100 >= 50"
+    for: 3m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Swap {{ '$value' | interp_float }}%"
+  - alert: HostPhysicalComponentTooHot
+    expr: "node_hwmon_temp_celsius > 79"
+    for: 3m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ '$value' | interp_float }}°C :
+                {{ '$labels.chip' | interp }},
+                {{ '$labels.sensor' | interp }}"
+  - alert: HostNodeOvertemperatureAlarm
+    expr: "node_hwmon_temp_crit_alarm_celsius == 1"
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ '$labels.chip' | interp }},
+                {{ '$labels.sensor' | interp }}"
+  - alert: HostRaidArrayGotInactive
+    expr: 'node_md_state{state="inactive"} > 0'
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ '$labels.device' | interp }}"
+  - alert: HostRaidDiskFailure
+    expr: 'node_md_disks{state="failed"} > 0'
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      severity: "{{ '$labels.md_device' | interp }}"
+  - alert: HostOomKillDetected
+    expr: "increase(node_vmstat_oom_kill[1m]) > 0"
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "PID {{ '$value' | interp }}"
+  - alert: HostEdacCorrectableErrorsDetected
+    expr: "increase(node_edac_correctable_errors_total[1m]) > 0"
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$value' | interp }} erreurs corrigées"
+  - alert: HostEdacUncorrectableErrorsDetected
+    expr: "increase(node_edac_uncorrectable_errors_total[1m]) > 0"
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$value' | interp }} erreurs corrigées"
+  - alert: OutOfDiskSpace
+    expr: "( node_filesystem_free_bytes
+             / node_filesystem_size_bytes * 100 < 10 )
+           and on (instance, device, mountpoint)
+            node_filesystem_readonly == 0"
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ '$labels.mountpoint' | interp }} :
+                {{ '$value' | interp_float }}% libre"
+  - alert: HostConntrackLimit
+    expr: "( node_nf_conntrack_entries
+             / node_nf_conntrack_entries_limit ) * 100 > 80"
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$value' | interp_float }}% complet"
+  - alert: HostClockSkew
+    expr: "(node_timex_offset_seconds > 0.05
+           and deriv(node_timex_offset_seconds[5m]) >= 0)
+           or (node_timex_offset_seconds < -0.05
+           and deriv(node_timex_offset_seconds[5m]) <= 0)"
+    for: 2m
+    labels:
+      severity: warning
+  - alert: HostClockNotSynchronising
+    expr: "min_over_time(node_timex_sync_status[1m]) == 0
+           and node_timex_maxerror_seconds >= 16"
+    for: 2m
+    labels:
+      severity: warning
+  - alert: HostRequiresReboot
+    expr: "node_reboot_required > 0"
+    for: 5m
+    labels:
+      severity: warning
+  - alert: OutOfInodes
+    expr: "node_filesystem_files_free
+           / node_filesystem_files * 100 < 10"
+    for: 3m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$labels.mountpoint' | interp }} :
+                {{ '$value' | interp_float }}% libre"
+  - alert: CpuUsage
+    expr: '( 100 - avg by (instance)
+             ( irate(node_cpu_seconds_total{mode="idle"}[5m]) ) 
+             * 100 ) > 75'
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$value' | interp_float }}%"
+  - alert: SystemdServiceFailed
+    expr: 'node_systemd_unit_state{state="failed"} == 1'
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$labels.name' | interp }}"
+  - alert: LoadUsage
+    expr: "node_load1 > 5"
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$value' | interp_float }}"
+  - alert: UnhealthyDisk
+    expr: "smartmon_device_smart_healthy < 1"
+    for: 10m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ '$labels.disk' | interp }}"
+  - alert: HostCpuStealNoisyNeighbor
+    expr: 'avg by(instance)
+            (rate(node_cpu_seconds_total{mode="steal"}[5m]))
+            * 100 > 10'
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$labels.disk' | interp }}"
+
+prometheus__alert_rules_keepalived:
+  - alert: KeepalivedVrrpFault
+    expr: 'keepalived_vrrp_state{state="fault"} > 0'
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ '$labels.instance' | interp }}"
+  - alert: KeepalivedMasterChange
+    expr: 'changes(
+             keepalived_vrrp_state
+               {keepalived_vvrp_state="master"}[1m]) > 1'
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ '$labels.instance' | interp }}"
+
+prometheus__alert_rules_bird:
+  - alert: BirdProtocolDown
+    expr: "bird_protocol_up == 0"
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ '$labels.name' | interp }} :
+                {{ '$labels.state' | interp }}"
+
+prometheus__alert_rules:
+  prometheus: "{{ prometheus__alert_rules_prometheus }}"
+  node: "{{ prometheus__alert_rules_node }}"
+  keepalived: "{{ prometheus__alert_rules_keepalived }}"
+  #bird: "{{ prometheus__alert_rules_bird }}"
+...

group_vars/prom/prometheus_snmp.yml

@@ -0,0 +1,35 @@
+---
+prometheus_snmp__modules:
+  quanta:
+    auth:
+      community: "{{ vault_snmp_quanta_community }}"
+    walk:
+      - interfaces
+      - ifXTable
+      - snAgGblQueueOverflow
+      - snAgGblDynMemTotal
+      - snAgGblDynMemFree
+      - snAgGblCpuUtil1SecAvg
+      - snAgGblCpuUtil5SecAvg
+      - snAgGblCpuUtil1MinAvg
+      - sysUpTime
+      - snAgentCpuUtilPercent
+      - snAgent
+    lookups:
+      - source_indexes:
+          - ifIndex
+        lookup: ifAlias
+      - source_indexes:
+          - ifIndex
+        lookup: ifDescr
+      - source_indexes:
+          - ifIndex
+        lookup: ifName
+    overrides:
+      ifAlias:
+        ignore: true
+      ifDescr:
+        ignore: true
+      ifName:
+        ignore: true
+...

host_vars/ns-master.int.infra.auro.re/knotd.yml

@@ -417,6 +417,9 @@ knotd__zones:
       prometheus-2.monit:
         - 2a09:6840:204::1:2
         - 10.204.1.2
+      ff-1.core.sw:
+        #- 2a09:6840:207::1:1
+        - 10.207.1.1
 
   108.66.45.in-addr.arpa:
     dnssec_policy: ripe

hosts

@@ -1,5 +1,11 @@
 # Aurore servers inventory
 
+[aruba]
+eb-1.acs.sw.infra.auro.re
+
+[quanta]
+ff-1.core.sw.infra.auro.re
+
 [vpn]
 #vpn-[1:2].back.infra.auro.re
 
@@ -19,9 +25,14 @@ isp-1.back.infra.auro.re
 [infra]
 infra-[1:2].back.infra.auro.re
 
-[prometheus]
+[prom]
 prometheus-[1:2].monit.infra.auro.re
 
+[router:children]
+isp
+infra
+edge
+
 [ldap]
 #ldap-[1:2].int.infra.auro.re
 
@@ -39,7 +50,7 @@ dns-[1:2].int.infra.auro.re
 radius-[1:2].isp.infra.auro.re
 ntp-[1:2].int.infra.auro.re
 #ldap-[1:2].int.infra.auro.re
-#isp-[1:2].back.infra.auro.re
+isp-[1:2].back.infra.auro.re
 infra-[1:2].back.infra.auro.re
 prometheus-[1:2].monit.infra.auro.re
 

playbooks/prometheus.yml

@@ -3,264 +3,19 @@
 - hosts:
     - pve_network
     - vm_network
-  vars:
-    prometheus_node__text_dir: /var/run/prometheus-node-exporter
   roles:
     - prometheus_node
 
 - hosts:
-    - edge-1.back.infra.auro.re
+    - router
-    - edge-2.back.infra.auro.re
-    - isp-1.back.infra.auro.re
-    - isp-2.back.infra.auro.re
-    - infra-1.back.infra.auro.re
-    - infra-2.back.infra.auro.re
-  vars:
-    prometheus_keepalived__dest: /var/run/prometheus-node-exporter/keepalived.prom
   roles:
     - prometheus_keepalived
-
-- hosts:
-    - edge-1.back.infra.auro.re
-    - edge-2.back.infra.auro.re
-    - isp-1.back.infra.auro.re
-    - isp-2.back.infra.auro.re
-    - infra-1.back.infra.auro.re
-    - infra-2.back.infra.auro.re
-  roles:
     - prometheus_bird
 
 - hosts:
-    - prometheus-1.monit.infra.auro.re
+    - prom
-    - prometheus-2.monit.infra.auro.re
-  vars:
-    prometheus__alertmanager_targets:
-      - docker-ovh.adm.auro.re:9093
-    prometheus__tsdb_retention_time: 90d
-    prometheus__scraping:
-      node:
-        targets: "{{ ['vm_network', 'pve_network']
-                     | map('extract', groups)
-                     | flatten }}"
-        address:
-          port: 9100
-      prometheus:
-        targets:
-          - prometheus-1.monit.infra.auro.re
-          - prometheus-2.monit.infra.auro.re
-        address:
-          port: 9090
-      kresd:
-        targets:
-          - dns-1.int.infra.auro.re
-          - dns-2.int.infra.auro.re
-        address:
-          port: 8453
-      bird:
-        targets:
-          - edge-1.back.infra.auro.re
-          - edge-2.back.infra.auro.re
-        address:
-          port: 9324
-    prometheus__alert_rules:
-      prometheus:
-        - alert: PrometheusTsdbCompactionFailed
-          expr: "increase(prometheus_tsdb_compactions_failed_total[1m]) > 0"
-          for: 0m
-          labels:
-            severity: critical
-      node:
-        - alert: MachineDown
-          expr: "up == 0"
-          for: 3m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Collecteur {{ '$labels.job' | interp }}"
-        - alert: OutOfMemory
-          expr: "( node_memory_MemFree_bytes
-                   + node_memory_Cached_bytes
-                   + node_memory_Buffers_bytes )
-                 / node_memory_MemTotal_bytes * 100 < 10"
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Mémoire libre à {{ '$value' | interp_float }}%"
-        - alert: HostSwapIsFillingUp
-          expr: "( 1 - ( node_memory_SwapFree_bytes
-                         / node_memory_SwapTotal_bytes ) )
-                 * 100 >= 50"
-          for: 3m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Swap {{ '$value' | interp_float }}%"
-        - alert: HostPhysicalComponentTooHot
-          expr: "node_hwmon_temp_celsius > 79"
-          for: 3m
-          labels:
-            severity: critical
-          annotations:
-            summary: "{{ '$value' | interp_float }}°C :
-                      {{ '$labels.chip' | interp }},
-                      {{ '$labels.sensor' | interp }}"
-        - alert: HostNodeOvertemperatureAlarm
-          expr: "node_hwmon_temp_crit_alarm_celsius == 1"
-          for: 0m
-          labels:
-            severity: critical
-          annotations:
-            summary: "{{ '$labels.chip' | interp }},
-                      {{ '$labels.sensor' | interp }}"
-        - alert: HostRaidArrayGotInactive
-          expr: 'node_md_state{state="inactive"} > 0'
-          for: 0m
-          labels:
-            severity: critical
-          annotations:
-            summary: "{{ '$labels.device' | interp }}"
-        - alert: HostRaidDiskFailure
-          expr: 'node_md_disks{state="failed"} > 0'
-          for: 0m
-          labels:
-            severity: critical
-          annotations:
-            severity: "{{ '$labels.md_device' | interp }}"
-        - alert: HostOomKillDetected
-          expr: "increase(node_vmstat_oom_kill[1m]) > 0"
-          for: 0m
-          labels:
-            severity: warning
-          annotations:
-            summary: "PID {{ '$value' | interp }}"
-        - alert: HostEdacCorrectableErrorsDetected
-          expr: "increase(node_edac_correctable_errors_total[1m]) > 0"
-          for: 0m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$value' | interp }} erreurs corrigées"
-        - alert: HostEdacUncorrectableErrorsDetected
-          expr: "increase(node_edac_uncorrectable_errors_total[1m]) > 0"
-          for: 0m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$value' | interp }} erreurs corrigées"
-        - alert: OutOfDiskSpace
-          expr: "( node_filesystem_free_bytes
-                   / node_filesystem_size_bytes * 100 < 10 )
-                 and on (instance, device, mountpoint)
-                  node_filesystem_readonly == 0"
-          for: 5m
-          labels:
-            severity: critical
-          annotations:
-            summary: "{{ '$labels.mountpoint' | interp }} :
-                      {{ '$value' | interp_float }}% libre"
-        - alert: HostConntrackLimit
-          expr: "( node_nf_conntrack_entries
-                   / node_nf_conntrack_entries_limit ) * 100 > 80"
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$value' | interp_float }}% complet"
-        - alert: HostClockSkew
-          expr: "(node_timex_offset_seconds > 0.05
-                 and deriv(node_timex_offset_seconds[5m]) >= 0)
-                 or (node_timex_offset_seconds < -0.05
-                 and deriv(node_timex_offset_seconds[5m]) <= 0)"
-          for: 2m
-          labels:
-            severity: warning
-        - alert: HostClockNotSynchronising
-          expr: "min_over_time(node_timex_sync_status[1m]) == 0
-                 and node_timex_maxerror_seconds >= 16"
-          for: 2m
-          labels:
-            severity: warning
-        - alert: HostRequiresReboot
-          expr: "node_reboot_required > 0"
-          for: 5m
-          labels:
-            severity: warning
-        - alert: OutOfInodes
-          expr: "node_filesystem_files_free
-                 / node_filesystem_files * 100 < 10"
-          for: 3m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$labels.mountpoint' | interp }} :
-                      {{ '$value' | interp_float }}% libre"
-        - alert: CpuUsage
-          expr: '( 100 - avg by (instance)
-                   ( irate(node_cpu_seconds_total{mode="idle"}[5m]) ) 
-                   * 100 ) > 75'
-          for: 10m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$value' | interp_float }}%"
-        - alert: SystemdServiceFailed
-          expr: 'node_systemd_unit_state{state="failed"} == 1'
-          for: 10m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$labels.name' | interp }}"
-        - alert: LoadUsage
-          expr: "node_load1 > 5"
-          for: 2m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$value' | interp_float }}"
-        - alert: UnhealthyDisk
-          expr: "smartmon_device_smart_healthy < 1"
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: "{{ '$labels.disk' | interp }}"
-        - alert: HostCpuStealNoisyNeighbor
-          expr: 'avg by(instance)
-                  (rate(node_cpu_seconds_total{mode="steal"}[5m]))
-                  * 100 > 10'
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$labels.disk' | interp }}"
-      keepalived:
-        - alert: KeepalivedVrrpFault
-          expr: 'keepalived_vrrp_state{state="fault"} > 0'
-          for: 0m
-          labels:
-            severity: critical
-          annotations:
-            summary: "{{ '$labels.instance' | interp }}"
-        - alert: KeepalivedMasterChange
-          expr: 'changes(
-                   keepalived_vrrp_state
-                     {keepalived_vvrp_state="master"}[1m]) > 1'
-          for: 0m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ '$labels.instance' | interp }}"
-      #bird:
-      #  - alert: BirdProtocolDown
-      #    expr: "bird_protocol_up == 0"
-      #    for: 0m
-      #    labels:
-      #      severity: critical
-      #    annotations:
-      #      summary: "{{ '$labels.name' | interp }} :
-      #                {{ '$labels.state' | interp }}"
   roles:
+    - prometheus_snmp
     - prometheus
 
 #- hosts: prometheus-fleming.adm.auro.re

roles/prometheus_snmp/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+prometheus_snmp__modules: {}
+prometheus_snmp__mib_dirs:
+  - /usr/share/snmp/mibs
+  - /usr/share/snmp/mibs/iana
+  - /usr/share/snmp/mibs/ietf
+  - /usr/local/share/snmp/mibs
+...

roles/prometheus_snmp/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart prometheus-snmp-exporter
+  systemd:
+    name: prometheus-snmp-exporter.service
+    state: restarted
+...

roles/prometheus_snmp/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+- name: Install prometheus-snmp-exporter
+  apt:
+    name:
+      - prometheus-snmp-exporter
+      - snmp-mibs-downloader
+
+- name: Create local MIB directory
+  file:
+    path: /usr/local/share/snmp/mibs
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=rx
+
+- name: Install MIBs
+  copy:
+    src: "mibs/{{ item }}"
+    dest: "/usr/local/share/snmp/mibs/{{ item }}"
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  loop:
+    - FOUNDRY-SN-AGENT-MIB.txt
+    - FOUNDRY-SN-ROOT-MIB.txt
+    - FOUNDRY-SN-SWITCH-GROUP-MIB.txt
+
+- name: Configure prometheus-snmp-exporter generator
+  template:
+    src: generator.yml.j2
+    dest: /etc/prometheus/generator.yml
+    owner: prometheus
+    group: prometheus
+    mode: u=rw,g=,o=
+  vars:
+    prometheus_snmp__config:
+      modules: "{{ prometheus_snmp__modules }}"
+  register: generator
+
+- name: Generate snmp.yml
+  command:
+    cmd: prometheus-snmp-generator generate
+    chdir: /etc/prometheus
+  environment:
+    MIBDIRS: "{{ prometheus_snmp__mib_dirs | join(':') }}"
+  when: generator.changed
+  notify:
+    - Restart prometheus-snmp-exporter
+
+- name: Enable prometheus-snmp-exporter
+  systemd:
+    name: prometheus-snmp-exporter.service
+    enabled: true
+...

roles/prometheus_snmp/templates/generator.yml.j2

@@ -0,0 +1,5 @@
+---
+{{ ansible_managed | comment }}
+
+{{ prometheus_snmp__config | to_nice_yaml }}
+...