Centralisation des journaux (pas encore Elastic) #40

Manually merged
jeltz merged 24 commits from logs-first-phase into master 2024-07-01 01:55:28 +02:00
30 changed files with 486 additions and 2 deletions

View file

@ -2,6 +2,7 @@ skip_list:
- no-changed-when
- load-failure
- document-start
- meta-no-info
warn_list:
- experimental # all rules tagged as experimental

View file

@ -97,3 +97,9 @@ apartment_block_dhcp: "{{ apartment_block }}"
ipv6_base_prefix: "2a09:6840"
is_aurore_host: "{{ 'aurore_vm' in group_names }}"
rsyslog_outputs:
- proto: relp
address: 10.128.0.241
port: 20514
...

View file

@ -0,0 +1,9 @@
---
rsyslog_collector_base_dir: /var/log/remote
rsyslog_inputs:
- proto: relp
port: 20514
- proto: udp
port: 514
rsyslog_outputs: []
...

1
hosts
View file

@ -36,6 +36,7 @@ wikijs.adm.auro.re
prometheus-aurore.adm.auro.re
portail.adm.auro.re
jitsi-aurore.adm.auro.re
log.adm.auro.re
bdd.adm.auro.re
bdd-ovh.adm.auro.re

5
log.yml Normal file
View file

@ -0,0 +1,5 @@
---
- hosts: log.adm.auro.re
roles:
- rsyslog_collector
...

View file

@ -29,6 +29,24 @@
dest: "/etc/nginx/sites-enabled/default"
state: absent
- name: Add 'extended' log format
template:
src: nginx/conf.d/extended_log.conf.j2
dest: /etc/nginx/conf.d/extended_log.conf
owner: root
group: root
mode: 0644
notify: Reload nginx
- name: Add syslog snippet
template:
src: nginx/snippets/syslog.conf.j2
dest: /etc/nginx/snippets/syslog.conf
owner: root
group: root
mode: 0644
notify: Reload nginx
- name: Copy reverse proxy sites
when: reverseproxy is defined
template:

View file

@ -0,0 +1,7 @@
{{ ansible_managed | comment }}
log_format extended
'$remote_addr - $http_x_forwarded_for - $connection '
'$remote_user [$time_local] '
'"$host" "$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';

View file

@ -8,6 +8,8 @@ server {
server_name {{ site.from }};
include "/etc/nginx/snippets/syslog.conf";
{% for realip in nginx.real_ip_from %}
set_real_ip_from {{ realip }};
{% endfor %}
@ -25,6 +27,8 @@ server {
server_name {{ site.from }};
include "/etc/nginx/snippets/syslog.conf";
# SSL common conf
include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf";
@ -52,6 +56,8 @@ server {
server_name {{ from }};
include "/etc/nginx/snippets/syslog.conf";
{% for realip in nginx.real_ip_from %}
set_real_ip_from {{ realip }};
{% endfor %}
@ -72,6 +78,8 @@ server {
# SSL common conf
include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf";
include "/etc/nginx/snippets/syslog.conf";
{% for realip in nginx.real_ip_from %}
set_real_ip_from {{ realip }};
{% endfor %}

View file

@ -15,6 +15,8 @@ server {
server_name {{ site.from }};
include "/etc/nginx/snippets/syslog.conf";
{% for realip in nginx.real_ip_from %}
set_real_ip_from {{ realip }};
{% endfor %}
@ -39,6 +41,8 @@ server {
access_log /var/log/nginx/{{ site.from }}.log;
error_log /var/log/nginx/{{ site.from }}_error.log;
include "/etc/nginx/snippets/syslog.conf";
# Keep the TCP connection open a bit for faster browsing
keepalive_timeout 70;

View file

@ -12,6 +12,8 @@ server {
server_name {{ from }};
include "/etc/nginx/snippets/syslog.conf";
{% for realip in nginx.real_ip_from %}
set_real_ip_from {{ realip }};
{% endfor %}
@ -29,6 +31,8 @@ server {
server_name {{ from }};
include "/etc/nginx/snippets/syslog.conf";
# SSL common conf
include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf";

View file

@ -19,6 +19,9 @@ upstream {{ upstream.name }} {
server {
listen 443 default_server ssl;
listen [::]:443 default_server ssl;
include "/etc/nginx/snippets/syslog.conf";
include "/etc/nginx/snippets/options-ssl.{{ nginx.default_ssl_domain }}.conf";
server_name _;
@ -50,6 +53,8 @@ server {
# Hide Nginx version
server_tokens off;
include "/etc/nginx/snippets/syslog.conf";
{% for realip in nginx.real_ip_from %}
set_real_ip_from {{ realip }};
{% endfor %}
@ -71,6 +76,8 @@ server {
server_name {{ server.server_name|join(" ") }};
charset utf-8;
include "/etc/nginx/snippets/syslog.conf";
# Hide Nginx version
server_tokens off;
@ -98,6 +105,8 @@ server {
server_name {{ server.server_name|join(" ") }};
charset utf-8;
include "/etc/nginx/snippets/syslog.conf";
# Hide Nginx version
server_tokens off;

View file

@ -0,0 +1,4 @@
{{ ansible_managed | comment }}
access_log syslog:server=unix:/dev/log,tag=nginx,nohostname,severity=info extended;
error_log syslog:server=unix:/dev/log,tag=nginx,nohostname,severity=error;

View file

@ -36,6 +36,11 @@ interfaces_type = {
'admin' : ['ens18']
}
log_ignore_v4 = [
'224.0.0.0/24',
'224.0.1.0/24',
'239.0.0.0/8',
]
### Specify nat settings: name, interfaces with range, and global range for nat
### WARNING : "interface_ip_to_nat' MUST contain /24 ranges, and ip_sources MUST

View file

@ -33,6 +33,12 @@ interfaces_type = {
'admin' : ['ens19', 'ens20', 'ens23']
}
log_ignore_v4 = [
'224.0.0.0/24',
'224.0.1.0/24',
'239.0.0.0/8',
]
### Specify nat settings: name, interfaces with range, and global range for nat
### WARNING : "interface_ip_to_nat' MUST contain /24 ranges, and ip_sources MUST
### contain /16 range

View file

@ -11,7 +11,6 @@ iface lo inet loopback
auto ens18
iface ens18 inet static
address 10.129.0.{{ router_hard_ip_suffix }}/16
gateway 10.129.0.1
iface ens18 inet6 static
address 2a09:6840:129::0:{{ router_hard_ip_suffix }}/64

View file

@ -39,7 +39,7 @@ vrrp_instance VI_ROUT_aurore_IPv4 {
10.129.0.254/16 brd 10.129.255.255 dev ens18 scope global
# Adm
10.128.0.254/16 brd 10.129.255.255 dev ens19 scope global
10.128.0.254/16 brd 10.128.255.255 dev ens19 scope global
# Switches
10.130.0.254/16 brd 10.130.255.255 dev ens20 scope global

View file

@ -0,0 +1,7 @@
---
rsyslog_inputs: []
rsyslog_collector_base_dir: /var/log/remote
rsyslog_collector_rotate_path: /usr/local/sbin/rotate_remote_logs
rsyslog_collector_keep_days: 0
rsyslog_collector_compress_days: 1
...

View file

@ -0,0 +1,60 @@
#!/usr/bin/env python3
import argparse
import datetime
import logging
import pathlib
import subprocess
def compress_file(filename):
subprocess.run(["xz", "-z", str(filename)])
def find_files(base_dir, extension, days):
delta = datetime.timedelta(days=days)
now = datetime.datetime.now()
for path in base_dir.rglob(f"*{extension}"):
stem = path.name.removesuffix(extension)
date = datetime.datetime.fromisoformat(stem)
if date < now - delta:
yield path
def compress_logs(base_dir, days):
for path in find_files(base_dir, ".log", days):
logging.info("Compressing log file %s", str(path))
compress_file(path)
def remove_logs(base_dir, days):
for path in find_files(base_dir, ".log.xz", days):
logging.info("Removing log file %s", str(path))
path.unlink()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--compress-days", type=int, default=0)
parser.add_argument("--keep-days", type=int, default=0)
parser.add_argument(
"--base-dir", type=pathlib.Path, default="/var/log/remote"
)
args = parser.parse_args()
logging.basicConfig(format="%(levelname)s %(message)s", level=logging.INFO)
logging.info("Rotate script started")
if args.compress_days > 0:
compress_logs(args.base_dir, args.compress_days)
if args.keep_days > 0:
remove_logs(args.base_dir, args.keep_days)
logging.info("Rotate script done")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,5 @@
---
- name: Run systemd daemon-reload
systemd:
daemon_reload: true
...

View file

@ -0,0 +1,4 @@
---
dependencies:
- role: rsyslog_common
...

View file

@ -0,0 +1,57 @@
---
- name: Install rsyslog-relp if needed
become: true
apt:
name: rsyslog-relp
state: present
when: "rsyslog_inputs | selectattr('proto', 'eq', 'relp') | list"
- name: Ensure log storage directory exists
become: true
file:
path: "{{ rsyslog_collector_base_dir }}"
state: directory
owner: root
group: adm
mode: u=rwx,g=rwx,o=
- name: Deploy rsyslog input configuration file
become: true
template:
src: 20-collector.conf.j2
dest: /etc/rsyslog.d/20-collector.conf
owner: root
group: root
mode: u=rw,g=r,o=r
notify: Restart rsyslog
- name: Install rotate script
become: true
copy:
src: rotate
dest: "{{ rsyslog_collector_rotate_path }}"
owner: root
group: root
mode: u=rwx,g=rx,o=
- name: Install timer and service for rotate script
become: true
template:
src: "{{ item }}.j2"
dest: "/etc/systemd/system/{{ item }}"
owner: root
group: root
mode: u=rw,g=r,o=
loop:
- rotate-remote-logs.timer
- rotate-remote-logs.service
notify:
- Run systemd daemon-reload
- name: Enable timer for log rotation
become: true
systemd:
name: rotate-remote-logs.timer
enabled: true
state: started
...

View file

@ -0,0 +1,54 @@
{{ ansible_managed | comment }}
module(load="mmrm1stspace")
{%
set input_modules = {
"relp": "imrelp",
"udp": "imudp",
}
%}
{%
for module in rsyslog_inputs
| map(attribute="proto")
| map("extract", input_modules)
| list
| unique
%}
module(load="{{ module }}")
{% endfor %}
template(name="incomingFilename" type="list") {
constant(value="{{ rsyslog_collector_base_dir }}/")
property(name="fromhost-ip")
constant(value="/")
property(name="timegenerated" dateFormat="year")
constant(value="-")
property(name="timegenerated" dateFormat="month")
constant(value="-")
property(name="timegenerated" dateFormat="day")
constant(value=".log")
}
ruleset(name="handleIncomingLogs") {
action(type="mmrm1stspace")
action(
type="omfile"
dynaFile="incomingFilename"
template="RSYSLOG_FileFormat"
)
call sendLogsToRemote
}
# TODO: add protocol-specific options (eg. TLS)
{% for input in rsyslog_inputs %}
input(
type="{{ input_modules[input.proto] }}"
{% if "address" in input %}
address="{{ input.address }}"
{% endif %}
port="{{ input.port }}"
ruleset="handleIncomingLogs"
)
{% endfor %}

View file

@ -0,0 +1,12 @@
{{ ansible_managed | comment }}
[Unit]
Description=Rotate remote logs
[Service]
User=root
Type=simple
ExecStart={{ rsyslog_collector_rotate_path }} \
--base-dir {{ rsyslog_collector_base_dir }} \
--compress-days {{ rsyslog_collector_compress_days }} \
--keep-days {{ rsyslog_collector_keep_days }}

View file

@ -0,0 +1,10 @@
{{ ansible_managed | comment }}
[Unit]
Description=Rotate remote logs daily
[Timer]
OnCalendar=daily
[Install]
WantedBy=timers.target

View file

@ -0,0 +1,3 @@
---
rsyslog_outputs: []
...

View file

@ -0,0 +1,13 @@
---
- name: Restart rsyslog
become: true
systemd:
name: rsyslog.service
state: restarted
- name: Restart systemd-journald
become: true
systemd:
name: systemd-journald.service
state: restarted
...

View file

@ -0,0 +1,60 @@
---
- name: Install rsyslog
become: true
apt:
name: rsyslog
state: present
- name: Install rsyslog modules if needed
become: true
apt:
name: "{{ item.pkg }}"
state: present
when: "rsyslog_outputs | selectattr('proto', 'eq', item.proto) | list"
loop:
- proto: relp
pkg: rsyslog-relp
- proto: redis
pkg: rsyslog-hiredis
- name: Deploy main rsyslog configuration
become: true
template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: root
group: root
mode: u=rw,g=r,o=r
loop:
- src: rsyslog.conf.j2
dest: /etc/rsyslog.conf
- src: 10-common.conf.j2
dest: /etc/rsyslog.d/10-common.conf
notify: Restart rsyslog
- name: Create journald.conf.d directory
become: true
file:
path: /etc/systemd/journald.conf.d
state: directory
owner: root
group: root
mode: u=rwx,g=rx,o=rx
- name: Deploy journald configuration
become: true
template:
src: forward-syslog.conf.j2
dest: /etc/systemd/journald.conf.d/forward-syslog.conf
owner: root
group: root
mode: u=rw,g=r,o=r
notify: Restart systemd-journald
- name: Enable rsyslog service
become: true
systemd:
name: rsyslog.service
state: started
enabled: true
...

View file

@ -0,0 +1,105 @@
{{ ansible_managed | comment }}
{%
set output_modules = {
"relp": "omrelp",
"udp": "omfwd",
"redis": "omhiredis",
}
%}
global(
workDirectory="/var/spool/rsyslog"
preserveFQDN="on"
)
# Collect logs via /dev/log
module(load="imuxsock")
# Collect kernel logs
module(load="imklog")
# Parse CEE logs
module(load="mmjsonparse")
# Load export modules
{%
for module in rsyslog_outputs
| map(attribute="proto")
| map("extract", output_modules)
| list
| unique
%}
module(load="{{ module }}")
{% endfor %}
# FIXME: Attention, il faut voir si rsyslog arrive bien à créer
# les fichiers de plusieurs jours (le 1er est peut-être crée avant
# de dropper les privilèges, mais les suivants je pense pas).
module(
load="builtin:omfile"
# Format avec dates précises
template="RSYSLOG_FileFormat"
fileOwner="root"
fileGroup="adm"
fileCreateMode="0640"
dirCreateMode="0755"
)
template(name="templateJson" type="list" option.jsonf="on") {
property(outname="hostname_reported" name="hostname" format="jsonf")
property(outname="src" name="fromhost-ip" format="jsonf")
property(outname="facility" name="syslogfacility-text" format="jsonf")
property(outname="program" name="programname" format="jsonf")
property(outname="pid" name="procid" format="jsonf")
property(outname="time_reported" name="timereported" format="jsonf"
dateformat="rfc3339")
property(outname="time_generated" name="timegenerated" format="jsonf"
dateformat="rfc3339")
property(outname="message" name="msg" format="jsonf")
}
ruleset(name="sendLogsToDisk") {
auth,authpriv.* action(type="omfile" file="/var/log/auth.log")
mail.* action(type="omfile" file="/var/log/mail.log" sync="off")
kern.* action(type="omfile" file="/var/log/kern.log")
*.*;auth,authpriv,mail,kern.none action(type="omfile"
file="/var/log/syslog.log" sync="off")
}
# Send logs to remote collector(s)
ruleset(name="sendLogsToRemote") {
{% for output in rsyslog_outputs %}
action(
type="{{ output_modules[output.proto] }}"
{% if output_modules[output.proto] == "omfwd" %}
protocol="{{ output.proto }}"
target="{{ output.address }}"
port="{{ output.port }}"
{% elif output_modules[output.proto] == "omhiredis" %}
server="{{ output.address }}"
serverport="{{ output.port }}"
mode="publish"
key="{{ output.key }}"
template="templateJson"
{% if output.password is defined %}
serverpassword="{{ output.password }}"
{% endif %}
{% elif output_modules[output.proto] == "omrelp" %}
target="{{ output.address }}"
port="{{ output.port }}"
{% endif %}
{% if loop.index > 1 and output.fallback %}
action.execOnlyWhenPreviousIsSuspended="on"
{% endif %}
)
{% endfor %}
}
# Send local logs to files (useful for debugging or if the collector is down)
call sendLogsToDisk
# Send local logs to the remote collector
call sendLogsToRemote

View file

@ -0,0 +1,5 @@
{{ ansible_managed | comment }}
[Journal]
ForwardToSyslog=yes
MaxLevelSyslog=debug

View file

@ -0,0 +1,3 @@
{{ ansible_managed | comment }}
include(file="/etc/rsyslog.d/*.conf")