diff --git a/.ansible-lint b/.ansible-lint index d98efd4..0e01ba3 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -2,6 +2,7 @@ skip_list: - no-changed-when - load-failure - document-start + - meta-no-info warn_list: - experimental # all rules tagged as experimental diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index e19f6de..6244182 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -97,3 +97,9 @@ apartment_block_dhcp: "{{ apartment_block }}" ipv6_base_prefix: "2a09:6840" is_aurore_host: "{{ 'aurore_vm' in group_names }}" + +rsyslog_outputs: + - proto: relp + address: 10.128.0.241 + port: 20514 +... diff --git a/host_vars/log.adm.auro.re.yml b/host_vars/log.adm.auro.re.yml new file mode 100644 index 0000000..1c2ba97 --- /dev/null +++ b/host_vars/log.adm.auro.re.yml @@ -0,0 +1,9 @@ +--- +rsyslog_collector_base_dir: /var/log/remote +rsyslog_inputs: + - proto: relp + port: 20514 + - proto: udp + port: 514 +rsyslog_outputs: [] +... diff --git a/hosts b/hosts index 4191852..f075a8a 100644 --- a/hosts +++ b/hosts @@ -36,6 +36,7 @@ wikijs.adm.auro.re prometheus-aurore.adm.auro.re portail.adm.auro.re jitsi-aurore.adm.auro.re +log.adm.auro.re bdd.adm.auro.re bdd-ovh.adm.auro.re diff --git a/log.yml b/log.yml new file mode 100644 index 0000000..8c8fc15 --- /dev/null +++ b/log.yml @@ -0,0 +1,5 @@ +--- +- hosts: log.adm.auro.re + roles: + - rsyslog_collector +... diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index 210c7f0..a8fb885 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -29,6 +29,24 @@ dest: "/etc/nginx/sites-enabled/default" state: absent +- name: Add 'extended' log format + template: + src: nginx/conf.d/extended_log.conf.j2 + dest: /etc/nginx/conf.d/extended_log.conf + owner: root + group: root + mode: 0644 + notify: Reload nginx + +- name: Add syslog snippet + template: + src: nginx/snippets/syslog.conf.j2 + dest: /etc/nginx/snippets/syslog.conf + owner: root + group: root + mode: 0644 + notify: Reload nginx + - name: Copy reverse proxy sites when: reverseproxy is defined template: diff --git a/roles/nginx/templates/nginx/conf.d/extended_log.conf.j2 b/roles/nginx/templates/nginx/conf.d/extended_log.conf.j2 new file mode 100644 index 0000000..b28809f --- /dev/null +++ b/roles/nginx/templates/nginx/conf.d/extended_log.conf.j2 @@ -0,0 +1,7 @@ +{{ ansible_managed | comment }} + +log_format extended + '$remote_addr - $http_x_forwarded_for - $connection ' + '$remote_user [$time_local] ' + '"$host" "$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; diff --git a/roles/nginx/templates/nginx/sites-available/redirect.j2 b/roles/nginx/templates/nginx/sites-available/redirect.j2 index 2543400..9e8e687 100644 --- a/roles/nginx/templates/nginx/sites-available/redirect.j2 +++ b/roles/nginx/templates/nginx/sites-available/redirect.j2 @@ -8,6 +8,8 @@ server { server_name {{ site.from }}; + include "/etc/nginx/snippets/syslog.conf"; + {% for realip in nginx.real_ip_from %} set_real_ip_from {{ realip }}; {% endfor %} @@ -25,6 +27,8 @@ server { server_name {{ site.from }}; + include "/etc/nginx/snippets/syslog.conf"; + # SSL common conf include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf"; @@ -52,6 +56,8 @@ server { server_name {{ from }}; + include "/etc/nginx/snippets/syslog.conf"; + {% for realip in nginx.real_ip_from %} set_real_ip_from {{ realip }}; {% endfor %} @@ -72,6 +78,8 @@ server { # SSL common conf include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf"; + include "/etc/nginx/snippets/syslog.conf"; + {% for realip in nginx.real_ip_from %} set_real_ip_from {{ realip }}; {% endfor %} diff --git a/roles/nginx/templates/nginx/sites-available/reverseproxy.j2 b/roles/nginx/templates/nginx/sites-available/reverseproxy.j2 index ae2d7a6..699d6d5 100644 --- a/roles/nginx/templates/nginx/sites-available/reverseproxy.j2 +++ b/roles/nginx/templates/nginx/sites-available/reverseproxy.j2 @@ -15,6 +15,8 @@ server { server_name {{ site.from }}; + include "/etc/nginx/snippets/syslog.conf"; + {% for realip in nginx.real_ip_from %} set_real_ip_from {{ realip }}; {% endfor %} @@ -39,6 +41,8 @@ server { access_log /var/log/nginx/{{ site.from }}.log; error_log /var/log/nginx/{{ site.from }}_error.log; + include "/etc/nginx/snippets/syslog.conf"; + # Keep the TCP connection open a bit for faster browsing keepalive_timeout 70; diff --git a/roles/nginx/templates/nginx/sites-available/reverseproxy_redirect_dname.j2 b/roles/nginx/templates/nginx/sites-available/reverseproxy_redirect_dname.j2 index 819fd7a..f90d53b 100644 --- a/roles/nginx/templates/nginx/sites-available/reverseproxy_redirect_dname.j2 +++ b/roles/nginx/templates/nginx/sites-available/reverseproxy_redirect_dname.j2 @@ -12,6 +12,8 @@ server { server_name {{ from }}; + include "/etc/nginx/snippets/syslog.conf"; + {% for realip in nginx.real_ip_from %} set_real_ip_from {{ realip }}; {% endfor %} @@ -29,6 +31,8 @@ server { server_name {{ from }}; + include "/etc/nginx/snippets/syslog.conf"; + # SSL common conf include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf"; diff --git a/roles/nginx/templates/nginx/sites-available/service.j2 b/roles/nginx/templates/nginx/sites-available/service.j2 index 39f25eb..77c3d74 100644 --- a/roles/nginx/templates/nginx/sites-available/service.j2 +++ b/roles/nginx/templates/nginx/sites-available/service.j2 @@ -19,6 +19,9 @@ upstream {{ upstream.name }} { server { listen 443 default_server ssl; listen [::]:443 default_server ssl; + + include "/etc/nginx/snippets/syslog.conf"; + include "/etc/nginx/snippets/options-ssl.{{ nginx.default_ssl_domain }}.conf"; server_name _; @@ -50,6 +53,8 @@ server { # Hide Nginx version server_tokens off; + include "/etc/nginx/snippets/syslog.conf"; + {% for realip in nginx.real_ip_from %} set_real_ip_from {{ realip }}; {% endfor %} @@ -71,6 +76,8 @@ server { server_name {{ server.server_name|join(" ") }}; charset utf-8; + include "/etc/nginx/snippets/syslog.conf"; + # Hide Nginx version server_tokens off; @@ -98,6 +105,8 @@ server { server_name {{ server.server_name|join(" ") }}; charset utf-8; + include "/etc/nginx/snippets/syslog.conf"; + # Hide Nginx version server_tokens off; diff --git a/roles/nginx/templates/nginx/snippets/syslog.conf.j2 b/roles/nginx/templates/nginx/snippets/syslog.conf.j2 new file mode 100644 index 0000000..b34867c --- /dev/null +++ b/roles/nginx/templates/nginx/snippets/syslog.conf.j2 @@ -0,0 +1,4 @@ +{{ ansible_managed | comment }} + +access_log syslog:server=unix:/dev/log,tag=nginx,nohostname,severity=info extended; +error_log syslog:server=unix:/dev/log,tag=nginx,nohostname,severity=error; diff --git a/roles/router/templates/firewall_config.py b/roles/router/templates/firewall_config.py index 9971765..07e25e0 100644 --- a/roles/router/templates/firewall_config.py +++ b/roles/router/templates/firewall_config.py @@ -36,6 +36,11 @@ interfaces_type = { 'admin' : ['ens18'] } +log_ignore_v4 = [ + '224.0.0.0/24', + '224.0.1.0/24', + '239.0.0.0/8', +] ### Specify nat settings: name, interfaces with range, and global range for nat ### WARNING : "interface_ip_to_nat' MUST contain /24 ranges, and ip_sources MUST diff --git a/roles/router/templates/firewall_config_aurore.py b/roles/router/templates/firewall_config_aurore.py index 9565e3b..91a4808 100644 --- a/roles/router/templates/firewall_config_aurore.py +++ b/roles/router/templates/firewall_config_aurore.py @@ -33,6 +33,12 @@ interfaces_type = { 'admin' : ['ens19', 'ens20', 'ens23'] } +log_ignore_v4 = [ + '224.0.0.0/24', + '224.0.1.0/24', + '239.0.0.0/8', +] + ### Specify nat settings: name, interfaces with range, and global range for nat ### WARNING : "interface_ip_to_nat' MUST contain /24 ranges, and ip_sources MUST ### contain /16 range diff --git a/roles/router/templates/interfaces-aurore b/roles/router/templates/interfaces-aurore index 401e5aa..7a5ef1d 100644 --- a/roles/router/templates/interfaces-aurore +++ b/roles/router/templates/interfaces-aurore @@ -11,7 +11,6 @@ iface lo inet loopback auto ens18 iface ens18 inet static address 10.129.0.{{ router_hard_ip_suffix }}/16 - gateway 10.129.0.1 iface ens18 inet6 static address 2a09:6840:129::0:{{ router_hard_ip_suffix }}/64 diff --git a/roles/router/templates/keepalived-aurore.conf b/roles/router/templates/keepalived-aurore.conf index cd22a5b..b8882fd 100644 --- a/roles/router/templates/keepalived-aurore.conf +++ b/roles/router/templates/keepalived-aurore.conf @@ -39,7 +39,7 @@ vrrp_instance VI_ROUT_aurore_IPv4 { 10.129.0.254/16 brd 10.129.255.255 dev ens18 scope global # Adm - 10.128.0.254/16 brd 10.129.255.255 dev ens19 scope global + 10.128.0.254/16 brd 10.128.255.255 dev ens19 scope global # Switches 10.130.0.254/16 brd 10.130.255.255 dev ens20 scope global diff --git a/roles/rsyslog_collector/defaults/main.yml b/roles/rsyslog_collector/defaults/main.yml new file mode 100644 index 0000000..6ded0ef --- /dev/null +++ b/roles/rsyslog_collector/defaults/main.yml @@ -0,0 +1,7 @@ +--- +rsyslog_inputs: [] +rsyslog_collector_base_dir: /var/log/remote +rsyslog_collector_rotate_path: /usr/local/sbin/rotate_remote_logs +rsyslog_collector_keep_days: 0 +rsyslog_collector_compress_days: 1 +... diff --git a/roles/rsyslog_collector/files/rotate b/roles/rsyslog_collector/files/rotate new file mode 100644 index 0000000..6dc23e0 --- /dev/null +++ b/roles/rsyslog_collector/files/rotate @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import argparse +import datetime +import logging +import pathlib +import subprocess + + +def compress_file(filename): + subprocess.run(["xz", "-z", str(filename)]) + + +def find_files(base_dir, extension, days): + delta = datetime.timedelta(days=days) + now = datetime.datetime.now() + for path in base_dir.rglob(f"*{extension}"): + stem = path.name.removesuffix(extension) + date = datetime.datetime.fromisoformat(stem) + if date < now - delta: + yield path + + +def compress_logs(base_dir, days): + for path in find_files(base_dir, ".log", days): + logging.info("Compressing log file %s", str(path)) + compress_file(path) + + +def remove_logs(base_dir, days): + for path in find_files(base_dir, ".log.xz", days): + logging.info("Removing log file %s", str(path)) + path.unlink() + + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument("--compress-days", type=int, default=0) + parser.add_argument("--keep-days", type=int, default=0) + parser.add_argument( + "--base-dir", type=pathlib.Path, default="/var/log/remote" + ) + + args = parser.parse_args() + + logging.basicConfig(format="%(levelname)s %(message)s", level=logging.INFO) + + logging.info("Rotate script started") + + if args.compress_days > 0: + compress_logs(args.base_dir, args.compress_days) + + if args.keep_days > 0: + remove_logs(args.base_dir, args.keep_days) + + logging.info("Rotate script done") + + +if __name__ == "__main__": + main() diff --git a/roles/rsyslog_collector/handlers/main.yml b/roles/rsyslog_collector/handlers/main.yml new file mode 100644 index 0000000..60f493a --- /dev/null +++ b/roles/rsyslog_collector/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Run systemd daemon-reload + systemd: + daemon_reload: true +... diff --git a/roles/rsyslog_collector/meta/main.yml b/roles/rsyslog_collector/meta/main.yml new file mode 100644 index 0000000..8e7f44c --- /dev/null +++ b/roles/rsyslog_collector/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: rsyslog_common +... diff --git a/roles/rsyslog_collector/tasks/main.yml b/roles/rsyslog_collector/tasks/main.yml new file mode 100644 index 0000000..dc64fcc --- /dev/null +++ b/roles/rsyslog_collector/tasks/main.yml @@ -0,0 +1,57 @@ +--- +- name: Install rsyslog-relp if needed + become: true + apt: + name: rsyslog-relp + state: present + when: "rsyslog_inputs | selectattr('proto', 'eq', 'relp') | list" + +- name: Ensure log storage directory exists + become: true + file: + path: "{{ rsyslog_collector_base_dir }}" + state: directory + owner: root + group: adm + mode: u=rwx,g=rwx,o= + +- name: Deploy rsyslog input configuration file + become: true + template: + src: 20-collector.conf.j2 + dest: /etc/rsyslog.d/20-collector.conf + owner: root + group: root + mode: u=rw,g=r,o=r + notify: Restart rsyslog + +- name: Install rotate script + become: true + copy: + src: rotate + dest: "{{ rsyslog_collector_rotate_path }}" + owner: root + group: root + mode: u=rwx,g=rx,o= + +- name: Install timer and service for rotate script + become: true + template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: u=rw,g=r,o= + loop: + - rotate-remote-logs.timer + - rotate-remote-logs.service + notify: + - Run systemd daemon-reload + +- name: Enable timer for log rotation + become: true + systemd: + name: rotate-remote-logs.timer + enabled: true + state: started +... diff --git a/roles/rsyslog_collector/templates/20-collector.conf.j2 b/roles/rsyslog_collector/templates/20-collector.conf.j2 new file mode 100644 index 0000000..897945f --- /dev/null +++ b/roles/rsyslog_collector/templates/20-collector.conf.j2 @@ -0,0 +1,54 @@ +{{ ansible_managed | comment }} + +module(load="mmrm1stspace") + +{% + set input_modules = { + "relp": "imrelp", + "udp": "imudp", + } +%} + +{% + for module in rsyslog_inputs + | map(attribute="proto") + | map("extract", input_modules) + | list + | unique +%} +module(load="{{ module }}") +{% endfor %} + +template(name="incomingFilename" type="list") { + constant(value="{{ rsyslog_collector_base_dir }}/") + property(name="fromhost-ip") + constant(value="/") + property(name="timegenerated" dateFormat="year") + constant(value="-") + property(name="timegenerated" dateFormat="month") + constant(value="-") + property(name="timegenerated" dateFormat="day") + constant(value=".log") +} + +ruleset(name="handleIncomingLogs") { + action(type="mmrm1stspace") + action( + type="omfile" + dynaFile="incomingFilename" + template="RSYSLOG_FileFormat" + ) + call sendLogsToRemote +} + +# TODO: add protocol-specific options (eg. TLS) +{% for input in rsyslog_inputs %} +input( + type="{{ input_modules[input.proto] }}" +{% if "address" in input %} + address="{{ input.address }}" +{% endif %} + port="{{ input.port }}" + ruleset="handleIncomingLogs" +) +{% endfor %} diff --git a/roles/rsyslog_collector/templates/rotate-remote-logs.service.j2 b/roles/rsyslog_collector/templates/rotate-remote-logs.service.j2 new file mode 100644 index 0000000..0e75a2b --- /dev/null +++ b/roles/rsyslog_collector/templates/rotate-remote-logs.service.j2 @@ -0,0 +1,12 @@ +{{ ansible_managed | comment }} + +[Unit] +Description=Rotate remote logs + +[Service] +User=root +Type=simple +ExecStart={{ rsyslog_collector_rotate_path }} \ + --base-dir {{ rsyslog_collector_base_dir }} \ + --compress-days {{ rsyslog_collector_compress_days }} \ + --keep-days {{ rsyslog_collector_keep_days }} diff --git a/roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2 b/roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2 new file mode 100644 index 0000000..f4b1151 --- /dev/null +++ b/roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2 @@ -0,0 +1,10 @@ +{{ ansible_managed | comment }} + +[Unit] +Description=Rotate remote logs daily + +[Timer] +OnCalendar=daily + +[Install] +WantedBy=timers.target diff --git a/roles/rsyslog_common/defaults/main.yml b/roles/rsyslog_common/defaults/main.yml new file mode 100644 index 0000000..e5e6024 --- /dev/null +++ b/roles/rsyslog_common/defaults/main.yml @@ -0,0 +1,3 @@ +--- +rsyslog_outputs: [] +... diff --git a/roles/rsyslog_common/handlers/main.yml b/roles/rsyslog_common/handlers/main.yml new file mode 100644 index 0000000..563f2fc --- /dev/null +++ b/roles/rsyslog_common/handlers/main.yml @@ -0,0 +1,13 @@ +--- +- name: Restart rsyslog + become: true + systemd: + name: rsyslog.service + state: restarted + +- name: Restart systemd-journald + become: true + systemd: + name: systemd-journald.service + state: restarted +... diff --git a/roles/rsyslog_common/tasks/main.yml b/roles/rsyslog_common/tasks/main.yml new file mode 100644 index 0000000..030fd10 --- /dev/null +++ b/roles/rsyslog_common/tasks/main.yml @@ -0,0 +1,60 @@ +--- +- name: Install rsyslog + become: true + apt: + name: rsyslog + state: present + +- name: Install rsyslog modules if needed + become: true + apt: + name: "{{ item.pkg }}" + state: present + when: "rsyslog_outputs | selectattr('proto', 'eq', item.proto) | list" + loop: + - proto: relp + pkg: rsyslog-relp + - proto: redis + pkg: rsyslog-hiredis + +- name: Deploy main rsyslog configuration + become: true + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: u=rw,g=r,o=r + loop: + - src: rsyslog.conf.j2 + dest: /etc/rsyslog.conf + - src: 10-common.conf.j2 + dest: /etc/rsyslog.d/10-common.conf + notify: Restart rsyslog + +- name: Create journald.conf.d directory + become: true + file: + path: /etc/systemd/journald.conf.d + state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx + +- name: Deploy journald configuration + become: true + template: + src: forward-syslog.conf.j2 + dest: /etc/systemd/journald.conf.d/forward-syslog.conf + owner: root + group: root + mode: u=rw,g=r,o=r + notify: Restart systemd-journald + +- name: Enable rsyslog service + become: true + systemd: + name: rsyslog.service + state: started + enabled: true +... diff --git a/roles/rsyslog_common/templates/10-common.conf.j2 b/roles/rsyslog_common/templates/10-common.conf.j2 new file mode 100644 index 0000000..36cd4dc --- /dev/null +++ b/roles/rsyslog_common/templates/10-common.conf.j2 @@ -0,0 +1,105 @@ +{{ ansible_managed | comment }} + +{% + set output_modules = { + "relp": "omrelp", + "udp": "omfwd", + "redis": "omhiredis", + } +%} + +global( + workDirectory="/var/spool/rsyslog" + preserveFQDN="on" +) + +# Collect logs via /dev/log +module(load="imuxsock") + +# Collect kernel logs +module(load="imklog") + +# Parse CEE logs +module(load="mmjsonparse") + +# Load export modules +{% + for module in rsyslog_outputs + | map(attribute="proto") + | map("extract", output_modules) + | list + | unique +%} +module(load="{{ module }}") +{% endfor %} + +# FIXME: Attention, il faut voir si rsyslog arrive bien à créer +# les fichiers de plusieurs jours (le 1er est peut-être crée avant +# de dropper les privilèges, mais les suivants je pense pas). +module( + load="builtin:omfile" + # Format avec dates précises + template="RSYSLOG_FileFormat" + fileOwner="root" + fileGroup="adm" + fileCreateMode="0640" + dirCreateMode="0755" +) + +template(name="templateJson" type="list" option.jsonf="on") { + property(outname="hostname_reported" name="hostname" format="jsonf") + property(outname="src" name="fromhost-ip" format="jsonf") + property(outname="facility" name="syslogfacility-text" format="jsonf") + property(outname="program" name="programname" format="jsonf") + property(outname="pid" name="procid" format="jsonf") + property(outname="time_reported" name="timereported" format="jsonf" + dateformat="rfc3339") + property(outname="time_generated" name="timegenerated" format="jsonf" + dateformat="rfc3339") + property(outname="message" name="msg" format="jsonf") +} + +ruleset(name="sendLogsToDisk") { + auth,authpriv.* action(type="omfile" file="/var/log/auth.log") + mail.* action(type="omfile" file="/var/log/mail.log" sync="off") + kern.* action(type="omfile" file="/var/log/kern.log") + *.*;auth,authpriv,mail,kern.none action(type="omfile" + file="/var/log/syslog.log" sync="off") +} + +# Send logs to remote collector(s) +ruleset(name="sendLogsToRemote") { +{% for output in rsyslog_outputs %} + action( + type="{{ output_modules[output.proto] }}" + +{% if output_modules[output.proto] == "omfwd" %} + protocol="{{ output.proto }}" + target="{{ output.address }}" + port="{{ output.port }}" +{% elif output_modules[output.proto] == "omhiredis" %} + server="{{ output.address }}" + serverport="{{ output.port }}" + mode="publish" + key="{{ output.key }}" + template="templateJson" +{% if output.password is defined %} + serverpassword="{{ output.password }}" +{% endif %} +{% elif output_modules[output.proto] == "omrelp" %} + target="{{ output.address }}" + port="{{ output.port }}" +{% endif %} + +{% if loop.index > 1 and output.fallback %} + action.execOnlyWhenPreviousIsSuspended="on" +{% endif %} + ) +{% endfor %} +} + +# Send local logs to files (useful for debugging or if the collector is down) +call sendLogsToDisk + +# Send local logs to the remote collector +call sendLogsToRemote diff --git a/roles/rsyslog_common/templates/forward-syslog.conf.j2 b/roles/rsyslog_common/templates/forward-syslog.conf.j2 new file mode 100644 index 0000000..c332de6 --- /dev/null +++ b/roles/rsyslog_common/templates/forward-syslog.conf.j2 @@ -0,0 +1,5 @@ +{{ ansible_managed | comment }} + +[Journal] +ForwardToSyslog=yes +MaxLevelSyslog=debug diff --git a/roles/rsyslog_common/templates/rsyslog.conf.j2 b/roles/rsyslog_common/templates/rsyslog.conf.j2 new file mode 100644 index 0000000..9c4c687 --- /dev/null +++ b/roles/rsyslog_common/templates/rsyslog.conf.j2 @@ -0,0 +1,3 @@ +{{ ansible_managed | comment }} + +include(file="/etc/rsyslog.d/*.conf")