Merge pull request 'Centralisation des journaux (pas encore Elastic)' (#40) from logs-first-phase into master

Reviewed-on: Aurore/ansible#40

.ansible-lint

@@ -2,6 +2,7 @@ skip_list:
   - no-changed-when
   - load-failure
   - document-start
+  - meta-no-info
 
 warn_list:
   - experimental  # all rules tagged as experimental

group_vars/all/vars.yml

@@ -97,3 +97,9 @@ apartment_block_dhcp: "{{ apartment_block }}"
 ipv6_base_prefix: "2a09:6840"
 
 is_aurore_host: "{{ 'aurore_vm' in group_names }}"
+
+rsyslog_outputs:
+  - proto: relp
+    address: 10.128.0.241
+    port: 20514
+...

host_vars/log.adm.auro.re.yml

@@ -0,0 +1,9 @@
+---
+rsyslog_collector_base_dir: /var/log/remote
+rsyslog_inputs:
+  - proto: relp
+    port: 20514
+  - proto: udp
+    port: 514
+rsyslog_outputs: []
+...

hosts

@@ -36,6 +36,7 @@ wikijs.adm.auro.re
 prometheus-aurore.adm.auro.re
 portail.adm.auro.re
 jitsi-aurore.adm.auro.re
+log.adm.auro.re
 bdd.adm.auro.re
 bdd-ovh.adm.auro.re
 

log.yml

@@ -0,0 +1,5 @@
+---
+- hosts: log.adm.auro.re
+  roles:
+    - rsyslog_collector
+...

roles/nginx/tasks/main.yml

@@ -29,6 +29,24 @@
     dest: "/etc/nginx/sites-enabled/default"
     state: absent
 
+- name: Add 'extended' log format
+  template:
+    src: nginx/conf.d/extended_log.conf.j2
+    dest: /etc/nginx/conf.d/extended_log.conf
+    owner: root
+    group: root
+    mode: 0644
+  notify: Reload nginx
+
+- name: Add syslog snippet
+  template:
+    src: nginx/snippets/syslog.conf.j2
+    dest: /etc/nginx/snippets/syslog.conf
+    owner: root
+    group: root
+    mode: 0644
+  notify: Reload nginx
+
 - name: Copy reverse proxy sites
   when: reverseproxy is defined
   template:

roles/nginx/templates/nginx/conf.d/extended_log.conf.j2

@@ -0,0 +1,7 @@
+{{ ansible_managed | comment }}
+
+log_format extended
+    '$remote_addr - $http_x_forwarded_for - $connection '
+    '$remote_user [$time_local] '
+    '"$host" "$request" $status $body_bytes_sent '
+    '"$http_referer" "$http_user_agent"';

roles/nginx/templates/nginx/sites-available/redirect.j2

@@ -8,6 +8,8 @@ server {
 
     server_name {{ site.from }};
 
+    include "/etc/nginx/snippets/syslog.conf";
+
 {% for realip in nginx.real_ip_from %}
     set_real_ip_from {{ realip }};
 {% endfor %}
@@ -25,6 +27,8 @@ server {
 
     server_name {{ site.from }};
 
+    include "/etc/nginx/snippets/syslog.conf";
+
     # SSL common conf
     include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf";
 
@@ -52,6 +56,8 @@ server {
 
     server_name {{ from }};
 
+    include "/etc/nginx/snippets/syslog.conf";
+
 {% for realip in nginx.real_ip_from %}
     set_real_ip_from {{ realip }};
 {% endfor %}
@@ -72,6 +78,8 @@ server {
     # SSL common conf
     include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf";
 
+    include "/etc/nginx/snippets/syslog.conf";
+
 {% for realip in nginx.real_ip_from %}
     set_real_ip_from {{ realip }};
 {% endfor %}

roles/nginx/templates/nginx/sites-available/reverseproxy.j2

@@ -15,6 +15,8 @@ server {
 
     server_name {{ site.from }};
 
+    include "/etc/nginx/snippets/syslog.conf";
+
 {% for realip in nginx.real_ip_from %}
     set_real_ip_from {{ realip }};
 {% endfor %}
@@ -39,6 +41,8 @@ server {
     access_log      /var/log/nginx/{{ site.from }}.log;
     error_log       /var/log/nginx/{{ site.from }}_error.log;
 
+    include "/etc/nginx/snippets/syslog.conf";
+
     # Keep the TCP connection open a bit for faster browsing
     keepalive_timeout 70;
 

roles/nginx/templates/nginx/sites-available/reverseproxy_redirect_dname.j2

@@ -12,6 +12,8 @@ server {
 
     server_name {{ from }};
 
+    include "/etc/nginx/snippets/syslog.conf";
+
 {% for realip in nginx.real_ip_from %}
     set_real_ip_from {{ realip }};
 {% endfor %}
@@ -29,6 +31,8 @@ server {
 
     server_name {{ from }};
 
+    include "/etc/nginx/snippets/syslog.conf";
+
     # SSL common conf
     include "/etc/nginx/snippets/options-ssl.{{ site.ssl|default(nginx.default_ssl_domain) }}.conf";
 

roles/nginx/templates/nginx/sites-available/service.j2

@@ -19,6 +19,9 @@ upstream {{ upstream.name }} {
 server {
     listen 443 default_server ssl;
     listen [::]:443 default_server ssl;
+
+    include "/etc/nginx/snippets/syslog.conf";
+
     include "/etc/nginx/snippets/options-ssl.{{ nginx.default_ssl_domain }}.conf";
 
     server_name _;
@@ -50,6 +53,8 @@ server {
     # Hide Nginx version
     server_tokens off;
 
+    include "/etc/nginx/snippets/syslog.conf";
+
 {% for realip in nginx.real_ip_from %}
     set_real_ip_from {{ realip }};
 {% endfor %}
@@ -71,6 +76,8 @@ server {
     server_name {{ server.server_name|join(" ") }};
     charset utf-8;
 
+    include "/etc/nginx/snippets/syslog.conf";
+
     # Hide Nginx version
     server_tokens off;
 
@@ -98,6 +105,8 @@ server {
     server_name {{ server.server_name|join(" ") }};
     charset utf-8;
 
+    include "/etc/nginx/snippets/syslog.conf";
+
     # Hide Nginx version
     server_tokens off;
 

roles/nginx/templates/nginx/snippets/syslog.conf.j2

@@ -0,0 +1,4 @@
+{{ ansible_managed | comment }}
+
+access_log syslog:server=unix:/dev/log,tag=nginx,nohostname,severity=info extended;
+error_log syslog:server=unix:/dev/log,tag=nginx,nohostname,severity=error;

roles/router/templates/firewall_config.py

@@ -36,6 +36,11 @@ interfaces_type = {
     'admin' : ['ens18']
 }
 
+log_ignore_v4 = [
+    '224.0.0.0/24',
+    '224.0.1.0/24',
+    '239.0.0.0/8',
+]
 
 ### Specify nat settings: name, interfaces with range, and global range for nat
 ### WARNING : "interface_ip_to_nat' MUST contain /24 ranges, and ip_sources MUST

roles/router/templates/firewall_config_aurore.py

@@ -33,6 +33,12 @@ interfaces_type = {
     'admin' : ['ens19', 'ens20', 'ens23']
 }
 
+log_ignore_v4 = [
+    '224.0.0.0/24',
+    '224.0.1.0/24',
+    '239.0.0.0/8',
+]
+
 ### Specify nat settings: name, interfaces with range, and global range for nat
 ### WARNING : "interface_ip_to_nat' MUST contain /24 ranges, and ip_sources MUST
 ### contain /16 range

roles/router/templates/interfaces-aurore

@@ -11,7 +11,6 @@ iface lo inet loopback
 auto ens18
 iface ens18 inet static
         address 10.129.0.{{ router_hard_ip_suffix }}/16
-        gateway 10.129.0.1
 
 iface ens18 inet6 static
         address 2a09:6840:129::0:{{ router_hard_ip_suffix }}/64

roles/router/templates/keepalived-aurore.conf

@@ -39,7 +39,7 @@ vrrp_instance VI_ROUT_aurore_IPv4 {
         10.129.0.254/16 brd 10.129.255.255 dev ens18 scope global
 
         # Adm
-        10.128.0.254/16 brd 10.129.255.255 dev ens19 scope global
+        10.128.0.254/16 brd 10.128.255.255 dev ens19 scope global
 
         # Switches
         10.130.0.254/16 brd 10.130.255.255 dev ens20 scope global

roles/rsyslog_collector/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+rsyslog_inputs: []
+rsyslog_collector_base_dir: /var/log/remote
+rsyslog_collector_rotate_path: /usr/local/sbin/rotate_remote_logs
+rsyslog_collector_keep_days: 0
+rsyslog_collector_compress_days: 1
+...

roles/rsyslog_collector/files/rotate

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import argparse
+import datetime
+import logging
+import pathlib
+import subprocess
+
+
+def compress_file(filename):
+    subprocess.run(["xz", "-z", str(filename)])
+
+
+def find_files(base_dir, extension, days):
+    delta = datetime.timedelta(days=days)
+    now = datetime.datetime.now()
+    for path in base_dir.rglob(f"*{extension}"):
+        stem = path.name.removesuffix(extension)
+        date = datetime.datetime.fromisoformat(stem)
+        if date < now - delta:
+            yield path
+
+
+def compress_logs(base_dir, days):
+    for path in find_files(base_dir, ".log", days):
+        logging.info("Compressing log file %s", str(path))
+        compress_file(path)
+
+
+def remove_logs(base_dir, days):
+    for path in find_files(base_dir, ".log.xz", days):
+        logging.info("Removing log file %s", str(path))
+        path.unlink()
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--compress-days", type=int, default=0)
+    parser.add_argument("--keep-days", type=int, default=0)
+    parser.add_argument(
+        "--base-dir", type=pathlib.Path, default="/var/log/remote"
+    )
+
+    args = parser.parse_args()
+
+    logging.basicConfig(format="%(levelname)s %(message)s", level=logging.INFO)
+
+    logging.info("Rotate script started")
+
+    if args.compress_days > 0:
+        compress_logs(args.base_dir, args.compress_days)
+
+    if args.keep_days > 0:
+        remove_logs(args.base_dir, args.keep_days)
+
+    logging.info("Rotate script done")
+
+
+if __name__ == "__main__":
+    main()

roles/rsyslog_collector/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Run systemd daemon-reload
+  systemd:
+    daemon_reload: true
+...

roles/rsyslog_collector/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: rsyslog_common
+...

roles/rsyslog_collector/tasks/main.yml

@@ -0,0 +1,57 @@
+---
+- name: Install rsyslog-relp if needed
+  become: true
+  apt:
+    name: rsyslog-relp
+    state: present
+  when: "rsyslog_inputs | selectattr('proto', 'eq', 'relp') | list"
+
+- name: Ensure log storage directory exists
+  become: true
+  file:
+    path: "{{ rsyslog_collector_base_dir }}"
+    state: directory
+    owner: root
+    group: adm
+    mode: u=rwx,g=rwx,o=
+
+- name: Deploy rsyslog input configuration file
+  become: true
+  template:
+    src: 20-collector.conf.j2
+    dest: /etc/rsyslog.d/20-collector.conf
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: Restart rsyslog
+
+- name: Install rotate script
+  become: true
+  copy:
+    src: rotate
+    dest: "{{ rsyslog_collector_rotate_path }}"
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=
+
+- name: Install timer and service for rotate script
+  become: true
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=
+  loop:
+    - rotate-remote-logs.timer
+    - rotate-remote-logs.service
+  notify:
+    - Run systemd daemon-reload
+
+- name: Enable timer for log rotation
+  become: true
+  systemd:
+    name: rotate-remote-logs.timer
+    enabled: true
+    state: started
+...

roles/rsyslog_collector/templates/20-collector.conf.j2

@@ -0,0 +1,54 @@
+{{ ansible_managed | comment }}
+
+module(load="mmrm1stspace")
+
+{%
+    set input_modules = {
+        "relp": "imrelp",
+        "udp": "imudp",
+    }
+%}
+
+{%
+    for module in rsyslog_inputs
+        | map(attribute="proto")
+        | map("extract", input_modules)
+        | list
+        | unique
+%}
+module(load="{{ module }}")
+{% endfor %}
+
+template(name="incomingFilename" type="list") {
+    constant(value="{{ rsyslog_collector_base_dir }}/")
+    property(name="fromhost-ip")
+    constant(value="/")
+    property(name="timegenerated" dateFormat="year")
+    constant(value="-")
+    property(name="timegenerated" dateFormat="month")
+    constant(value="-")
+    property(name="timegenerated" dateFormat="day")
+    constant(value=".log")
+}
+
+ruleset(name="handleIncomingLogs") {
+    action(type="mmrm1stspace")
+    action(
+        type="omfile"
+        dynaFile="incomingFilename"
+        template="RSYSLOG_FileFormat"
+    )
+    call sendLogsToRemote
+}
+
+# TODO: add protocol-specific options (eg. TLS)
+{% for input in rsyslog_inputs %}
+input(
+    type="{{ input_modules[input.proto] }}"
+{% if "address" in input %}
+    address="{{ input.address }}"
+{% endif %}
+    port="{{ input.port }}"
+    ruleset="handleIncomingLogs"
+)
+{% endfor %}

roles/rsyslog_collector/templates/rotate-remote-logs.service.j2

@@ -0,0 +1,12 @@
+{{ ansible_managed | comment }}
+
+[Unit]
+Description=Rotate remote logs
+
+[Service]
+User=root
+Type=simple
+ExecStart={{ rsyslog_collector_rotate_path }} \
+    --base-dir {{ rsyslog_collector_base_dir }} \
+    --compress-days {{ rsyslog_collector_compress_days }} \
+    --keep-days {{ rsyslog_collector_keep_days }}

roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2

@@ -0,0 +1,10 @@
+{{ ansible_managed | comment }}
+
+[Unit]
+Description=Rotate remote logs daily
+
+[Timer]
+OnCalendar=daily
+
+[Install]
+WantedBy=timers.target

roles/rsyslog_common/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+rsyslog_outputs: []
+...

roles/rsyslog_common/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Restart rsyslog
+  become: true
+  systemd:
+    name: rsyslog.service
+    state: restarted
+
+- name: Restart systemd-journald
+  become: true
+  systemd:
+    name: systemd-journald.service
+    state: restarted
+...

roles/rsyslog_common/tasks/main.yml

@@ -0,0 +1,60 @@
+---
+- name: Install rsyslog
+  become: true
+  apt:
+    name: rsyslog
+    state: present
+
+- name: Install rsyslog modules if needed
+  become: true
+  apt:
+    name: "{{ item.pkg }}"
+    state: present
+  when: "rsyslog_outputs | selectattr('proto', 'eq', item.proto) | list"
+  loop:
+    - proto: relp
+      pkg: rsyslog-relp
+    - proto: redis
+      pkg: rsyslog-hiredis
+
+- name: Deploy main rsyslog configuration
+  become: true
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  loop:
+    - src: rsyslog.conf.j2
+      dest: /etc/rsyslog.conf
+    - src: 10-common.conf.j2
+      dest: /etc/rsyslog.d/10-common.conf
+  notify: Restart rsyslog
+
+- name: Create journald.conf.d directory
+  become: true
+  file:
+    path: /etc/systemd/journald.conf.d
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=rx
+
+- name: Deploy journald configuration
+  become: true
+  template:
+    src: forward-syslog.conf.j2
+    dest: /etc/systemd/journald.conf.d/forward-syslog.conf
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify: Restart systemd-journald
+
+- name: Enable rsyslog service
+  become: true
+  systemd:
+    name: rsyslog.service
+    state: started
+    enabled: true
+...

roles/rsyslog_common/templates/10-common.conf.j2

@@ -0,0 +1,105 @@
+{{ ansible_managed | comment }}
+
+{%
+    set output_modules = {
+        "relp": "omrelp",
+        "udp": "omfwd",
+        "redis": "omhiredis",
+    }
+%}
+
+global(
+    workDirectory="/var/spool/rsyslog"
+    preserveFQDN="on"
+)
+
+# Collect logs via /dev/log
+module(load="imuxsock")
+
+# Collect kernel logs
+module(load="imklog")
+
+# Parse CEE logs
+module(load="mmjsonparse")
+
+# Load export modules
+{%
+    for module in rsyslog_outputs
+        | map(attribute="proto")
+        | map("extract", output_modules)
+        | list
+        | unique
+%}
+module(load="{{ module }}")
+{% endfor %}
+
+# FIXME: Attention, il faut voir si rsyslog arrive bien à créer
+# les fichiers de plusieurs jours (le 1er est peut-être crée avant
+# de dropper les privilèges, mais les suivants je pense pas).
+module(
+    load="builtin:omfile"
+    # Format avec dates précises
+    template="RSYSLOG_FileFormat"
+    fileOwner="root"
+    fileGroup="adm"
+    fileCreateMode="0640"
+    dirCreateMode="0755"
+)
+
+template(name="templateJson" type="list" option.jsonf="on") {
+    property(outname="hostname_reported" name="hostname" format="jsonf")
+    property(outname="src" name="fromhost-ip" format="jsonf")
+    property(outname="facility" name="syslogfacility-text" format="jsonf")
+    property(outname="program" name="programname" format="jsonf")
+    property(outname="pid" name="procid" format="jsonf")
+    property(outname="time_reported" name="timereported" format="jsonf"
+        dateformat="rfc3339")
+    property(outname="time_generated" name="timegenerated" format="jsonf"
+        dateformat="rfc3339")
+    property(outname="message" name="msg" format="jsonf")
+}
+
+ruleset(name="sendLogsToDisk") {
+    auth,authpriv.* action(type="omfile" file="/var/log/auth.log")
+    mail.* action(type="omfile" file="/var/log/mail.log" sync="off")
+    kern.* action(type="omfile" file="/var/log/kern.log")
+    *.*;auth,authpriv,mail,kern.none action(type="omfile"
+        file="/var/log/syslog.log" sync="off")
+}
+
+# Send logs to remote collector(s)
+ruleset(name="sendLogsToRemote") {
+{% for output in rsyslog_outputs %}
+    action(
+        type="{{ output_modules[output.proto] }}"
+
+{% if output_modules[output.proto] == "omfwd" %}
+        protocol="{{ output.proto }}"
+        target="{{ output.address }}"
+        port="{{ output.port }}"
+{% elif output_modules[output.proto] == "omhiredis" %}
+        server="{{ output.address }}"
+        serverport="{{ output.port }}"
+        mode="publish"
+        key="{{ output.key }}"
+        template="templateJson"
+{% if output.password is defined %}
+        serverpassword="{{ output.password }}"
+{% endif %}
+{% elif output_modules[output.proto] == "omrelp" %}
+        target="{{ output.address }}"
+        port="{{ output.port }}"
+{% endif %}
+
+{% if loop.index > 1 and output.fallback %}
+        action.execOnlyWhenPreviousIsSuspended="on"
+{% endif %}
+    )
+{% endfor %}
+}
+
+# Send local logs to files (useful for debugging or if the collector is down)
+call sendLogsToDisk
+
+# Send local logs to the remote collector
+call sendLogsToRemote

roles/rsyslog_common/templates/forward-syslog.conf.j2

@@ -0,0 +1,5 @@
+{{ ansible_managed | comment }}
+
+[Journal]
+ForwardToSyslog=yes
+MaxLevelSyslog=debug

roles/rsyslog_common/templates/rsyslog.conf.j2

@@ -0,0 +1,3 @@
+{{ ansible_managed | comment }}
+
+include(file="/etc/rsyslog.d/*.conf")