From c65b3f090b85211c08d26588091b60826640623b Mon Sep 17 00:00:00 2001
From: Jeltz <tom.barthe+aurore@ens-paris-saclay.fr>
Date: Mon, 1 Mar 2021 03:58:58 +0100
Subject: [PATCH] Compress and delete old remote logs

Logrotate is not used because I didn't found an easy way to configure it
to handle the compression/deletion of log files already rotated by
rsyslog (it is probably possible, but I found the script to be easier).
---
 roles/rsyslog_collector/defaults/main.yml     |  3 +
 roles/rsyslog_collector/files/rotate          | 62 +++++++++++++++++++
 roles/rsyslog_collector/handlers/main.yml     |  5 ++
 roles/rsyslog_collector/tasks/main.yml        | 30 +++++++++
 .../templates/rotate-remote-logs.service.j2   | 12 ++++
 .../templates/rotate-remote-logs.timer.j2     | 10 +++
 6 files changed, 122 insertions(+)
 create mode 100644 roles/rsyslog_collector/files/rotate
 create mode 100644 roles/rsyslog_collector/handlers/main.yml
 create mode 100644 roles/rsyslog_collector/templates/rotate-remote-logs.service.j2
 create mode 100644 roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2

diff --git a/roles/rsyslog_collector/defaults/main.yml b/roles/rsyslog_collector/defaults/main.yml
index d0f9337..6ded0ef 100644
--- a/roles/rsyslog_collector/defaults/main.yml
+++ b/roles/rsyslog_collector/defaults/main.yml
@@ -1,4 +1,7 @@
 ---
 rsyslog_inputs: []
 rsyslog_collector_base_dir: /var/log/remote
+rsyslog_collector_rotate_path: /usr/local/sbin/rotate_remote_logs
+rsyslog_collector_keep_days: 0
+rsyslog_collector_compress_days: 1
 ...
diff --git a/roles/rsyslog_collector/files/rotate b/roles/rsyslog_collector/files/rotate
new file mode 100644
index 0000000..8738fef
--- /dev/null
+++ b/roles/rsyslog_collector/files/rotate
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import argparse
+import datetime
+import logging
+import pathlib
+import subprocess
+
+
+def compress_file(filename):
+    subprocess.run(["xz", "-z", str(filename)])
+
+
+def find_files(base_dir, extension, days):
+    delta = datetime.timedelta(days=days)
+    now = datetime.datetime.now()
+    for path in base_dir.rglob(f"*{extension}"):
+        stem = path.name.removesuffix(extension)
+        date = datetime.datetime.fromisoformat(stem)
+        if date < now - delta:
+            yield path
+
+
+def compress_logs(base_dir, days):
+    for path in find_files(base_dir, ".log", days):
+        logging.info("Compressing log file %s", str(path))
+        compress_file(path)
+
+
+def remove_logs(base_dir, days):
+    for path in find_files(base_dir, ".log.xz", days):
+        logging.info("Removing log file %s", str(path))
+        path.unlink()
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--compress-days", type=int, default=0)
+    parser.add_argument("--keep-days", type=int, default=0)
+    parser.add_argument(
+        "--base-dir", type=pathlib.Path, default="/var/log/remote"
+    )
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        format="[%(asctime)s] %(levelname)s %(message)s", level=logging.INFO
+    )
+
+    logging.info("Rotate script started")
+
+    if args.compress_days > 0:
+        compress_logs(args.base_dir, args.compress_days)
+
+    if args.keep_days > 0:
+        remove_logs(args.base_dir, args.keep_days)
+
+    logging.info("Rotate script done")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/roles/rsyslog_collector/handlers/main.yml b/roles/rsyslog_collector/handlers/main.yml
new file mode 100644
index 0000000..60f493a
--- /dev/null
+++ b/roles/rsyslog_collector/handlers/main.yml
@@ -0,0 +1,5 @@
+---
+- name: Run systemd daemon-reload
+  systemd:
+    daemon_reload: true
+...
diff --git a/roles/rsyslog_collector/tasks/main.yml b/roles/rsyslog_collector/tasks/main.yml
index 16a3ab9..0c122e9 100644
--- a/roles/rsyslog_collector/tasks/main.yml
+++ b/roles/rsyslog_collector/tasks/main.yml
@@ -24,4 +24,34 @@
     group: root
     mode: u=rw,g=r,o=r
   notify: Restart rsyslog
+
+- name: Install rotate script
+  become: true
+  copy:
+    src: rotate
+    dest: "{{ rsyslog_collector_rotate_path }}"
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=
+
+- name: Install timer and service for rotate script
+  become: true
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=
+  loop:
+    - rotate-remote-logs.timer
+    - rotate-remote-logs.service
+  notify:
+    - Run systemd daemon-reload
+
+- name: Enable timer for log rotation
+  become: true
+  systemd:
+    name: rotate-remote-logs.timer
+    enabled: true
+    state: started
 ...
diff --git a/roles/rsyslog_collector/templates/rotate-remote-logs.service.j2 b/roles/rsyslog_collector/templates/rotate-remote-logs.service.j2
new file mode 100644
index 0000000..3b915e7
--- /dev/null
+++ b/roles/rsyslog_collector/templates/rotate-remote-logs.service.j2
@@ -0,0 +1,12 @@
+{{ ansible_managed | comment }}
+
+[Unit]
+Description=Rotate remote logs
+
+[Service]
+User=root
+Type=OneShot
+ExecStart={{ rsyslog_collector_rotate_path }} \
+    --base-dir {{ rsyslog_collector_keep_days }} \
+    --compress-days {{ rsyslog_collector_compress_days }} \
+    --keep-days {{ rsyslog_collector_base_dir }}
diff --git a/roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2 b/roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2
new file mode 100644
index 0000000..f4b1151
--- /dev/null
+++ b/roles/rsyslog_collector/templates/rotate-remote-logs.timer.j2
@@ -0,0 +1,10 @@
+{{ ansible_managed | comment }}
+
+[Unit]
+Description=Rotate remote logs daily
+
+[Timer]
+OnCalendar=daily
+
+[Install]
+WantedBy=timers.target